@mailwoman/corpus 10.3.0 → 10.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -17
- package/data/PROVENANCE.md +287 -273
- package/data/address-source-register.json +545 -23
- package/data/license-decisions.json +120 -0
- package/data/licenses/PROVENANCE.md +5 -5
- package/data/licenses/cdla-permissive-2.0.md +6 -6
- package/data/licenses/geonames-publication.md +5 -5
- package/data/licenses/licence-ouverte-2.0.md +3 -3
- package/data/licenses/national-address-database.md +12 -12
- package/data/licenses/nlod-2.0.md +3 -3
- package/data/licenses/sirene-publication.md +13 -6
- package/data/licenses/whosonfirst-licenses.md +4 -4
- package/data/source-resolutions.json +46 -0
- package/data/training-manifests/v0.7.2-address-systems.effective.json +1169 -0
- package/data/training-manifests/v0.7.2-address-systems.json +550 -0
- package/lib/adapters/cordis/adapter.ts +9 -9
- package/lib/adapters/gleif/adapter.ts +7 -8
- package/lib/adapters/po-box/adapter.ts +1 -1
- package/lib/adapters/source-id.ts +1 -1
- package/lib/adapters.ts +4 -2
- package/lib/au/adapters/gnaf/adapter.ts +1 -1
- package/lib/be/adapters/brussels/adapter.ts +3 -3
- package/lib/be/adapters/vlaanderen/adapter.ts +1 -1
- package/lib/be/adapters/wallonie/adapter.ts +1 -1
- package/lib/build/align.ts +1 -1
- package/lib/build.ts +1 -1
- package/lib/cz/adapters/cuzk/adapter.ts +2 -2
- package/lib/cz/tools/fetch/cuzk.ts +6 -6
- package/lib/de/recipes/locale.ts +1 -1
- package/lib/dk/adapters/inspire/adapter.ts +1 -1
- package/lib/ee/tools/fetch/ads.ts +2 -2
- package/lib/es/adapters/bizkaia/adapter.ts +1 -1
- package/lib/es/adapters/navarra/adapter.ts +2 -2
- package/lib/es/cadastre.ts +2 -2
- package/lib/es/tools/fetch/bizkaia.ts +8 -8
- package/lib/es/tools/fetch/catastro.ts +6 -6
- package/lib/es/tools/fetch/navarra.ts +4 -4
- package/lib/fingerprints.ts +3 -3
- package/lib/fr/adapters/finess/adapter.ts +6 -5
- package/lib/fr/adapters/finess/voie-type.ts +1 -1
- package/lib/fr/tools/fetch/annuaire-education.ts +3 -3
- package/lib/gb/adapters/os-open/adapter.ts +453 -0
- package/lib/gb/constituent-country.ts +52 -0
- package/lib/gb/os-open/nearest-link.ts +169 -0
- package/lib/gb/os-open/open-names.ts +180 -0
- package/lib/gb/os-open/open-roads.ts +145 -0
- package/lib/gb/tools/fetch/os-open.ts +251 -0
- package/lib/inspire/address.ts +3 -3
- package/lib/international/recipes/locale.ts +41 -7
- package/lib/it/adapters/anac/adapter.ts +4 -3
- package/lib/jp/adapters/wof-admin/adapter.ts +2 -2
- package/lib/kr/adapters/juso/label-rows.ts +1 -1
- package/lib/kr/adapters/localdata/key-index.ts +2 -1
- package/lib/lu/adapters/bd-adresses/adapter.ts +1 -1
- package/lib/nl/adapters/kadaster/adapter.ts +2 -2
- package/lib/nl/tools/fetch/kadaster.ts +2 -2
- package/lib/parquet/shuffle.ts +2 -2
- package/lib/parquet/span-sql.ts +8 -6
- package/lib/parquet/streams.ts +1 -1
- package/lib/parquet/writers.ts +2 -2
- package/lib/pl/tools/fetch/emuia.ts +2 -2
- package/lib/recipes/scaffold.ts +2 -2
- package/lib/recipes/sources.ts +3 -3
- package/lib/recipes/sub/venue/render.ts +1 -1
- package/lib/recipes/sub/venue/sources.ts +4 -3
- package/lib/recipes/sub/venue.ts +4 -3
- package/lib/recipes/venue.ts +11 -11
- package/lib/registers.ts +7 -0
- package/lib/ro/adapters/retea-scolara/adapter.ts +2 -2
- package/lib/runner.ts +3 -3
- package/lib/sk/tools/fetch/inspire.ts +3 -3
- package/lib/source-register/effective-manifest.ts +14 -8
- package/lib/source-register/training-manifest.ts +30 -2
- package/lib/source-register/types.ts +2 -2
- package/lib/source-register.ts +1 -0
- package/lib/subvenue/harvest.ts +6 -5
- package/lib/subvenue/promotions.ts +3 -3
- package/lib/subvenue/surfaces.ts +1 -1
- package/lib/subvenue/table.ts +6 -6
- package/lib/surfaces/locale.ts +1 -1
- package/lib/synthesizers/anchor-absorption.ts +1 -1
- package/lib/synthesizers/boundary-stress.ts +3 -2
- package/lib/synthesizers/no-street.ts +1 -1
- package/lib/synthesizers/po-box.ts +1 -1
- package/lib/synthesizers/tail.ts +2 -2
- package/lib/test-kit/corpus-recipe.ts +1 -1
- package/lib/tools/align-canonical.ts +3 -2
- package/lib/tools/audit.ts +1 -1
- package/lib/tools/corpus-stats.ts +1 -1
- package/lib/tools/fetch/download/network.ts +1 -1
- package/lib/tools/fetch/geonames/dump.ts +1 -1
- package/lib/tools/fetch/gleif.ts +2 -2
- package/lib/tools/fetch/header.ts +1 -1
- package/lib/tools/fetch/inspire-addresses.ts +1 -1
- package/lib/tools/fetch/vlaanderen-ad.ts +1 -1
- package/lib/tools/fetch/wfs-harvest.ts +6 -6
- package/lib/tools/fetch/zip-archive.ts +2 -2
- package/lib/tools/fetch.ts +2 -0
- package/lib/tools/golden/expand.ts +1 -1
- package/lib/tools/golden/promote.ts +3 -3
- package/lib/tools/golden/relabel-street/directory.ts +2 -1
- package/lib/tools/lint/recipe-output/vocab.ts +2 -2
- package/lib/tools/lint/recipe-output.ts +1 -1
- package/lib/tools/migrate/source-names.ts +1 -1
- package/lib/tools/overlay/manifest.ts +62 -10
- package/lib/tools/overlay/split-slice.ts +2 -10
- package/lib/tools/postcode-triples.ts +2 -2
- package/lib/tools/sub/venue/head-nouns.ts +4 -3
- package/lib/types.ts +3 -3
- package/lib/us/adapters/state/tx-notaries/adapter.ts +1 -1
- package/lib/us/adapters/usgov/irs-bmf/adapter.ts +2 -2
- package/lib/us/adapters/usgov/nad/adapter.ts +1 -1
- package/lib/us/adapters/usgov/nppes/adapter.ts +2 -2
- package/lib/us/synthesizers/intersection.ts +4 -4
- package/lib/us/synthesizers/street.ts +3 -3
- package/lib/us/tools/fetch/nad.ts +2 -2
- package/lib/utils/split.ts +59 -6
- package/lib/utils/wof-json.ts +1 -1
- package/out/adapters/cordis/adapter.d.ts +7 -7
- package/out/adapters/cordis/adapter.js +9 -9
- package/out/adapters/cordis/adapter.js.map +1 -1
- package/out/adapters/gleif/adapter.d.ts +7 -8
- package/out/adapters/gleif/adapter.d.ts.map +1 -1
- package/out/adapters/gleif/adapter.js +7 -8
- package/out/adapters/gleif/adapter.js.map +1 -1
- package/out/adapters/po-box/adapter.d.ts +1 -1
- package/out/adapters/source-id.d.ts +1 -1
- package/out/adapters/source-id.js +1 -1
- package/out/adapters.d.ts +2 -2
- package/out/adapters.d.ts.map +1 -1
- package/out/adapters.js +4 -2
- package/out/adapters.js.map +1 -1
- package/out/au/adapters/gnaf/adapter.js +1 -1
- package/out/be/adapters/brussels/adapter.d.ts +2 -2
- package/out/be/adapters/brussels/adapter.js +3 -3
- package/out/be/adapters/vlaanderen/adapter.d.ts +1 -1
- package/out/be/adapters/wallonie/adapter.d.ts +1 -1
- package/out/be/adapters/wallonie/adapter.js +1 -1
- package/out/build/align.d.ts +1 -1
- package/out/build.js +1 -1
- package/out/build.js.map +1 -1
- package/out/cz/adapters/cuzk/adapter.d.ts +2 -2
- package/out/cz/adapters/cuzk/adapter.js +2 -2
- package/out/cz/tools/fetch/cuzk.d.ts +2 -2
- package/out/cz/tools/fetch/cuzk.js +4 -4
- package/out/cz/tools/fetch/cuzk.js.map +1 -1
- package/out/de/recipes/locale.d.ts +1 -1
- package/out/de/recipes/locale.js +1 -1
- package/out/dk/adapters/inspire/adapter.js +1 -1
- package/out/ee/tools/fetch/ads.d.ts +2 -2
- package/out/ee/tools/fetch/ads.js +1 -1
- package/out/es/adapters/bizkaia/adapter.js +1 -1
- package/out/es/adapters/navarra/adapter.d.ts +2 -2
- package/out/es/adapters/navarra/adapter.js +2 -2
- package/out/es/cadastre.d.ts +2 -2
- package/out/es/cadastre.js +2 -2
- package/out/es/tools/fetch/bizkaia.d.ts +5 -5
- package/out/es/tools/fetch/bizkaia.js +6 -6
- package/out/es/tools/fetch/bizkaia.js.map +1 -1
- package/out/es/tools/fetch/catastro.d.ts +4 -4
- package/out/es/tools/fetch/catastro.js +2 -2
- package/out/es/tools/fetch/navarra.d.ts +2 -2
- package/out/es/tools/fetch/navarra.js +2 -2
- package/out/fingerprints.d.ts +3 -3
- package/out/fingerprints.js +3 -3
- package/out/fr/adapters/finess/adapter.d.ts +6 -5
- package/out/fr/adapters/finess/adapter.d.ts.map +1 -1
- package/out/fr/adapters/finess/adapter.js +6 -5
- package/out/fr/adapters/finess/adapter.js.map +1 -1
- package/out/fr/adapters/finess/voie-type.d.ts +1 -1
- package/out/fr/adapters/finess/voie-type.js +1 -1
- package/out/fr/tools/fetch/annuaire-education.d.ts +3 -3
- package/out/fr/tools/fetch/annuaire-education.js +3 -3
- package/out/gb/adapters/os-open/adapter.d.ts +165 -0
- package/out/gb/adapters/os-open/adapter.d.ts.map +1 -0
- package/out/gb/adapters/os-open/adapter.js +298 -0
- package/out/gb/adapters/os-open/adapter.js.map +1 -0
- package/out/gb/constituent-country.d.ts +36 -0
- package/out/gb/constituent-country.d.ts.map +1 -0
- package/out/gb/constituent-country.js +30 -0
- package/out/gb/constituent-country.js.map +1 -0
- package/out/gb/os-open/nearest-link.d.ts +73 -0
- package/out/gb/os-open/nearest-link.d.ts.map +1 -0
- package/out/gb/os-open/nearest-link.js +133 -0
- package/out/gb/os-open/nearest-link.js.map +1 -0
- package/out/gb/os-open/open-names.d.ts +92 -0
- package/out/gb/os-open/open-names.d.ts.map +1 -0
- package/out/gb/os-open/open-names.js +107 -0
- package/out/gb/os-open/open-names.js.map +1 -0
- package/out/gb/os-open/open-roads.d.ts +80 -0
- package/out/gb/os-open/open-roads.d.ts.map +1 -0
- package/out/gb/os-open/open-roads.js +79 -0
- package/out/gb/os-open/open-roads.js.map +1 -0
- package/out/gb/tools/fetch/os-open.d.ts +91 -0
- package/out/gb/tools/fetch/os-open.d.ts.map +1 -0
- package/out/gb/tools/fetch/os-open.js +166 -0
- package/out/gb/tools/fetch/os-open.js.map +1 -0
- package/out/inspire/address.d.ts +3 -3
- package/out/inspire/address.js +3 -3
- package/out/international/recipes/locale.d.ts +6 -0
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +29 -7
- package/out/international/recipes/locale.js.map +1 -1
- package/out/it/adapters/anac/adapter.d.ts +1 -1
- package/out/it/adapters/anac/adapter.d.ts.map +1 -1
- package/out/it/adapters/anac/adapter.js +4 -3
- package/out/it/adapters/anac/adapter.js.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.d.ts +2 -2
- package/out/jp/adapters/wof-admin/adapter.js +2 -2
- package/out/kr/adapters/juso/label-rows.d.ts +1 -1
- package/out/kr/adapters/localdata/key-index.d.ts +2 -1
- package/out/kr/adapters/localdata/key-index.d.ts.map +1 -1
- package/out/kr/adapters/localdata/key-index.js +2 -1
- package/out/kr/adapters/localdata/key-index.js.map +1 -1
- package/out/lu/adapters/bd-adresses/adapter.d.ts +1 -1
- package/out/lu/adapters/bd-adresses/adapter.js +1 -1
- package/out/nl/adapters/kadaster/adapter.js +2 -2
- package/out/nl/tools/fetch/kadaster.d.ts +2 -2
- package/out/nl/tools/fetch/kadaster.js +2 -2
- package/out/parquet/shuffle.d.ts +2 -2
- package/out/parquet/shuffle.js +2 -2
- package/out/parquet/span-sql.d.ts +8 -6
- package/out/parquet/span-sql.d.ts.map +1 -1
- package/out/parquet/span-sql.js +8 -6
- package/out/parquet/span-sql.js.map +1 -1
- package/out/parquet/streams.d.ts +1 -1
- package/out/parquet/writers.d.ts +2 -2
- package/out/parquet/writers.js +1 -1
- package/out/pl/tools/fetch/emuia.d.ts +2 -2
- package/out/recipes/scaffold.d.ts +2 -2
- package/out/recipes/sources.d.ts +2 -2
- package/out/recipes/sources.js +3 -3
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/sub/venue/render.d.ts +1 -1
- package/out/recipes/sub/venue/render.js +1 -1
- package/out/recipes/sub/venue/sources.d.ts +4 -3
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +4 -3
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts +3 -2
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +2 -2
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/venue.d.ts +7 -6
- package/out/recipes/venue.d.ts.map +1 -1
- package/out/recipes/venue.js +11 -10
- package/out/recipes/venue.js.map +1 -1
- package/out/registers.d.ts +7 -0
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +7 -0
- package/out/registers.js.map +1 -1
- package/out/ro/adapters/retea-scolara/adapter.d.ts +2 -2
- package/out/ro/adapters/retea-scolara/adapter.js +2 -2
- package/out/runner.d.ts +3 -3
- package/out/runner.js +2 -2
- package/out/sk/tools/fetch/inspire.d.ts +3 -3
- package/out/sk/tools/fetch/inspire.js +1 -1
- package/out/source-register/effective-manifest.d.ts +11 -5
- package/out/source-register/effective-manifest.d.ts.map +1 -1
- package/out/source-register/effective-manifest.js +9 -4
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/training-manifest.d.ts +27 -0
- package/out/source-register/training-manifest.d.ts.map +1 -1
- package/out/source-register/training-manifest.js +13 -1
- package/out/source-register/training-manifest.js.map +1 -1
- package/out/source-register/types.d.ts +2 -2
- package/out/source-register.d.ts +1 -1
- package/out/source-register.d.ts.map +1 -1
- package/out/source-register.js +1 -1
- package/out/source-register.js.map +1 -1
- package/out/subvenue/harvest.d.ts +4 -3
- package/out/subvenue/harvest.d.ts.map +1 -1
- package/out/subvenue/harvest.js +6 -5
- package/out/subvenue/harvest.js.map +1 -1
- package/out/subvenue/promotions.d.ts +2 -2
- package/out/subvenue/promotions.js +3 -3
- package/out/subvenue/promotions.js.map +1 -1
- package/out/subvenue/surfaces.d.ts +1 -1
- package/out/subvenue/table.d.ts +6 -6
- package/out/subvenue/table.js +1 -1
- package/out/surfaces/locale.d.ts +1 -1
- package/out/surfaces/locale.js +1 -1
- package/out/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/synthesizers/boundary-stress.js +2 -1
- package/out/synthesizers/boundary-stress.js.map +1 -1
- package/out/synthesizers/tail.d.ts +2 -2
- package/out/synthesizers/tail.js +2 -2
- package/out/test-kit/corpus-recipe.d.ts +1 -1
- package/out/test-kit/corpus-recipe.js +1 -1
- package/out/tools/align-canonical.d.ts.map +1 -1
- package/out/tools/align-canonical.js +3 -2
- package/out/tools/align-canonical.js.map +1 -1
- package/out/tools/audit.js +1 -1
- package/out/tools/corpus-stats.d.ts +1 -1
- package/out/tools/fetch/download/network.d.ts +1 -1
- package/out/tools/fetch/download/network.js +1 -1
- package/out/tools/fetch/geonames/dump.d.ts +1 -1
- package/out/tools/fetch/geonames/dump.js +1 -1
- package/out/tools/fetch/gleif.d.ts +2 -2
- package/out/tools/fetch/gleif.js +1 -1
- package/out/tools/fetch/header.d.ts +1 -1
- package/out/tools/fetch/header.js +1 -1
- package/out/tools/fetch/inspire-addresses.d.ts +1 -1
- package/out/tools/fetch/vlaanderen-ad.d.ts +1 -1
- package/out/tools/fetch/wfs-harvest.d.ts +3 -3
- package/out/tools/fetch/wfs-harvest.js +4 -4
- package/out/tools/fetch/wfs-harvest.js.map +1 -1
- package/out/tools/fetch/zip-archive.d.ts +2 -2
- package/out/tools/fetch/zip-archive.js +2 -2
- package/out/tools/fetch.d.ts +2 -0
- package/out/tools/fetch.d.ts.map +1 -1
- package/out/tools/fetch.js +2 -0
- package/out/tools/fetch.js.map +1 -1
- package/out/tools/golden/expand.js +1 -1
- package/out/tools/golden/expand.js.map +1 -1
- package/out/tools/golden/promote.d.ts +1 -1
- package/out/tools/golden/promote.js +3 -3
- package/out/tools/golden/promote.js.map +1 -1
- package/out/tools/golden/relabel-street/directory.d.ts +2 -1
- package/out/tools/golden/relabel-street/directory.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/directory.js.map +1 -1
- package/out/tools/lint/recipe-output/vocab.js +2 -2
- package/out/tools/lint/recipe-output.d.ts +1 -1
- package/out/tools/migrate/source-names.d.ts +1 -1
- package/out/tools/overlay/manifest.d.ts +15 -2
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +41 -10
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/overlay/split-slice.d.ts.map +1 -1
- package/out/tools/overlay/split-slice.js +2 -9
- package/out/tools/overlay/split-slice.js.map +1 -1
- package/out/tools/postcode-triples.d.ts +1 -1
- package/out/tools/postcode-triples.js +2 -2
- package/out/tools/postcode-triples.js.map +1 -1
- package/out/tools/sub/venue/head-nouns.d.ts +1 -1
- package/out/tools/sub/venue/head-nouns.d.ts.map +1 -1
- package/out/tools/sub/venue/head-nouns.js +4 -3
- package/out/tools/sub/venue/head-nouns.js.map +1 -1
- package/out/types.d.ts +3 -3
- package/out/types.js +1 -1
- package/out/us/adapters/state/tx-notaries/adapter.d.ts +1 -1
- package/out/us/adapters/state/tx-notaries/adapter.js +1 -1
- package/out/us/adapters/usgov/irs-bmf/adapter.d.ts +1 -1
- package/out/us/adapters/usgov/irs-bmf/adapter.js +2 -2
- package/out/us/adapters/usgov/nad/adapter.d.ts +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +1 -1
- package/out/us/adapters/usgov/nppes/adapter.d.ts +1 -1
- package/out/us/adapters/usgov/nppes/adapter.js +1 -1
- package/out/us/synthesizers/intersection.js +4 -4
- package/out/us/synthesizers/intersection.js.map +1 -1
- package/out/us/synthesizers/street.js +3 -3
- package/out/us/synthesizers/street.js.map +1 -1
- package/out/us/tools/fetch/nad.d.ts +2 -2
- package/out/us/tools/fetch/nad.js +2 -2
- package/out/utils/split.d.ts +21 -0
- package/out/utils/split.d.ts.map +1 -1
- package/out/utils/split.js +49 -5
- package/out/utils/split.js.map +1 -1
- package/out/utils/wof-json.d.ts +1 -1
- package/out/utils/wof-json.js +1 -1
- package/package.json +15 -14
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
* public bodies, and the one role whose parties can be sole traders publishes no address.
|
|
30
30
|
* {@linkcode readANACParty} still applies a name check and refuses a party whose name takes the
|
|
31
31
|
* `Surname, Firstname` form, because a later edition could add one. On the 2025 edition the check
|
|
32
|
-
* refuses no party: 0 of the 18,740
|
|
32
|
+
* refuses no party: 0 of the 18,740 buyer and payer parties that have an address take that form, while 56
|
|
33
33
|
* hold a comma inside an organization name — `PARCO NAZIONALE DEL CILENTO, VALLO DI DIANO E
|
|
34
34
|
* ALBURNI` and `SETTORE PATRIMONIO, AMBIENTE E LAVORI PUBBLICI` among them. The check therefore
|
|
35
35
|
* requires one word on each side of the comma rather than any comma.
|
|
@@ -304,8 +304,9 @@ const KILOMETRE_POINT = /[\s,]+km\.?[\s,]*\d+(?:[.,]\d+)?$/iu
|
|
|
304
304
|
/**
|
|
305
305
|
* A name of the form `Surname, Firstname`: one comma, one word on each side, letters only.
|
|
306
306
|
*
|
|
307
|
-
* An organization name in this release
|
|
308
|
-
* `PARCO NAZIONALE DEL CILENTO, VALLO DI DIANO E ALBURNI`
|
|
307
|
+
* An organization name in this release can hold a comma, but a name with a single word on
|
|
308
|
+
* each side of the comma is rare: `PARCO NAZIONALE DEL CILENTO, VALLO DI DIANO E ALBURNI`
|
|
309
|
+
* has eight words after the comma.
|
|
309
310
|
* The single word on each side is what the inverted personal form has
|
|
310
311
|
* and what a department or authority name does not.
|
|
311
312
|
*/
|
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
* 1. **Reversed ordering** — region → county → locality → block → house_number "東京都世田谷区南町1-2-3" not
|
|
14
14
|
* "1-2-3 Minamicho, Setagaya, Tokyo"
|
|
15
15
|
* 2. **No street names** — most JP addresses use a block/sub-block grid system. The "neighborhood"
|
|
16
|
-
* placetype (丁目) is the closest analog to a street but is
|
|
17
|
-
* 3. **Postcode-first convention** — addresses
|
|
16
|
+
* placetype (丁目) is the closest analog to a street but is a grid cell.
|
|
17
|
+
* 3. **Postcode-first convention** — addresses can be prefixed with `〒NNN-nnnn`.
|
|
18
18
|
*
|
|
19
19
|
* Schema mapping to ComponentTags (subset of Stage 3 + JP-specific Phase 6 tags):
|
|
20
20
|
*
|
|
@@ -152,7 +152,7 @@ function joinNumber(main: string, sub: string): string {
|
|
|
152
152
|
|
|
153
153
|
export interface ReadJusoOptions {
|
|
154
154
|
/**
|
|
155
|
-
* Stop after this
|
|
155
|
+
* Stop after this number of rows per 시도, for a fixture or a smoke run.
|
|
156
156
|
* Omit for the whole register.
|
|
157
157
|
*/
|
|
158
158
|
maxRowsPerRegion?: number
|
|
@@ -142,7 +142,8 @@ export function aliasKeyIndex(index: KeyIndex): void {
|
|
|
142
142
|
}
|
|
143
143
|
|
|
144
144
|
/**
|
|
145
|
-
*
|
|
145
|
+
* The number of tokens from `start` that form one of the region's 시군구 —
|
|
146
|
+
* 2 for `수원시 장안구`, 1 for `종로구`, 0 for none.
|
|
146
147
|
*
|
|
147
148
|
* Two-token first, because `수원시` by itself is also listed and taking it would
|
|
148
149
|
* leave `장안구` to be read as a road.
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
* Measured over the 179,622 rows of the 2026-09-28 edition with a generic-word probe rather than a
|
|
34
34
|
* language classifier: 133,449 values open with a French street generic such as `Rue`, `Avenue` or
|
|
35
35
|
* `Montée`, 18,629 close with a Luxembourgish one such as `-strooss`, `-wee` or `-gaass`, and the
|
|
36
|
-
* remaining 27,544 match neither probe and include
|
|
36
|
+
* remaining 27,544 match neither probe and include Luxembourgish forms such as
|
|
37
37
|
* `Am Stengber`, `An Hiesel` and `Um Reebou`.
|
|
38
38
|
* No column marks a row's language, so `locale` reads `und-LU`.
|
|
39
39
|
* `und` is BCP-47's undetermined language subtag, and the repository already stores it in
|
|
@@ -360,8 +360,8 @@ function documentChunks(documentPath: PathBuilderLike): AsyncIterable<Uint8Array
|
|
|
360
360
|
/**
|
|
361
361
|
* Every document under `inputPath`, where `inputPath` is one GML file or a directory holding them.
|
|
362
362
|
*
|
|
363
|
-
* A directory is read because the download is one national file
|
|
364
|
-
* service document could list more than one entry
|
|
363
|
+
* A directory is read because the download is one national file now and a later
|
|
364
|
+
* service document could list more than one entry.
|
|
365
365
|
* A directory holding no document raises rather than yielding zero rows, so a wrong
|
|
366
366
|
* path reports itself instead of reading as an empty publisher.
|
|
367
367
|
*/
|
|
@@ -54,12 +54,12 @@ export const NL_KADASTER_FEED_URL = "https://service.pdok.nl/kadaster/ad/atom/ad
|
|
|
54
54
|
* The adapter's `inputPath` states this name.
|
|
55
55
|
*
|
|
56
56
|
* The publisher's own name for the file.
|
|
57
|
-
* The adapter reads a `.gml.gz` by inflating it, so the suffix is
|
|
57
|
+
* The adapter reads a `.gml.gz` by inflating it, so the suffix is required.
|
|
58
58
|
*/
|
|
59
59
|
export const NL_KADASTER_DOWNLOAD_FILENAME = "addresses.gml.gz"
|
|
60
60
|
|
|
61
61
|
/**
|
|
62
|
-
* The
|
|
62
|
+
* The manifest records this attribution.
|
|
63
63
|
*
|
|
64
64
|
* CC0 1.0 reserves no act and owes no attribution clause.
|
|
65
65
|
* This attribution is a credit the repository chooses, rather than a condition of the grant.
|
package/lib/parquet/shuffle.ts
CHANGED
|
@@ -10,8 +10,8 @@ import { shuffleWith } from "@mailwoman/core/random"
|
|
|
10
10
|
/**
|
|
11
11
|
* Rows held in memory while shuffling, when the caller names no window.
|
|
12
12
|
*
|
|
13
|
-
* Five 50,000-row groups span more than one output row-group and reach `wof-admin`'s
|
|
14
|
-
*
|
|
13
|
+
* Five 50,000-row groups span more than one output row-group and reach `wof-admin`'s whole
|
|
14
|
+
* 13-country set, while a window below `ROW_GROUP_SIZE` draws from no more than one row-group.
|
|
15
15
|
*/
|
|
16
16
|
export const DEFAULT_SHUFFLE_WINDOW = 250_000
|
|
17
17
|
|
package/lib/parquet/span-sql.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* A parquet row stores component offsets in `span_starts` and `span_ends` arrays.
|
|
8
8
|
* A third array, `span_tags`, identifies each component. The text comes from slicing `raw` at those offsets.
|
|
9
9
|
* A query that groups or filters by a
|
|
10
|
-
* component
|
|
10
|
+
* component takes that substring in SQL instead of decoding every row in the process.
|
|
11
11
|
*
|
|
12
12
|
* `holdoutComponents` in `#tools/overlay/split-slice` performs the same read in JavaScript.
|
|
13
13
|
* The SQL and JavaScript operations use different string indexing. The difference is described below.
|
|
@@ -24,8 +24,8 @@
|
|
|
24
24
|
* `alignRow` records a span in UTF-16 code units.
|
|
25
25
|
*
|
|
26
26
|
* DuckDB's `substr` counts characters.
|
|
27
|
-
* The two agree for the Basic Multilingual Plane and diverge by one unit per astral character,
|
|
28
|
-
*
|
|
27
|
+
* The two agree for the Basic Multilingual Plane and diverge by one unit per astral character, so in a
|
|
28
|
+
* row that contains one, the substring starts late by the number of astral characters before the span.
|
|
29
29
|
*
|
|
30
30
|
* In the first 40 train files of `v0.6.0-register-surface`, 6,982 of 40,000,000
|
|
31
31
|
* rows (0.0175%) contain an astral character.
|
|
@@ -34,15 +34,17 @@
|
|
|
34
34
|
* `𐍀𐍂𐍉𐍆𐌹𐌳𐌰𐌹𐌽𐍃, RHODE ISLAND` read its region as `ND`, taken from `ISLAND`.
|
|
35
35
|
*
|
|
36
36
|
* So this is sound for an aggregate over millions of rows and unsound for a decision about one row.
|
|
37
|
-
* A caller that acts per row narrows with this and confirms with `holdoutComponents
|
|
38
|
-
*
|
|
37
|
+
* A caller that acts per row narrows with this and confirms with `holdoutComponents`.
|
|
38
|
+
*
|
|
39
|
+
* That function indexes with JavaScript string semantics and therefore reads
|
|
40
|
+
* the offsets as they were written.
|
|
39
41
|
*/
|
|
40
42
|
export function componentAtSpanSQL(tag: string, rawColumn = "raw"): string {
|
|
41
43
|
return componentAtIndexSQL(`list_position(span_tags, '${tag}')`, rawColumn)
|
|
42
44
|
}
|
|
43
45
|
|
|
44
46
|
/**
|
|
45
|
-
* The same
|
|
47
|
+
* The same substring against a span index the caller already computed.
|
|
46
48
|
*
|
|
47
49
|
* A query can hoist `list_position(span_tags, …)` into a subquery column and read it by name.
|
|
48
50
|
* A second evaluation in the projection would run the search twice per row.
|
package/lib/parquet/streams.ts
CHANGED
package/lib/parquet/writers.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* @file Writes Parquet files and the manifest that describes their contents.
|
|
6
6
|
*
|
|
7
7
|
* This module exposes two writers. {@linkcode writeParquetFile} writes one file from rows already in memory.
|
|
8
|
-
* {@linkcode writeParquetSplits} streams per-split row iterables into
|
|
8
|
+
* {@linkcode writeParquetSplits} streams per-split row iterables into the number of files the row cap requires. It
|
|
9
9
|
* returns the manifest. Both writers create their parent directory, as `@mailwoman/core/fs` writers do
|
|
10
10
|
* encode rather than repeat at each call site.
|
|
11
11
|
*
|
|
@@ -212,7 +212,7 @@ export interface WriteParquetSplitsOptions {
|
|
|
212
212
|
* A descriptor is appended only after its `.parquet` file closes, so each recorded file is complete.
|
|
213
213
|
*
|
|
214
214
|
* A resumed run rereads each descriptor's size.
|
|
215
|
-
* It discards that
|
|
215
|
+
* It discards that number of rows from the front of each split's iterable and opens the next file index.
|
|
216
216
|
*
|
|
217
217
|
* The caller therefore has to supply the same iterables in the same order.
|
|
218
218
|
* For `buildCorpus` that means the same labeled files and the same shuffle seed.
|
|
@@ -155,7 +155,7 @@ export interface HarvestEMUiAOptions {
|
|
|
155
155
|
*/
|
|
156
156
|
pageSize?: number
|
|
157
157
|
/**
|
|
158
|
-
* Stop once the manifest holds this
|
|
158
|
+
* Stop once the manifest holds this number of pages, for a probe rather than a full harvest.
|
|
159
159
|
*/
|
|
160
160
|
maxPages?: number
|
|
161
161
|
/**
|
|
@@ -165,7 +165,7 @@ export interface HarvestEMUiAOptions {
|
|
|
165
165
|
* The measurement costs about 2·log2(n) requests — 51 for a register of this size — and the
|
|
166
166
|
* harvest does not need it: paging to the first empty page reads the type whole either way.
|
|
167
167
|
*
|
|
168
|
-
* It
|
|
168
|
+
* It supplies a denominator for the progress lines and a figure for the manifest,
|
|
169
169
|
* and it rests on the monotone-presence assumption this module's header states.
|
|
170
170
|
*/
|
|
171
171
|
measureCount?: boolean
|
package/lib/recipes/scaffold.ts
CHANGED
|
@@ -170,7 +170,7 @@ export interface OATupleFields {
|
|
|
170
170
|
|
|
171
171
|
export interface ReadOATuplesOptions<T> {
|
|
172
172
|
/**
|
|
173
|
-
* Stop after this
|
|
173
|
+
* Stop after this number of distinct tuples.
|
|
174
174
|
*
|
|
175
175
|
* The `break` closes the reader and releases the archive.
|
|
176
176
|
* GB-scale countrywide extracts need this behavior.
|
|
@@ -496,7 +496,7 @@ export interface RecipeOptions {
|
|
|
496
496
|
*/
|
|
497
497
|
venueNames?: PathBuilderLike
|
|
498
498
|
/**
|
|
499
|
-
* `venue`: share of locality rows that also
|
|
499
|
+
* `venue`: share of locality rows that also print the tuple's region after the locality.
|
|
500
500
|
*
|
|
501
501
|
* @defaultValue `0.5`
|
|
502
502
|
*/
|
package/lib/recipes/sources.ts
CHANGED
|
@@ -285,7 +285,7 @@ export const CARRIED_SOURCES: ReadonlyArray<string> = [
|
|
|
285
285
|
"osm-bd",
|
|
286
286
|
// One id per French overseas jurisdiction for the BAN department files,
|
|
287
287
|
// written by `corpus run ban --source-name`.
|
|
288
|
-
// Metropolitan rows keep `ban`, so its weight does not decide
|
|
288
|
+
// Metropolitan rows keep `ban`, so its weight does not decide the rate at which a territory is drawn.
|
|
289
289
|
"ban-gp",
|
|
290
290
|
"ban-mq",
|
|
291
291
|
"ban-gf",
|
|
@@ -371,7 +371,7 @@ export function defaultRecipeSource(retired: string): string {
|
|
|
371
371
|
*
|
|
372
372
|
* Its `source-names.json` records the mapping applied per file with the row count
|
|
373
373
|
* and the ordered-`source_id` digest verified on both sides.
|
|
374
|
-
* `v0.7.0-de-holdout/corpus-v0.7.0-de-holdout` is assembled from those bytes over 766
|
|
374
|
+
* `v0.7.0-de-holdout/corpus-v0.7.0-de-holdout` is assembled from those bytes over 766 recipe-output files.
|
|
375
375
|
*
|
|
376
376
|
* A recipe writing the retired spelling from here on would produce an output
|
|
377
377
|
* disagreeing with the corpus it joins.
|
|
@@ -391,7 +391,7 @@ export const WRITE_CURRENT_SOURCE_NAMES = true
|
|
|
391
391
|
* throw on every row in the corpus.
|
|
392
392
|
*
|
|
393
393
|
* Callers of those two tables resolve the key through this, so a lookup answers under
|
|
394
|
-
* either spelling and a
|
|
394
|
+
* either spelling and a source unknown under both spellings still refuses.
|
|
395
395
|
*
|
|
396
396
|
* The value is returned unchanged for a source outside the table — an adapter id,
|
|
397
397
|
* or an overlay source id — because those tables are keyed by whatever the producer emits.
|
|
@@ -31,7 +31,7 @@ export type Group = Piece[]
|
|
|
31
31
|
/**
|
|
32
32
|
* The case and punctuation style of a rendered row.
|
|
33
33
|
*
|
|
34
|
-
*
|
|
34
|
+
* Some users type addresses in lowercase, so every recipe output includes lowercase rows.
|
|
35
35
|
*/
|
|
36
36
|
export const Register = {
|
|
37
37
|
Canonical: "canonical",
|
|
@@ -330,7 +330,7 @@ const MIN_OWN_SHAPE_OBSERVATIONS = 20
|
|
|
330
330
|
/**
|
|
331
331
|
* Builds a region's identifier model from the lexicon's `identifierShapes`.
|
|
332
332
|
*
|
|
333
|
-
* The pooled fallback combines the gate, terminal and campus distributions.
|
|
333
|
+
* The pooled fallback combines the airport gate, terminal and campus distributions.
|
|
334
334
|
*/
|
|
335
335
|
export function buildIdentifierModel(lexicon: SubVenueLexiconTable, region: string): IdentifierModel {
|
|
336
336
|
const byDesignator = new Map<string, ShapeBucket[]>()
|
|
@@ -515,8 +515,9 @@ const POI_CONFOUND_CATEGORIES: readonly string[] = [
|
|
|
515
515
|
]
|
|
516
516
|
|
|
517
517
|
/**
|
|
518
|
-
* Reads the venue and confound name pools for one country from the Overture place-names file
|
|
519
|
-
*
|
|
518
|
+
* Reads the venue and confound name pools for one country from the Overture place-names file.
|
|
519
|
+
*
|
|
520
|
+
* That file holds `name`, `category` and `country` for every country the release covers.
|
|
520
521
|
*
|
|
521
522
|
* The venue categories fill the `venues` pool.
|
|
522
523
|
* Every row of a venue or confound category is tested against the rejected phrases
|
package/lib/recipes/sub/venue.ts
CHANGED
|
@@ -52,7 +52,8 @@ export interface SubVenueLeg {
|
|
|
52
52
|
/**
|
|
53
53
|
* The ISO 3166-1 alpha-2 key into the lexicon's `identifierShapes`.
|
|
54
54
|
*
|
|
55
|
-
*
|
|
55
|
+
* Airport gate and terminal numbering differs by country, so each leg samples
|
|
56
|
+
* identifiers from its own region.
|
|
56
57
|
*/
|
|
57
58
|
region: string
|
|
58
59
|
/**
|
|
@@ -142,7 +143,7 @@ export const SUBVENUE_LEGS: readonly SubVenueLeg[] = [
|
|
|
142
143
|
* The region whose identifier distribution the en-US leg uses.
|
|
143
144
|
*
|
|
144
145
|
* The en-US leg has no OSM extract.
|
|
145
|
-
* The Overture names file
|
|
146
|
+
* The Overture names file holds venue names without refs.
|
|
146
147
|
* The recipe borrows the identifier distribution from GB.
|
|
147
148
|
*/
|
|
148
149
|
export const US_IDENTIFIER_REGION_BORROWED_FROM = "GB"
|
|
@@ -514,7 +515,7 @@ function emitPositives(
|
|
|
514
515
|
const subGroup: Group = [{ text: form.text, tag: "unit" }]
|
|
515
516
|
const venueGroup: Group = [{ text: venue, tag: "venue" }]
|
|
516
517
|
const r = random()
|
|
517
|
-
// Some rows omit the street because venue addresses such as airport terminals
|
|
518
|
+
// Some rows omit the street because venue addresses such as airport terminals can lack one.
|
|
518
519
|
const body = addressGroups(leg.country, tuple, r >= NO_STREET_SHARE)
|
|
519
520
|
|
|
520
521
|
// An empty body means that no address layout covers this country.
|
package/lib/recipes/venue.ts
CHANGED
|
@@ -5,11 +5,11 @@
|
|
|
5
5
|
*
|
|
6
6
|
* `venue` recipe. A real venue name written as the whole query, and the same name followed by a
|
|
7
7
|
* locality drawn from the country's admin tuples. The rows teach the model that
|
|
8
|
-
* `Manchester Art Gallery` is a venue rather than a locality and a region
|
|
9
|
-
* prior in `@mailwoman/neural`
|
|
8
|
+
* `Manchester Art Gallery` is a venue rather than a locality and a region. The venue-head
|
|
9
|
+
* prior in `@mailwoman/neural` asserts the same reading, so the model and the prior agree.
|
|
10
10
|
*
|
|
11
11
|
* Venue names come from the Overture place-names Parquet file the venue-head table is built
|
|
12
|
-
* from
|
|
12
|
+
* from. That file holds `name`, `category` and `country` for every place at confidence 0.85 or
|
|
13
13
|
* higher. The file has no locality column, so the locality row joins a name to a tuple of the
|
|
14
14
|
* same country and asserts no fact about where the venue stands, so its surface is
|
|
15
15
|
* `SurfaceOrigin.Composed`. The bare row is the publisher's own string, so its surface is
|
|
@@ -25,7 +25,8 @@
|
|
|
25
25
|
|
|
26
26
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
27
27
|
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
28
|
-
import {
|
|
28
|
+
import { foldCaseWhitespace } from "@mailwoman/normalize/fold"
|
|
29
|
+
import { isSpacelessScriptWord, venueHeadLexicon } from "@mailwoman/poi-taxonomy/venue-heads"
|
|
29
30
|
import type { PathBuilderLike } from "path-ts"
|
|
30
31
|
|
|
31
32
|
import { normalizeGauntletSurface, readGauntletInputs } from "#gauntlet-inputs"
|
|
@@ -59,7 +60,7 @@ export const DEFAULT_VENUE_NAMES: PathBuilderLike = dataRootPath(
|
|
|
59
60
|
export const DEFAULT_NAMES_PER_COUNTRY = 20_000
|
|
60
61
|
|
|
61
62
|
/**
|
|
62
|
-
* Share of locality rows that also
|
|
63
|
+
* Share of locality rows that also print the region when `--venue-region-fraction` is absent.
|
|
63
64
|
*/
|
|
64
65
|
export const DEFAULT_REGION_FRACTION = 0.5
|
|
65
66
|
|
|
@@ -105,13 +106,12 @@ export interface VenueHeadSuffixLookup {
|
|
|
105
106
|
suffix(word: string): number | null
|
|
106
107
|
}
|
|
107
108
|
|
|
108
|
-
const SPACELESS_SCRIPT_RE = /^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Thai}]+$/u
|
|
109
|
-
|
|
110
109
|
/**
|
|
111
110
|
* Characters a spaceless-script name needs before it is written as a bare query.
|
|
112
111
|
*
|
|
113
|
-
* A two-character name such as `東京` is the length of a city name
|
|
114
|
-
* the name's last character, so a name shorter
|
|
112
|
+
* A two-character name such as `東京` is the length of a city name.
|
|
113
|
+
* The head the table recognizes is the name's last character, so a name shorter
|
|
114
|
+
* than this is a head with one character of stem.
|
|
115
115
|
*/
|
|
116
116
|
export const MIN_SPACELESS_BARE_CHARACTERS = 3
|
|
117
117
|
|
|
@@ -135,7 +135,7 @@ export function isBareVenueName(name: string, heads?: VenueHeadSuffixLookup): bo
|
|
|
135
135
|
return (
|
|
136
136
|
Boolean(heads) &&
|
|
137
137
|
[...word].length >= MIN_SPACELESS_BARE_CHARACTERS &&
|
|
138
|
-
|
|
138
|
+
isSpacelessScriptWord(word) &&
|
|
139
139
|
heads!.suffix(word) !== null
|
|
140
140
|
)
|
|
141
141
|
}
|
|
@@ -144,7 +144,7 @@ export function isBareVenueName(name: string, heads?: VenueHeadSuffixLookup): bo
|
|
|
144
144
|
* The comparison key for a name against the country's localities.
|
|
145
145
|
*/
|
|
146
146
|
export function venueNameKey(name: string): string {
|
|
147
|
-
return name.normalize("NFC")
|
|
147
|
+
return foldCaseWhitespace(name.normalize("NFC"))
|
|
148
148
|
}
|
|
149
149
|
|
|
150
150
|
/**
|
package/lib/registers.ts
CHANGED
|
@@ -204,6 +204,13 @@ export const SourceRegister = {
|
|
|
204
204
|
* HM Land Registry Price Paid Data, England and Wales.
|
|
205
205
|
*/
|
|
206
206
|
LandRegistryPricePaid: "gb-hm-land-registry-ppd",
|
|
207
|
+
/**
|
|
208
|
+
* Ordnance Survey's OpenData products under the Open Government Licence v3.0.
|
|
209
|
+
*
|
|
210
|
+
* The `gb-os-open` adapter joins three of them per row.
|
|
211
|
+
* Code-Point Open, OS Open Names and OS Open Roads share one publisher and one license.
|
|
212
|
+
*/
|
|
213
|
+
OrdnanceSurveyOpenData: "gb-os-opendata",
|
|
207
214
|
/**
|
|
208
215
|
* Australia's Geocoded National Address File.
|
|
209
216
|
*/
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* The sheet splits the address into `Strada`, `Numar`, `Cod postal` and `Localitate unitate`, so the
|
|
17
17
|
* adapter splits no street line. Measured over the 2025–2026 edition's 18,022 units:
|
|
18
18
|
*
|
|
19
|
-
* - `Strada` is the street name as the school writes it,
|
|
19
|
+
* - `Strada` is the street name as the school writes it, in most rows without the street-type word:
|
|
20
20
|
* `REPUBLICII`, `PRINCIPALA`, `Piața 1848`. 198 values begin with `STR.` and 247 with `CALEA`.
|
|
21
21
|
* The value is copied unchanged, because a street type the publisher did not write is not evidence.
|
|
22
22
|
* - `Numar` holds the house number. `FN` (`fără număr`, "without number") and `-` state that the premise
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
*
|
|
37
37
|
* ## Identity and license
|
|
38
38
|
*
|
|
39
|
-
* The row id is content-addressed over the aligned components. A school and its kindergarten
|
|
39
|
+
* The row id is content-addressed over the aligned components. A school and its kindergarten can share
|
|
40
40
|
* one address under two names, so they are two rows.
|
|
41
41
|
*
|
|
42
42
|
* data.gov.ro publishes the dataset under `CC-BY-4.0`, the license its CKAN record names for the
|
package/lib/runner.ts
CHANGED
|
@@ -47,9 +47,9 @@ export interface RunnerProgress {
|
|
|
47
47
|
export const DEFAULT_DEDUP_SLOTS_LOG2 = 27
|
|
48
48
|
|
|
49
49
|
/**
|
|
50
|
-
*
|
|
50
|
+
* The number of distinct dedup keys a `Set`-backed run holds before it stops adding new ones.
|
|
51
51
|
*
|
|
52
|
-
* A V8 `Set`
|
|
52
|
+
* A V8 `Set` throws on a 16,777,216th entry, so this cap was a real bound rather than a preference.
|
|
53
53
|
* It applies only to a run that opts out of the fingerprint table
|
|
54
54
|
* through {@linkcode RunAdapterOptions.dedupMaxSize}.
|
|
55
55
|
*/
|
|
@@ -78,7 +78,7 @@ export interface RunAdapterOptions {
|
|
|
78
78
|
adapterOptions: AdapterOptions
|
|
79
79
|
|
|
80
80
|
/**
|
|
81
|
-
* Hold dedup keys in a V8 `Set` capped at this
|
|
81
|
+
* Hold dedup keys in a V8 `Set` capped at this number of keys, rather than in a fingerprint table.
|
|
82
82
|
*
|
|
83
83
|
* The cap is what the fingerprint table replaces.
|
|
84
84
|
* A run that sets this reproduces the pre-2026-09-28 behavior.
|
|
@@ -237,7 +237,7 @@ export interface HarvestSKInspireTypeOptions {
|
|
|
237
237
|
*/
|
|
238
238
|
pageSize?: number
|
|
239
239
|
/**
|
|
240
|
-
* Stop once the manifest holds this
|
|
240
|
+
* Stop once the manifest holds this number of pages, for a probe rather than a full harvest.
|
|
241
241
|
*/
|
|
242
242
|
maxPages?: number
|
|
243
243
|
signal?: AbortSignal
|
|
@@ -314,7 +314,7 @@ export interface FetchSKInspireOptions extends BaseFetchOptions {
|
|
|
314
314
|
*/
|
|
315
315
|
pageSize?: number
|
|
316
316
|
/**
|
|
317
|
-
* Stop each type once its manifest holds this
|
|
317
|
+
* Stop each type once its manifest holds this number of pages, for a probe rather than a full harvest.
|
|
318
318
|
*/
|
|
319
319
|
maxPages?: number
|
|
320
320
|
signal?: AbortSignal
|
|
@@ -339,7 +339,7 @@ export function skInspireInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBu
|
|
|
339
339
|
* for `mailwoman corpus fetch sk-inspire`.
|
|
340
340
|
*
|
|
341
341
|
* Each type is reported separately and a type that fails does not stop the rest, because the
|
|
342
|
-
* documents are read together by the adapter and a
|
|
342
|
+
* documents are read together by the adapter and a resumed run completes a partial directory.
|
|
343
343
|
* Re-runnable per type: a complete harvest whose file still hashes to its manifest's digest
|
|
344
344
|
* makes no request, and an interrupted one resumes at the page after the last that reached disk.
|
|
345
345
|
*/
|
|
@@ -54,7 +54,8 @@ export function epochMixtureAuditPath(configPath: string): PathBuilder {
|
|
|
54
54
|
*/
|
|
55
55
|
export const ExclusionReason = {
|
|
56
56
|
/**
|
|
57
|
-
*
|
|
57
|
+
* Neither the config's `source_weights` map nor its `source_reps` map lists the source,
|
|
58
|
+
* so the sampler never considers it.
|
|
58
59
|
*/
|
|
59
60
|
UnweightedSource: "unweighted-source",
|
|
60
61
|
|
|
@@ -87,7 +88,8 @@ export interface EffectiveSourceRecord {
|
|
|
87
88
|
license: string
|
|
88
89
|
|
|
89
90
|
/**
|
|
90
|
-
* The weight `source_weights` gives it,
|
|
91
|
+
* The weight `source_weights` gives it, the repetition count `source_reps` gives it,
|
|
92
|
+
* or `null` where both maps omit it.
|
|
91
93
|
*/
|
|
92
94
|
weight: number | null
|
|
93
95
|
|
|
@@ -278,7 +280,7 @@ export interface EffectiveTrainingManifest {
|
|
|
278
280
|
* Sources the epoch emitted that the corpus's frozen manifest does not name, with their emitted rows.
|
|
279
281
|
*
|
|
280
282
|
* `freezeTrainingManifest` runs inside `buildCorpus` and records the base build's sources.
|
|
281
|
-
*
|
|
283
|
+
* A corpus overlay is merged into the corpus afterwards by `overlay-manifest`,
|
|
282
284
|
* so its rows are in the corpus and outside that manifest.
|
|
283
285
|
*
|
|
284
286
|
* Measured over `v0.6.0-register-surface` on 2026-09-28: the frozen manifest names 11 sources
|
|
@@ -329,7 +331,7 @@ export interface EpochMixtureAudit {
|
|
|
329
331
|
/**
|
|
330
332
|
* Per-source detail the audit computes from the corpus files rather than from the corpus manifest.
|
|
331
333
|
*
|
|
332
|
-
* `rows` is
|
|
334
|
+
* `rows` is the number of rows of that source the corpus holds.
|
|
333
335
|
* It is the only place a source outside the frozen manifest has a corpus row count, so it is what
|
|
334
336
|
* lets a reader read a source's rows in the corpus beside the rows one epoch drew and emitted.
|
|
335
337
|
*/
|
|
@@ -348,7 +350,11 @@ export interface EffectiveConfigView {
|
|
|
348
350
|
}
|
|
349
351
|
|
|
350
352
|
/**
|
|
351
|
-
* Read `data.source_weights` and `data.augment_exclude_sources` from a training config.
|
|
353
|
+
* Read `data.source_weights`, `data.source_reps` and `data.augment_exclude_sources` from a training config.
|
|
354
|
+
*
|
|
355
|
+
* `source_weights` and `source_reps` both admit a source to the sampler, one as a share
|
|
356
|
+
* and one as a repetition count, so an entry in either block counts as the source's weight here.
|
|
357
|
+
* A source in neither block is one the sampler never considers.
|
|
352
358
|
*
|
|
353
359
|
* A line scanner instead of a YAML parse.
|
|
354
360
|
* `corpus audit` already reads these blocks this way.
|
|
@@ -368,7 +374,7 @@ export function readConfigView(text: string): EffectiveConfigView {
|
|
|
368
374
|
const sourceWeights: Record<string, number> = {}
|
|
369
375
|
const augmentExcludeSources: string[] = []
|
|
370
376
|
|
|
371
|
-
type ConfigField = "source_weights" | "augment_exclude_sources"
|
|
377
|
+
type ConfigField = "source_weights" | "source_reps" | "augment_exclude_sources"
|
|
372
378
|
|
|
373
379
|
let field: ConfigField | null = null
|
|
374
380
|
let fieldIndent = -1
|
|
@@ -385,7 +391,7 @@ export function readConfigView(text: string): EffectiveConfigView {
|
|
|
385
391
|
}
|
|
386
392
|
|
|
387
393
|
if (!field) {
|
|
388
|
-
const opener = /^([\t ]*)(source_weights|augment_exclude_sources):\s*(#.*)?$/.exec(raw)
|
|
394
|
+
const opener = /^([\t ]*)(source_weights|source_reps|augment_exclude_sources):\s*(#.*)?$/.exec(raw)
|
|
389
395
|
|
|
390
396
|
if (opener) {
|
|
391
397
|
field = opener[2] as ConfigField
|
|
@@ -399,7 +405,7 @@ export function readConfigView(text: string): EffectiveConfigView {
|
|
|
399
405
|
continue
|
|
400
406
|
}
|
|
401
407
|
|
|
402
|
-
if (field === "source_weights") {
|
|
408
|
+
if (field === "source_weights" || field === "source_reps") {
|
|
403
409
|
const entry = /^[\t ]+([\w-]+):\s*([\d.]+)/.exec(raw)
|
|
404
410
|
|
|
405
411
|
if (entry) {
|
|
@@ -15,6 +15,18 @@ import { stringifyJSON } from "@mailwoman/core/json"
|
|
|
15
15
|
|
|
16
16
|
import { LicenseReviewState, type LicenseDecision, type SourceOperation } from "#source-register/types"
|
|
17
17
|
|
|
18
|
+
/**
|
|
19
|
+
* The license label recorded for a source whose rows carry no label.
|
|
20
|
+
*
|
|
21
|
+
* A recipe can write rows with an empty `license` column.
|
|
22
|
+
* An empty string in the record would read as a source that carries no terms,
|
|
23
|
+
* and `summarizeLicense` would find no identifier in it and report no obligation.
|
|
24
|
+
*
|
|
25
|
+
* This label is unrecognized by design, so every reader reports the source's
|
|
26
|
+
* terms as unknown rather than as none.
|
|
27
|
+
*/
|
|
28
|
+
export const NO_LICENSE_RECORDED = "none-recorded-on-rows"
|
|
29
|
+
|
|
18
30
|
/**
|
|
19
31
|
* What one source contributed to a build.
|
|
20
32
|
*
|
|
@@ -30,7 +42,22 @@ export interface TrainingSourceRecord {
|
|
|
30
42
|
* A synthetic row counts under the source it was fanned from.
|
|
31
43
|
*/
|
|
32
44
|
rows: number
|
|
45
|
+
/**
|
|
46
|
+
* The license label the source's rows carry.
|
|
47
|
+
*
|
|
48
|
+
* When the rows carry more than one label, this is the label on most of them
|
|
49
|
+
* and {@linkcode licenseValues} lists every label with its rows.
|
|
50
|
+
*/
|
|
33
51
|
license: string
|
|
52
|
+
/**
|
|
53
|
+
* Every license label the source's rows carry, with the rows under each, when there is more than one.
|
|
54
|
+
*
|
|
55
|
+
* A recipe stamps a description of its own output into the column, and one recipe can stamp
|
|
56
|
+
* several, so a synthetic source reads several labels where an adapter's source reads one.
|
|
57
|
+
* The record keeps the whole set because a reader that saw only {@linkcode license}
|
|
58
|
+
* would take the majority label for the terms of every row.
|
|
59
|
+
*/
|
|
60
|
+
licenseValues?: Record<string, number>
|
|
34
61
|
/**
|
|
35
62
|
* The register's decision for that license when the build ran, or `null`
|
|
36
63
|
* when the register contains no decision for it.
|
|
@@ -97,18 +124,19 @@ export function freezeTrainingManifest(input: {
|
|
|
97
124
|
corpusVersion: string
|
|
98
125
|
builtAt: string
|
|
99
126
|
profile: string
|
|
100
|
-
rowsBySource: ReadonlyMap<string, { rows: number; license: string }>
|
|
127
|
+
rowsBySource: ReadonlyMap<string, { rows: number; license: string; licenseValues?: Record<string, number> }>
|
|
101
128
|
decisionsByLicense: ReadonlyMap<string, LicenseDecision>
|
|
102
129
|
refused: ReadonlyMap<string, readonly string[]>
|
|
103
130
|
}): TrainingManifest {
|
|
104
131
|
const sources: TrainingSourceRecord[] = [...input.rowsBySource.entries()]
|
|
105
|
-
.map(([source, { rows, license }]): TrainingSourceRecord => {
|
|
132
|
+
.map(([source, { rows, license, licenseValues }]): TrainingSourceRecord => {
|
|
106
133
|
const decision = input.decisionsByLicense.get(license)
|
|
107
134
|
|
|
108
135
|
return {
|
|
109
136
|
source,
|
|
110
137
|
rows,
|
|
111
138
|
license,
|
|
139
|
+
...(licenseValues ? { licenseValues } : {}),
|
|
112
140
|
decision: decision
|
|
113
141
|
? {
|
|
114
142
|
licenseID: decision.licenseID,
|
|
@@ -351,8 +351,8 @@ export interface ElectedLicense {
|
|
|
351
351
|
* Creative Commons permits that.
|
|
352
352
|
*
|
|
353
353
|
* Where that happens, {@linkcode ElectedLicense.spdx} records the one a build filters on
|
|
354
|
-
* and this records what the publisher
|
|
355
|
-
*
|
|
354
|
+
* and this records what the publisher said, so a later reviewer reads the conflict
|
|
355
|
+
* rather than one reading presented as settled.
|
|
356
356
|
*
|
|
357
357
|
* The engineering policy elects the strictest grant for what a released corpus may include.
|
|
358
358
|
* It does not establish which grant governs, and `electedBecause` states
|