@mailwoman/corpus 10.2.2 → 10.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -17
- package/data/PROVENANCE.md +273 -273
- package/data/address-source-register.json +517 -23
- package/data/license-decisions.json +120 -0
- package/data/licenses/PROVENANCE.md +5 -5
- package/data/licenses/cdla-permissive-2.0.md +6 -6
- package/data/licenses/geonames-publication.md +5 -5
- package/data/licenses/licence-ouverte-2.0.md +3 -3
- package/data/licenses/national-address-database.md +12 -12
- package/data/licenses/nlod-2.0.md +3 -3
- package/data/licenses/sirene-publication.md +13 -6
- package/data/licenses/whosonfirst-licenses.md +4 -4
- package/data/source-resolutions.json +11 -0
- package/lib/adapters/cordis/adapter.ts +8 -8
- package/lib/adapters/gleif/adapter.ts +7 -8
- package/lib/adapters/po-box/adapter.ts +1 -1
- package/lib/adapters/source-id.ts +1 -1
- package/lib/adapters.ts +2 -2
- package/lib/au/adapters/gnaf/adapter.ts +1 -1
- package/lib/be/adapters/brussels/adapter.ts +2 -2
- package/lib/be/adapters/vlaanderen/adapter.ts +1 -1
- package/lib/build.ts +1 -1
- package/lib/cz/adapters/cuzk/adapter.ts +1 -1
- package/lib/cz/tools/fetch/cuzk.ts +4 -4
- package/lib/de/recipes/locale.ts +1 -1
- package/lib/dk/adapters/inspire/adapter.ts +1 -1
- package/lib/ee/tools/fetch/ads.ts +2 -2
- package/lib/es/adapters/bizkaia/adapter.ts +1 -1
- package/lib/es/adapters/navarra/adapter.ts +2 -2
- package/lib/es/cadastre.ts +2 -2
- package/lib/es/tools/fetch/bizkaia.ts +7 -7
- package/lib/es/tools/fetch/catastro.ts +6 -6
- package/lib/es/tools/fetch/navarra.ts +4 -4
- package/lib/fingerprints.ts +3 -3
- package/lib/fr/adapters/finess/adapter.ts +6 -5
- package/lib/fr/adapters/finess/voie-type.ts +1 -1
- package/lib/fr/tools/fetch/annuaire-education.ts +3 -3
- package/lib/gb/constituent-country.ts +52 -0
- package/lib/inspire/address.ts +3 -3
- package/lib/international/recipes/locale.ts +41 -7
- package/lib/it/adapters/anac/adapter.ts +4 -3
- package/lib/jp/adapters/wof-admin/adapter.ts +2 -2
- package/lib/kr/adapters/juso/label-rows.ts +1 -1
- package/lib/kr/adapters/localdata/key-index.ts +2 -1
- package/lib/lu/adapters/bd-adresses/adapter.ts +1 -1
- package/lib/nl/adapters/kadaster/adapter.ts +2 -2
- package/lib/nl/tools/fetch/kadaster.ts +1 -1
- package/lib/parquet/shuffle.ts +2 -2
- package/lib/parquet/span-sql.ts +8 -6
- package/lib/parquet/streams.ts +1 -1
- package/lib/parquet/writers.ts +2 -2
- package/lib/pl/tools/fetch/emuia.ts +2 -2
- package/lib/recipes/scaffold.ts +2 -2
- package/lib/recipes/sources.ts +3 -3
- package/lib/recipes/sub/venue/render.ts +1 -1
- package/lib/recipes/sub/venue/sources.ts +4 -3
- package/lib/recipes/sub/venue.ts +4 -3
- package/lib/recipes/venue.ts +11 -11
- package/lib/ro/adapters/retea-scolara/adapter.ts +2 -2
- package/lib/runner.ts +3 -3
- package/lib/sk/tools/fetch/inspire.ts +3 -3
- package/lib/source-register/effective-manifest.ts +2 -2
- package/lib/source-register/types.ts +2 -2
- package/lib/subvenue/harvest.ts +6 -5
- package/lib/subvenue/promotions.ts +3 -3
- package/lib/subvenue/table.ts +6 -6
- package/lib/surfaces/locale.ts +1 -1
- package/lib/synthesizers/anchor-absorption.ts +1 -1
- package/lib/synthesizers/boundary-stress.ts +3 -2
- package/lib/synthesizers/no-street.ts +1 -1
- package/lib/synthesizers/po-box.ts +1 -1
- package/lib/synthesizers/tail.ts +2 -2
- package/lib/test-kit/corpus-recipe.ts +1 -1
- package/lib/tools/align-canonical.ts +3 -2
- package/lib/tools/audit.ts +1 -1
- package/lib/tools/corpus-stats.ts +1 -1
- package/lib/tools/fetch/download/network.ts +1 -1
- package/lib/tools/fetch/geonames/dump.ts +1 -1
- package/lib/tools/fetch/gleif.ts +1 -1
- package/lib/tools/fetch/header.ts +1 -1
- package/lib/tools/fetch/inspire-addresses.ts +1 -1
- package/lib/tools/fetch/vlaanderen-ad.ts +1 -1
- package/lib/tools/fetch/wfs-harvest.ts +6 -6
- package/lib/tools/fetch/zip-archive.ts +2 -2
- package/lib/tools/golden/expand.ts +1 -1
- package/lib/tools/golden/promote.ts +3 -3
- package/lib/tools/golden/relabel-street/directory.ts +2 -1
- package/lib/tools/lint/recipe-output/vocab.ts +2 -2
- package/lib/tools/overlay/manifest.ts +62 -10
- package/lib/tools/overlay/split-slice.ts +2 -10
- package/lib/tools/postcode-triples.ts +2 -2
- package/lib/tools/sub/venue/head-nouns.ts +4 -3
- package/lib/types.ts +3 -3
- package/lib/us/adapters/state/tx-notaries/adapter.ts +1 -1
- package/lib/us/adapters/usgov/irs-bmf/adapter.ts +2 -2
- package/lib/us/adapters/usgov/nppes/adapter.ts +1 -1
- package/lib/us/synthesizers/intersection.ts +4 -4
- package/lib/us/synthesizers/street.ts +3 -3
- package/lib/us/tools/fetch/nad.ts +2 -2
- package/lib/utils/split.ts +59 -6
- package/lib/utils/wof-json.ts +1 -1
- package/out/adapters/cordis/adapter.d.ts +7 -7
- package/out/adapters/cordis/adapter.js +8 -8
- package/out/adapters/cordis/adapter.js.map +1 -1
- package/out/adapters/gleif/adapter.d.ts +7 -8
- package/out/adapters/gleif/adapter.d.ts.map +1 -1
- package/out/adapters/gleif/adapter.js +7 -8
- package/out/adapters/gleif/adapter.js.map +1 -1
- package/out/adapters/po-box/adapter.d.ts +1 -1
- package/out/adapters/source-id.d.ts +1 -1
- package/out/adapters/source-id.js +1 -1
- package/out/adapters.d.ts +2 -2
- package/out/adapters.js +2 -2
- package/out/au/adapters/gnaf/adapter.js +1 -1
- package/out/be/adapters/brussels/adapter.d.ts +1 -1
- package/out/be/adapters/brussels/adapter.js +2 -2
- package/out/be/adapters/vlaanderen/adapter.d.ts +1 -1
- package/out/build.js +1 -1
- package/out/build.js.map +1 -1
- package/out/cz/adapters/cuzk/adapter.d.ts +1 -1
- package/out/cz/adapters/cuzk/adapter.js +1 -1
- package/out/cz/tools/fetch/cuzk.d.ts +2 -2
- package/out/cz/tools/fetch/cuzk.js +2 -2
- package/out/de/recipes/locale.d.ts +1 -1
- package/out/de/recipes/locale.js +1 -1
- package/out/dk/adapters/inspire/adapter.js +1 -1
- package/out/ee/tools/fetch/ads.d.ts +2 -2
- package/out/ee/tools/fetch/ads.js +1 -1
- package/out/es/adapters/bizkaia/adapter.js +1 -1
- package/out/es/adapters/navarra/adapter.d.ts +2 -2
- package/out/es/adapters/navarra/adapter.js +2 -2
- package/out/es/cadastre.d.ts +2 -2
- package/out/es/cadastre.js +2 -2
- package/out/es/tools/fetch/bizkaia.d.ts +5 -5
- package/out/es/tools/fetch/bizkaia.js +5 -5
- package/out/es/tools/fetch/catastro.d.ts +4 -4
- package/out/es/tools/fetch/catastro.js +2 -2
- package/out/es/tools/fetch/navarra.d.ts +2 -2
- package/out/es/tools/fetch/navarra.js +2 -2
- package/out/fingerprints.d.ts +3 -3
- package/out/fingerprints.js +3 -3
- package/out/fr/adapters/finess/adapter.d.ts +6 -5
- package/out/fr/adapters/finess/adapter.d.ts.map +1 -1
- package/out/fr/adapters/finess/adapter.js +6 -5
- package/out/fr/adapters/finess/adapter.js.map +1 -1
- package/out/fr/adapters/finess/voie-type.d.ts +1 -1
- package/out/fr/adapters/finess/voie-type.js +1 -1
- package/out/fr/tools/fetch/annuaire-education.d.ts +3 -3
- package/out/fr/tools/fetch/annuaire-education.js +3 -3
- package/out/gb/constituent-country.d.ts +36 -0
- package/out/gb/constituent-country.d.ts.map +1 -0
- package/out/gb/constituent-country.js +30 -0
- package/out/gb/constituent-country.js.map +1 -0
- package/out/inspire/address.d.ts +3 -3
- package/out/inspire/address.js +3 -3
- package/out/international/recipes/locale.d.ts +6 -0
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +29 -7
- package/out/international/recipes/locale.js.map +1 -1
- package/out/it/adapters/anac/adapter.d.ts +1 -1
- package/out/it/adapters/anac/adapter.d.ts.map +1 -1
- package/out/it/adapters/anac/adapter.js +4 -3
- package/out/it/adapters/anac/adapter.js.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.d.ts +2 -2
- package/out/jp/adapters/wof-admin/adapter.js +2 -2
- package/out/kr/adapters/juso/label-rows.d.ts +1 -1
- package/out/kr/adapters/localdata/key-index.d.ts +2 -1
- package/out/kr/adapters/localdata/key-index.d.ts.map +1 -1
- package/out/kr/adapters/localdata/key-index.js +2 -1
- package/out/kr/adapters/localdata/key-index.js.map +1 -1
- package/out/lu/adapters/bd-adresses/adapter.d.ts +1 -1
- package/out/lu/adapters/bd-adresses/adapter.js +1 -1
- package/out/nl/adapters/kadaster/adapter.js +2 -2
- package/out/nl/tools/fetch/kadaster.d.ts +1 -1
- package/out/nl/tools/fetch/kadaster.js +1 -1
- package/out/parquet/shuffle.d.ts +2 -2
- package/out/parquet/shuffle.js +2 -2
- package/out/parquet/span-sql.d.ts +8 -6
- package/out/parquet/span-sql.d.ts.map +1 -1
- package/out/parquet/span-sql.js +8 -6
- package/out/parquet/span-sql.js.map +1 -1
- package/out/parquet/streams.d.ts +1 -1
- package/out/parquet/writers.d.ts +2 -2
- package/out/parquet/writers.js +1 -1
- package/out/pl/tools/fetch/emuia.d.ts +2 -2
- package/out/recipes/scaffold.d.ts +2 -2
- package/out/recipes/sources.d.ts +2 -2
- package/out/recipes/sources.js +3 -3
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/sub/venue/render.d.ts +1 -1
- package/out/recipes/sub/venue/render.js +1 -1
- package/out/recipes/sub/venue/sources.d.ts +4 -3
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +4 -3
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts +3 -2
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +2 -2
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/venue.d.ts +7 -6
- package/out/recipes/venue.d.ts.map +1 -1
- package/out/recipes/venue.js +11 -10
- package/out/recipes/venue.js.map +1 -1
- package/out/ro/adapters/retea-scolara/adapter.d.ts +2 -2
- package/out/ro/adapters/retea-scolara/adapter.js +2 -2
- package/out/runner.d.ts +3 -3
- package/out/runner.js +2 -2
- package/out/sk/tools/fetch/inspire.d.ts +3 -3
- package/out/sk/tools/fetch/inspire.js +1 -1
- package/out/source-register/effective-manifest.d.ts +2 -2
- package/out/source-register/types.d.ts +2 -2
- package/out/subvenue/harvest.d.ts +4 -3
- package/out/subvenue/harvest.d.ts.map +1 -1
- package/out/subvenue/harvest.js +6 -5
- package/out/subvenue/harvest.js.map +1 -1
- package/out/subvenue/promotions.d.ts +2 -2
- package/out/subvenue/promotions.js +3 -3
- package/out/subvenue/promotions.js.map +1 -1
- package/out/subvenue/table.d.ts +6 -6
- package/out/subvenue/table.js +1 -1
- package/out/surfaces/locale.d.ts +1 -1
- package/out/surfaces/locale.js +1 -1
- package/out/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/synthesizers/boundary-stress.js +2 -1
- package/out/synthesizers/boundary-stress.js.map +1 -1
- package/out/synthesizers/tail.d.ts +2 -2
- package/out/synthesizers/tail.js +2 -2
- package/out/test-kit/corpus-recipe.d.ts +1 -1
- package/out/test-kit/corpus-recipe.js +1 -1
- package/out/tools/align-canonical.d.ts.map +1 -1
- package/out/tools/align-canonical.js +3 -2
- package/out/tools/align-canonical.js.map +1 -1
- package/out/tools/audit.js +1 -1
- package/out/tools/corpus-stats.d.ts +1 -1
- package/out/tools/fetch/download/network.d.ts +1 -1
- package/out/tools/fetch/download/network.js +1 -1
- package/out/tools/fetch/geonames/dump.d.ts +1 -1
- package/out/tools/fetch/geonames/dump.js +1 -1
- package/out/tools/fetch/gleif.d.ts +1 -1
- package/out/tools/fetch/gleif.js +1 -1
- package/out/tools/fetch/header.d.ts +1 -1
- package/out/tools/fetch/header.js +1 -1
- package/out/tools/fetch/inspire-addresses.d.ts +1 -1
- package/out/tools/fetch/vlaanderen-ad.d.ts +1 -1
- package/out/tools/fetch/wfs-harvest.d.ts +3 -3
- package/out/tools/fetch/wfs-harvest.js +4 -4
- package/out/tools/fetch/wfs-harvest.js.map +1 -1
- package/out/tools/fetch/zip-archive.d.ts +2 -2
- package/out/tools/fetch/zip-archive.js +2 -2
- package/out/tools/golden/expand.js +1 -1
- package/out/tools/golden/expand.js.map +1 -1
- package/out/tools/golden/promote.d.ts +1 -1
- package/out/tools/golden/promote.js +3 -3
- package/out/tools/golden/promote.js.map +1 -1
- package/out/tools/golden/relabel-street/directory.d.ts +2 -1
- package/out/tools/golden/relabel-street/directory.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/directory.js.map +1 -1
- package/out/tools/lint/recipe-output/vocab.js +2 -2
- package/out/tools/overlay/manifest.d.ts +15 -2
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +41 -10
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/overlay/split-slice.d.ts.map +1 -1
- package/out/tools/overlay/split-slice.js +2 -9
- package/out/tools/overlay/split-slice.js.map +1 -1
- package/out/tools/postcode-triples.d.ts +1 -1
- package/out/tools/postcode-triples.js +2 -2
- package/out/tools/postcode-triples.js.map +1 -1
- package/out/tools/sub/venue/head-nouns.d.ts +1 -1
- package/out/tools/sub/venue/head-nouns.d.ts.map +1 -1
- package/out/tools/sub/venue/head-nouns.js +4 -3
- package/out/tools/sub/venue/head-nouns.js.map +1 -1
- package/out/types.d.ts +3 -3
- package/out/types.js +1 -1
- package/out/us/adapters/state/tx-notaries/adapter.d.ts +1 -1
- package/out/us/adapters/state/tx-notaries/adapter.js +1 -1
- package/out/us/adapters/usgov/irs-bmf/adapter.d.ts +1 -1
- package/out/us/adapters/usgov/irs-bmf/adapter.js +2 -2
- package/out/us/synthesizers/intersection.js +4 -4
- package/out/us/synthesizers/intersection.js.map +1 -1
- package/out/us/synthesizers/street.js +3 -3
- package/out/us/synthesizers/street.js.map +1 -1
- package/out/us/tools/fetch/nad.d.ts +2 -2
- package/out/us/tools/fetch/nad.js +2 -2
- package/out/utils/split.d.ts +21 -0
- package/out/utils/split.d.ts.map +1 -1
- package/out/utils/split.js +49 -5
- package/out/utils/split.js.map +1 -1
- package/out/utils/wof-json.d.ts +1 -1
- package/out/utils/wof-json.js +1 -1
- package/package.json +14 -14
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* to read. The `ad:component` hrefs look external, because ČÚZK writes an absolute WFS stored-query
|
|
16
16
|
* URL on `services.cuzk.cz`. The `Id=` parameter of each one matches a `gml:id` in the file at hand.
|
|
17
17
|
* Measured over `584282.xml`: 5,460 component references across 1,365 addresses, 0 of them reaching
|
|
18
|
-
* outside the archive. Only `ad:parcel` and `ad:building` point at
|
|
18
|
+
* outside the archive. Only `ad:parcel` and `ad:building` point at separate themes, and
|
|
19
19
|
* the adapter reads neither.
|
|
20
20
|
*
|
|
21
21
|
* ČÚZK states one condition of use and states it uniformly. Every one of the 6,259 `rights` elements
|
|
@@ -123,11 +123,11 @@ export const CZ_CUZK_REQUEST_INTERVAL_MS = 250
|
|
|
123
123
|
const ZIP_SIGNATURE = "PK"
|
|
124
124
|
|
|
125
125
|
/**
|
|
126
|
-
*
|
|
126
|
+
* The number of archives written between two manifest writes.
|
|
127
127
|
*
|
|
128
128
|
* The manifest is what makes an interrupted harvest resumable, and writing it
|
|
129
129
|
* after every archive would rewrite a 6,258-entry document 6,258 times.
|
|
130
|
-
* At this interval an interruption re-fetches at most this
|
|
130
|
+
* At this interval an interruption re-fetches at most this number of archives that are already on disk.
|
|
131
131
|
*/
|
|
132
132
|
const MANIFEST_INTERVAL = 25
|
|
133
133
|
|
|
@@ -352,7 +352,7 @@ export interface CzCuzkHarvestManifest extends SourceCollectionManifest {
|
|
|
352
352
|
*/
|
|
353
353
|
projection: CzCuzkProjection
|
|
354
354
|
/**
|
|
355
|
-
*
|
|
355
|
+
* The number of municipalities the service document listed when this manifest was written.
|
|
356
356
|
*
|
|
357
357
|
* The denominator for `files`, read from the feed on each run rather than fixed here.
|
|
358
358
|
*/
|
|
@@ -379,7 +379,7 @@ export interface HarvestCzCuzkOptions {
|
|
|
379
379
|
*/
|
|
380
380
|
municipalities?: readonly string[]
|
|
381
381
|
/**
|
|
382
|
-
* Stop after this
|
|
382
|
+
* Stop after this number of municipalities, counting the ones already on disk.
|
|
383
383
|
*
|
|
384
384
|
* A bounded run rather than all 6,258.
|
|
385
385
|
*/
|
package/lib/de/recipes/locale.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* Two registers are absent from the OA tuples. A comma-free single line (`Neusser Str. 12 Nippes 50733
|
|
16
16
|
* Köln`) segments as one unit at stage 2. With one segment, the placetype-pair prior never fires.
|
|
17
17
|
* The model then reads `Nippes` as a second street. The one-value-per-tag projection deletes it.
|
|
18
|
-
* `--comma-free-fraction` renders that
|
|
18
|
+
* `--comma-free-fraction` renders that share of native-order rows with `" "` as the line separator. OA
|
|
19
19
|
* rows also lack a district, so `--ortsteil-fraction` rows borrow a WOF Ortsteil of the tuple's
|
|
20
20
|
* own locality as `dependent_locality`. The German spelling is recovered from the `names` table
|
|
21
21
|
* ({@link ortsteilSurface}) because WOF's `spr.name` for DE neighborhoods is the ascii-folded label.
|
|
@@ -93,7 +93,7 @@ export const DK_INSPIRE_COUNTRIES: readonly string[] = ["DK"]
|
|
|
93
93
|
export const DK_INSPIRE_REQUIRED_TABLES: readonly string[] = ["address", "postaldescriptor", "thoroughfarename"]
|
|
94
94
|
|
|
95
95
|
/**
|
|
96
|
-
*
|
|
96
|
+
* The number of rows Kysely buffers per chunk while streaming the join.
|
|
97
97
|
*/
|
|
98
98
|
const STREAM_CHUNK_ROWS = 2000
|
|
99
99
|
|
|
@@ -76,7 +76,7 @@ export const EE_ADS_SORT_BY = "gml_id"
|
|
|
76
76
|
* Features per page.
|
|
77
77
|
*
|
|
78
78
|
* The service's `CountDefault` is 1,000,000 and it honors `count=10000`, which is where the page
|
|
79
|
-
* size is set: a 10,000-feature page is 31,198,909 bytes, and a larger page
|
|
79
|
+
* size is set: a 10,000-feature page is 31,198,909 bytes, and a larger page needs fewer requests
|
|
80
80
|
* at the cost of holding a larger body in memory and losing more work when one request fails.
|
|
81
81
|
*/
|
|
82
82
|
export const EE_ADS_PAGE_SIZE = 10_000
|
|
@@ -135,7 +135,7 @@ export interface HarvestADSOptions {
|
|
|
135
135
|
*/
|
|
136
136
|
pageSize?: number
|
|
137
137
|
/**
|
|
138
|
-
* Stop once the manifest holds this
|
|
138
|
+
* Stop once the manifest holds this number of pages, for a probe rather than a full harvest.
|
|
139
139
|
*/
|
|
140
140
|
maxPages?: number
|
|
141
141
|
signal?: AbortSignal
|
|
@@ -164,7 +164,7 @@ const MUNICIPALITY_LEVEL = "3rdOrder"
|
|
|
164
164
|
const HOUSE_NUMBER_TYPE = "buildingIdentifier"
|
|
165
165
|
|
|
166
166
|
/**
|
|
167
|
-
*
|
|
167
|
+
* The number of dot-separated fields a designator has.
|
|
168
168
|
*
|
|
169
169
|
* Municipality, street, number: `48.001.00026.003` splits the municipality code across two.
|
|
170
170
|
*/
|
|
@@ -14,8 +14,8 @@
|
|
|
14
14
|
* no such mapping and this adapter claims none. It reads one archive or a directory of them. A
|
|
15
15
|
* caller harvests all 272 by pointing it at the directory.
|
|
16
16
|
*
|
|
17
|
-
* This publisher is read entirely from its references' titles
|
|
18
|
-
*
|
|
17
|
+
* This publisher is read entirely from its references' titles. No other INSPIRE reader
|
|
18
|
+
* here reads its addresses that way. Its own file publishes no `AD:ThoroughfareName`,
|
|
19
19
|
* `AD:PostalDescriptor` or `AD:AdminUnitName` feature at all: partition 1 holds 29 distinct element
|
|
20
20
|
* names across 115 addresses and none of them is a component feature. Each address instead holds
|
|
21
21
|
* six `AD:component` references whose `xlink:title` states the value:
|
package/lib/es/cadastre.ts
CHANGED
|
@@ -72,8 +72,8 @@ export function spanishHouseNumber(
|
|
|
72
72
|
* `ES.SDGC.PD.55.101.` and `ES.GFA.PD.059.`, against no feature of that id.
|
|
73
73
|
*
|
|
74
74
|
* Those truncated references are the only unresolvable ones measured in either file.
|
|
75
|
-
* A reader taking every unresolved reference as an absence would also swallow a
|
|
76
|
-
*
|
|
75
|
+
* A reader taking every unresolved reference as an absence would also swallow a broken join,
|
|
76
|
+
* so the two are separated here and every other unresolved reference raises.
|
|
77
77
|
*/
|
|
78
78
|
export function referenceStatesNoValue(key: string): boolean {
|
|
79
79
|
return key.trim().endsWith(".")
|
|
@@ -16,10 +16,10 @@
|
|
|
16
16
|
* archives therefore reads every address in the province without its street, its postcode or
|
|
17
17
|
* its municipality. This harvest acquires both services into one directory for that reason.
|
|
18
18
|
*
|
|
19
|
-
* Each component type is read in one request
|
|
19
|
+
* Each component type is read in one request. This service serves a type no other way. Its
|
|
20
20
|
* capabilities document states `ImplementsResultPaging` as `FALSE` and `CountDefault` as `1000`, so
|
|
21
21
|
* a reader that paged a type would receive the first page again. The harvest therefore asks
|
|
22
|
-
* `RESULTTYPE=hits` for the type's count and then asks for that
|
|
22
|
+
* `RESULTTYPE=hits` for the type's count and then asks for that number of features in one `GetFeature`,
|
|
23
23
|
* and {@linkcode fetchBizkaiaComponentDocument} raises where the document returns fewer than the
|
|
24
24
|
* count. A truncated component document is the failure that would otherwise reach the adapter as
|
|
25
25
|
* an unresolved reference per address.
|
|
@@ -121,11 +121,11 @@ export const ES_BIZKAIA_REQUEST_INTERVAL_MS = 250
|
|
|
121
121
|
const BODY_TIMEOUT_MS = 600_000
|
|
122
122
|
|
|
123
123
|
/**
|
|
124
|
-
*
|
|
124
|
+
* The number of archives written between two manifest writes.
|
|
125
125
|
*
|
|
126
126
|
* The manifest is what makes an interrupted harvest resumable, and writing it
|
|
127
127
|
* after every archive would rewrite the whole document once per municipality.
|
|
128
|
-
* At this interval an interruption re-fetches at most this
|
|
128
|
+
* At this interval an interruption re-fetches at most this number of archives already on disk.
|
|
129
129
|
*/
|
|
130
130
|
const MANIFEST_INTERVAL = 25
|
|
131
131
|
|
|
@@ -306,7 +306,7 @@ export interface ESBizkaiaHarvestManifest extends SourceCollectionManifest {
|
|
|
306
306
|
*/
|
|
307
307
|
feed_updated: string | null
|
|
308
308
|
/**
|
|
309
|
-
*
|
|
309
|
+
* The number of municipalities the service document listed when this manifest was written.
|
|
310
310
|
*
|
|
311
311
|
* The denominator for `files`, read from the feed on each run rather than fixed here.
|
|
312
312
|
*/
|
|
@@ -332,7 +332,7 @@ export interface HarvestESBizkaiaOptions {
|
|
|
332
332
|
*/
|
|
333
333
|
municipalities?: readonly string[]
|
|
334
334
|
/**
|
|
335
|
-
* Stop after this
|
|
335
|
+
* Stop after this number of municipalities, counting the ones already on disk.
|
|
336
336
|
*/
|
|
337
337
|
limit?: number
|
|
338
338
|
/**
|
|
@@ -390,7 +390,7 @@ async function archiveIsCurrent(
|
|
|
390
390
|
/**
|
|
391
391
|
* Reads one component feature type whole and writes it beside the archives.
|
|
392
392
|
*
|
|
393
|
-
* Two requests per type: `RESULTTYPE=hits` for the count, then one `GetFeature` for that
|
|
393
|
+
* Two requests per type: `RESULTTYPE=hits` for the count, then one `GetFeature` for that number of features.
|
|
394
394
|
* `COUNT` is asked one above the stated count, so a service whose hits response
|
|
395
395
|
* undercounts its own matches answers with more features than it claimed
|
|
396
396
|
* rather than with exactly the number the harvest asked for.
|
|
@@ -105,11 +105,11 @@ export const ES_CATASTRO_REQUEST_INTERVAL_MS = 250
|
|
|
105
105
|
const SLUG = ES_CATASTRO_ADAPTER_ID
|
|
106
106
|
|
|
107
107
|
/**
|
|
108
|
-
*
|
|
108
|
+
* The number of archives written between two manifest writes.
|
|
109
109
|
*
|
|
110
110
|
* The manifest is what makes an interrupted harvest resumable, and writing it after every
|
|
111
111
|
* archive would rewrite a document with one entry per municipality once per municipality.
|
|
112
|
-
* At this interval an interruption re-fetches at most this
|
|
112
|
+
* At this interval an interruption re-fetches at most this number of archives that are already on disk.
|
|
113
113
|
*/
|
|
114
114
|
const MANIFEST_INTERVAL = 25
|
|
115
115
|
|
|
@@ -416,18 +416,18 @@ export interface ESCatastroArchiveEntry extends SourceManifest {
|
|
|
416
416
|
*/
|
|
417
417
|
export interface ESCatastroHarvestManifest extends SourceCollectionManifest {
|
|
418
418
|
/**
|
|
419
|
-
*
|
|
419
|
+
* The number of territorial offices the service document listed, read from the feed on each run.
|
|
420
420
|
*/
|
|
421
421
|
provinces_listed: number
|
|
422
422
|
/**
|
|
423
|
-
*
|
|
423
|
+
* The number of those offices this run read a province feed for.
|
|
424
424
|
*
|
|
425
425
|
* A bounded run reads fewer, so this is the denominator for `municipalities_listed`
|
|
426
426
|
* rather than a count of the country.
|
|
427
427
|
*/
|
|
428
428
|
provinces_harvested: number
|
|
429
429
|
/**
|
|
430
|
-
*
|
|
430
|
+
* The number of municipalities the province feeds this run read listed between them.
|
|
431
431
|
*/
|
|
432
432
|
municipalities_listed: number
|
|
433
433
|
files: ESCatastroArchiveEntry[]
|
|
@@ -453,7 +453,7 @@ export interface HarvestESCatastroOptions {
|
|
|
453
453
|
*/
|
|
454
454
|
municipalities?: readonly string[]
|
|
455
455
|
/**
|
|
456
|
-
* Stop after this
|
|
456
|
+
* Stop after this number of municipalities, counting the ones already on disk.
|
|
457
457
|
*/
|
|
458
458
|
limit?: number
|
|
459
459
|
/**
|
|
@@ -91,10 +91,10 @@ export const ES_NAVARRA_REQUEST_INTERVAL_MS = 250
|
|
|
91
91
|
const SLUG = ES_NAVARRA_ADAPTER_ID
|
|
92
92
|
|
|
93
93
|
/**
|
|
94
|
-
*
|
|
94
|
+
* The number of archives written between two manifest writes.
|
|
95
95
|
*
|
|
96
96
|
* The manifest is what makes an interrupted harvest resumable.
|
|
97
|
-
* At this interval an interruption re-fetches at most this
|
|
97
|
+
* At this interval an interruption re-fetches at most this number of archives that are already on disk.
|
|
98
98
|
*/
|
|
99
99
|
const MANIFEST_INTERVAL = 25
|
|
100
100
|
|
|
@@ -217,7 +217,7 @@ export interface ESNavarraArchiveEntry extends SourceManifest {
|
|
|
217
217
|
*/
|
|
218
218
|
export interface ESNavarraHarvestManifest extends SourceCollectionManifest {
|
|
219
219
|
/**
|
|
220
|
-
*
|
|
220
|
+
* The number of partitions the service document listed when this manifest was written.
|
|
221
221
|
*
|
|
222
222
|
* The denominator for `files`, read from the feed on each run rather than fixed here.
|
|
223
223
|
*/
|
|
@@ -238,7 +238,7 @@ export interface HarvestESNavarraOptions {
|
|
|
238
238
|
*/
|
|
239
239
|
partitions?: readonly string[]
|
|
240
240
|
/**
|
|
241
|
-
* Stop after this
|
|
241
|
+
* Stop after this number of partitions, counting the ones already on disk.
|
|
242
242
|
*/
|
|
243
243
|
limit?: number
|
|
244
244
|
/**
|
package/lib/fingerprints.ts
CHANGED
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
* @file A fixed-capacity set of 128-bit fingerprints in one flat `Uint32Array`.
|
|
6
6
|
*
|
|
7
|
-
* The adapter runner deduplicates by holding every key it has seen. A V8 `Set`
|
|
8
|
-
* entry
|
|
7
|
+
* The adapter runner deduplicates by holding every key it has seen. A V8 `Set` throws on its 16,777,217th
|
|
8
|
+
* entry, so the runner capped itself at 10,000,000 keys. Three adapters passed that cap in
|
|
9
9
|
* `v0.7.0-de-holdout`. The duplicate census over that build measured what the cap costs: of the
|
|
10
10
|
* 2,491,399 duplicate rows the three adapters wrote, 1,490,992 duplicate a key first seen after the cap.
|
|
11
11
|
* A membership test on a capped set cannot refuse those rows.
|
|
@@ -77,7 +77,7 @@ export class FingerprintSet {
|
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
/**
|
|
80
|
-
*
|
|
80
|
+
* The number of keys this table accepts before {@linkcode add} throws.
|
|
81
81
|
*/
|
|
82
82
|
get limit(): number {
|
|
83
83
|
return this.#limit
|
|
@@ -32,8 +32,8 @@
|
|
|
32
32
|
*
|
|
33
33
|
* Every FINESS establishment is an organization's site: a hospital, a pharmacy, a laboratory, a
|
|
34
34
|
* service office. Three categories house the people they serve in a private dwelling. In `462`
|
|
35
|
-
* Lieux de vie et d'accueil the staff live with the people they receive. A `238`
|
|
36
|
-
*
|
|
35
|
+
* Lieux de vie et d'accueil the staff live with the people they receive. A FINESS category `238`
|
|
36
|
+
* establishment, specialized family reception, places patients with host families, as a `236` Service de placement familial
|
|
37
37
|
* does. An address in those three can be a family's home, so they are refused under
|
|
38
38
|
* {@linkcode FinessRefusal.FamilyPlacement}, as is an establishment whose name states family
|
|
39
39
|
* placement or a lieu de vie under another category. A row whose street type is TRE-R35's `CHEZ`,
|
|
@@ -56,7 +56,7 @@
|
|
|
56
56
|
*
|
|
57
57
|
* A digit in `voie` is refused under {@linkcode FinessRefusal.StreetDigitUnplaced} unless the digits
|
|
58
58
|
* name the street, as a numbered route (`NATIONALE 2`) or a date (`DU 22 MAI`) does. The number has
|
|
59
|
-
* its own field, so a digit left in the name is a box, a building or a door: `VALLÉE 3 BP 208`,
|
|
59
|
+
* its own field, so a digit left in the street name is a box, a building or a door: `VALLÉE 3 BP 208`,
|
|
60
60
|
* `DIAKA BAT A4 PORTE 2`.
|
|
61
61
|
*
|
|
62
62
|
* ## The lieu-dit and box line, and the routing line
|
|
@@ -166,8 +166,9 @@ export const FR_FINESS_COUNTRIES: readonly string[] = [
|
|
|
166
166
|
]
|
|
167
167
|
|
|
168
168
|
/**
|
|
169
|
-
* The establishment categories whose address can be a private dwelling:
|
|
170
|
-
*
|
|
169
|
+
* The establishment categories whose address can be a private dwelling: FINESS
|
|
170
|
+
* categories `462` (lieux de vie et d'accueil), `238` (specialized family reception)
|
|
171
|
+
* and `236` (services de placement familial).
|
|
171
172
|
*/
|
|
172
173
|
export const FINESS_FAMILY_PLACEMENT_CATEGORIES: ReadonlySet<string> = new Set(["236", "238", "462"])
|
|
173
174
|
|
|
@@ -41,7 +41,7 @@ import { decomposeFrStreet } from "#fr/street-decompose"
|
|
|
41
41
|
*
|
|
42
42
|
* `VAL` keeps the code as its word.
|
|
43
43
|
* TRE-R35 displays it as `Vallée, vallon`, two words the code does not choose between,
|
|
44
|
-
* and `Val` is itself the written word
|
|
44
|
+
* and `Val` is itself the written word in French place names such as `Val-d'Isère`.
|
|
45
45
|
*/
|
|
46
46
|
export const TRE_R35_VOIE_TYPES: Readonly<Record<string, string | null>> = {
|
|
47
47
|
ANSE: "anse",
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
*
|
|
16
16
|
* Two checks keep a short transfer from reading as a small directory. The records endpoint is asked
|
|
17
17
|
* for the `total_count` of the same filter before the export, and the written file must hold exactly
|
|
18
|
-
* that
|
|
18
|
+
* that number of lines. A mismatch removes the file and fails the fetch. A re-run compares the catalog
|
|
19
19
|
* record's `modified` against the manifest and makes no transfer when they agree and the file is on
|
|
20
20
|
* disk at the recorded length.
|
|
21
21
|
*/
|
|
@@ -75,7 +75,7 @@ export function annuaireExportURL(): string {
|
|
|
75
75
|
}
|
|
76
76
|
|
|
77
77
|
/**
|
|
78
|
-
* The records URL that answers
|
|
78
|
+
* The records URL that answers with the count of records the overseas filter selects.
|
|
79
79
|
*/
|
|
80
80
|
export function annuaireCountURL(): string {
|
|
81
81
|
return `${ANNUAIRE_EDUCATION_API_URL}/records?where=${encodeURIComponent(annuaireOverseasFilter())}&limit=0`
|
|
@@ -135,7 +135,7 @@ export async function readAnnuaireCatalog(
|
|
|
135
135
|
}
|
|
136
136
|
|
|
137
137
|
/**
|
|
138
|
-
* Read
|
|
138
|
+
* Read the count of records the overseas filter selects.
|
|
139
139
|
*
|
|
140
140
|
* @throws When the endpoint answers no `total_count`.
|
|
141
141
|
*/
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* The constituent country of a GB postcode, read from the ONS National Statistics UPRN Lookup.
|
|
7
|
+
*
|
|
8
|
+
* A GB row labels its constituent country `region`, because WOF records England, Scotland, Wales and
|
|
9
|
+
* Northern Ireland as `macroregion` places under the United Kingdom.
|
|
10
|
+
* The postcode-area table in the codex assigns a whole area to one country. That assignment is wrong on the border:
|
|
11
|
+
* every `SY` postcode maps to Wales, and `SY1` to `SY5` lie in Shrewsbury, England.
|
|
12
|
+
* NSUL records the country of each UPRN, so the postcode's country is a count of its UPRNs.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { GB_COUNTRIES } from "@mailwoman/codex/gb/country"
|
|
16
|
+
import { NSULLookup } from "@mailwoman/resolver-wof-sqlite/nsul/lookup"
|
|
17
|
+
import { nsulDatabasePath } from "@mailwoman/resolver-wof-sqlite/paths"
|
|
18
|
+
import type { PathBuilderLike } from "path-ts"
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* What {@link openConstituentCountryReader} answered for one postcode.
|
|
22
|
+
*/
|
|
23
|
+
export interface ConstituentCountryAnswer {
|
|
24
|
+
/**
|
|
25
|
+
* The English name of the country holding most of the postcode's UPRNs, such as `Wales`.
|
|
26
|
+
*
|
|
27
|
+
* Null when the register assigns no UPRN to the postcode.
|
|
28
|
+
*/
|
|
29
|
+
name: string | null
|
|
30
|
+
/**
|
|
31
|
+
* Whether the postcode's UPRNs lie in more than one constituent country.
|
|
32
|
+
*/
|
|
33
|
+
straddles: boolean
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Opens `nsul.db` and returns a reader from a GB postcode to its constituent country.
|
|
38
|
+
*
|
|
39
|
+
* The reader holds the database open for the life of the process, as a corpus build reads millions of rows.
|
|
40
|
+
*/
|
|
41
|
+
export function openConstituentCountryReader(
|
|
42
|
+
databasePath: PathBuilderLike = nsulDatabasePath("nsul.db")
|
|
43
|
+
): (postcode: string) => ConstituentCountryAnswer {
|
|
44
|
+
const lookup = new NSULLookup({ databasePath })
|
|
45
|
+
|
|
46
|
+
return (postcode) => {
|
|
47
|
+
const countries = lookup.countriesForPostcode(postcode)
|
|
48
|
+
const top = countries[0]
|
|
49
|
+
|
|
50
|
+
return { name: top ? GB_COUNTRIES[top.country].name : null, straddles: countries.length > 1 }
|
|
51
|
+
}
|
|
52
|
+
}
|
package/lib/inspire/address.ts
CHANGED
|
@@ -123,7 +123,7 @@ export function codelistValue(uri: string | null | undefined): string | null {
|
|
|
123
123
|
/**
|
|
124
124
|
* Whether a publisher marked this element void.
|
|
125
125
|
*
|
|
126
|
-
* INSPIRE writes `xsi:nil="true"
|
|
126
|
+
* INSPIRE writes `xsi:nil="true"`, in most cases with a `nilReason`.
|
|
127
127
|
* Brussels writes the nil marker and no reason on all 232,003 of its void `ad:validFrom`
|
|
128
128
|
* elements, and Czechia writes four `gn:pronunciation` elements nil with no reason.
|
|
129
129
|
*
|
|
@@ -355,7 +355,7 @@ const REFERENCE_ID_PARAMETERS = new Set(["id", "featureid", "resourceid"])
|
|
|
355
355
|
* `http://vocab.belgif.be/auth/refnis1995/1000#id` addresses the same term as that URI without it.
|
|
356
356
|
*
|
|
357
357
|
* Whether a key joins is not a property of its spelling, so this reads a key
|
|
358
|
-
* and the caller decides joinability against the features the publisher
|
|
358
|
+
* and the caller decides joinability against the features the publisher served.
|
|
359
359
|
* A check based on the URL's shape once reported 15 of Flanders' 20 references as belonging
|
|
360
360
|
* to another register, when every one of them addressed a feature of the same service.
|
|
361
361
|
*
|
|
@@ -392,7 +392,7 @@ export function componentJoinKey(href: string): string | null {
|
|
|
392
392
|
/**
|
|
393
393
|
* The spelling a `gn:GeographicalName` holds, reached from a feature that has one.
|
|
394
394
|
*
|
|
395
|
-
* Every
|
|
395
|
+
* Every INSPIRE feature that has a name nests the text at
|
|
396
396
|
* `…/gn:GeographicalName/gn:spelling/gn:SpellingOfName/gn:text`, under a per-feature
|
|
397
397
|
* prefix: an `ad:ThoroughfareName` adds `ad:name/ad:ThoroughfareNameValue/ad:name`,
|
|
398
398
|
* while an `ad:AdminUnitName` and an `ad:AddressAreaName` add only `ad:name`.
|
|
@@ -19,6 +19,7 @@ import type { PathBuilderLike } from "path-ts"
|
|
|
19
19
|
import { CSVSpliterator } from "spliterator"
|
|
20
20
|
|
|
21
21
|
import { stableSourceID } from "#adapters/source-id"
|
|
22
|
+
import { openConstituentCountryReader } from "#gb/constituent-country"
|
|
22
23
|
import type { CorpusRecipe } from "#recipes/scaffold"
|
|
23
24
|
import { defaultRecipeSource } from "#recipes/sources"
|
|
24
25
|
import { SourceRegister } from "#registers"
|
|
@@ -47,6 +48,12 @@ export interface LocalePart {
|
|
|
47
48
|
* The street combines `tipo_vial` with `nombre_via`.
|
|
48
49
|
*/
|
|
49
50
|
cnigRaw?: boolean
|
|
51
|
+
/**
|
|
52
|
+
* Opens a reader that answers the row's region from its postcode, replacing the region column.
|
|
53
|
+
*
|
|
54
|
+
* A GB part answers the constituent country this way, and its county column is not read.
|
|
55
|
+
*/
|
|
56
|
+
regionOfPostcode?: () => (postcode: string) => string | null
|
|
50
57
|
}
|
|
51
58
|
|
|
52
59
|
/**
|
|
@@ -158,7 +165,18 @@ const COUNTRY_SOURCES: Record<string, LocaleCountrySource> = {
|
|
|
158
165
|
source: defaultRecipeSource("synth-gb"),
|
|
159
166
|
register: SourceRegister.LandRegistryPricePaid,
|
|
160
167
|
corpusVersion: "0.9.9",
|
|
161
|
-
|
|
168
|
+
// The `REGION` column holds a county, so the constituent country is read from NSUL by postcode instead.
|
|
169
|
+
parts: [
|
|
170
|
+
{
|
|
171
|
+
path: dataRootPath("ppd", "2026-07-22", "gb-tuples.csv"),
|
|
172
|
+
districtAsLocality: true,
|
|
173
|
+
regionOfPostcode: () => {
|
|
174
|
+
const countryOf = openConstituentCountryReader()
|
|
175
|
+
|
|
176
|
+
return (postcode) => countryOf(postcode).name
|
|
177
|
+
},
|
|
178
|
+
},
|
|
179
|
+
],
|
|
162
180
|
},
|
|
163
181
|
}
|
|
164
182
|
|
|
@@ -224,6 +242,8 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
|
|
|
224
242
|
let header: string[] | null = null
|
|
225
243
|
let seen = 0
|
|
226
244
|
let dropped = 0
|
|
245
|
+
const regionOf = part.regionOfPostcode?.()
|
|
246
|
+
let regionUnknown = 0
|
|
227
247
|
|
|
228
248
|
try {
|
|
229
249
|
// The first row holds the column names.
|
|
@@ -304,12 +324,24 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
|
|
|
304
324
|
continue
|
|
305
325
|
}
|
|
306
326
|
|
|
327
|
+
const postcode = get(cells, cols.post)
|
|
328
|
+
|
|
329
|
+
let region = get(cells, cols.region) || part.region || ""
|
|
330
|
+
|
|
331
|
+
if (regionOf) {
|
|
332
|
+
region = (postcode && regionOf(postcode)) || ""
|
|
333
|
+
|
|
334
|
+
if (!region) {
|
|
335
|
+
regionUnknown++
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
|
|
307
339
|
const tuple: LocaleBaseTuple = {
|
|
308
340
|
house_number: get(cells, cols.num),
|
|
309
341
|
street,
|
|
310
342
|
locality,
|
|
311
|
-
region
|
|
312
|
-
postcode
|
|
343
|
+
region,
|
|
344
|
+
postcode,
|
|
313
345
|
...(dependent_locality ? { dependent_locality } : {}),
|
|
314
346
|
}
|
|
315
347
|
|
|
@@ -333,6 +365,10 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
|
|
|
333
365
|
|
|
334
366
|
console.error(` ${part.path ?? part.csv}: ${reservoir.length} sampled of ${seen} rows (${dropped} city-noise drops)`)
|
|
335
367
|
|
|
368
|
+
if (regionOf) {
|
|
369
|
+
console.error(` ${part.path ?? part.csv}: ${regionUnknown} rows have no region for their postcode`)
|
|
370
|
+
}
|
|
371
|
+
|
|
336
372
|
return reservoir
|
|
337
373
|
}
|
|
338
374
|
|
|
@@ -425,10 +461,8 @@ export const localeRecipe: CorpusRecipe = {
|
|
|
425
461
|
throw new Error(`--country-fraction must be in [0, 1], got ${countryFraction}`)
|
|
426
462
|
}
|
|
427
463
|
|
|
428
|
-
// `COUNTRY_SOURCES`
|
|
429
|
-
|
|
430
|
-
// rather than in twelve table entries.
|
|
431
|
-
const source = opts.sourceName ?? defaultRecipeSource(countrySource.source)
|
|
464
|
+
// Each `COUNTRY_SOURCES` entry already holds the current spelling from `defaultRecipeSource`.
|
|
465
|
+
const source = opts.sourceName ?? countrySource.source
|
|
432
466
|
const count = opts.count ?? 4000
|
|
433
467
|
// When unset, each part keeps its own setting.
|
|
434
468
|
// True also selects the ES pedanía parts.
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
* public bodies, and the one role whose parties can be sole traders publishes no address.
|
|
30
30
|
* {@linkcode readANACParty} still applies a name check and refuses a party whose name takes the
|
|
31
31
|
* `Surname, Firstname` form, because a later edition could add one. On the 2025 edition the check
|
|
32
|
-
* refuses no party: 0 of the 18,740
|
|
32
|
+
* refuses no party: 0 of the 18,740 buyer and payer parties that have an address take that form, while 56
|
|
33
33
|
* hold a comma inside an organization name — `PARCO NAZIONALE DEL CILENTO, VALLO DI DIANO E
|
|
34
34
|
* ALBURNI` and `SETTORE PATRIMONIO, AMBIENTE E LAVORI PUBBLICI` among them. The check therefore
|
|
35
35
|
* requires one word on each side of the comma rather than any comma.
|
|
@@ -304,8 +304,9 @@ const KILOMETRE_POINT = /[\s,]+km\.?[\s,]*\d+(?:[.,]\d+)?$/iu
|
|
|
304
304
|
/**
|
|
305
305
|
* A name of the form `Surname, Firstname`: one comma, one word on each side, letters only.
|
|
306
306
|
*
|
|
307
|
-
* An organization name in this release
|
|
308
|
-
* `PARCO NAZIONALE DEL CILENTO, VALLO DI DIANO E ALBURNI`
|
|
307
|
+
* An organization name in this release can hold a comma, but a name with a single word on
|
|
308
|
+
* each side of the comma is rare: `PARCO NAZIONALE DEL CILENTO, VALLO DI DIANO E ALBURNI`
|
|
309
|
+
* has eight words after the comma.
|
|
309
310
|
* The single word on each side is what the inverted personal form has
|
|
310
311
|
* and what a department or authority name does not.
|
|
311
312
|
*/
|
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
* 1. **Reversed ordering** — region → county → locality → block → house_number "東京都世田谷区南町1-2-3" not
|
|
14
14
|
* "1-2-3 Minamicho, Setagaya, Tokyo"
|
|
15
15
|
* 2. **No street names** — most JP addresses use a block/sub-block grid system. The "neighborhood"
|
|
16
|
-
* placetype (丁目) is the closest analog to a street but is
|
|
17
|
-
* 3. **Postcode-first convention** — addresses
|
|
16
|
+
* placetype (丁目) is the closest analog to a street but is a grid cell.
|
|
17
|
+
* 3. **Postcode-first convention** — addresses can be prefixed with `〒NNN-nnnn`.
|
|
18
18
|
*
|
|
19
19
|
* Schema mapping to ComponentTags (subset of Stage 3 + JP-specific Phase 6 tags):
|
|
20
20
|
*
|
|
@@ -152,7 +152,7 @@ function joinNumber(main: string, sub: string): string {
|
|
|
152
152
|
|
|
153
153
|
export interface ReadJusoOptions {
|
|
154
154
|
/**
|
|
155
|
-
* Stop after this
|
|
155
|
+
* Stop after this number of rows per 시도, for a fixture or a smoke run.
|
|
156
156
|
* Omit for the whole register.
|
|
157
157
|
*/
|
|
158
158
|
maxRowsPerRegion?: number
|
|
@@ -142,7 +142,8 @@ export function aliasKeyIndex(index: KeyIndex): void {
|
|
|
142
142
|
}
|
|
143
143
|
|
|
144
144
|
/**
|
|
145
|
-
*
|
|
145
|
+
* The number of tokens from `start` that form one of the region's 시군구 —
|
|
146
|
+
* 2 for `수원시 장안구`, 1 for `종로구`, 0 for none.
|
|
146
147
|
*
|
|
147
148
|
* Two-token first, because `수원시` by itself is also listed and taking it would
|
|
148
149
|
* leave `장안구` to be read as a road.
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
* Measured over the 179,622 rows of the 2026-09-28 edition with a generic-word probe rather than a
|
|
34
34
|
* language classifier: 133,449 values open with a French street generic such as `Rue`, `Avenue` or
|
|
35
35
|
* `Montée`, 18,629 close with a Luxembourgish one such as `-strooss`, `-wee` or `-gaass`, and the
|
|
36
|
-
* remaining 27,544 match neither probe and include
|
|
36
|
+
* remaining 27,544 match neither probe and include Luxembourgish forms such as
|
|
37
37
|
* `Am Stengber`, `An Hiesel` and `Um Reebou`.
|
|
38
38
|
* No column marks a row's language, so `locale` reads `und-LU`.
|
|
39
39
|
* `und` is BCP-47's undetermined language subtag, and the repository already stores it in
|
|
@@ -360,8 +360,8 @@ function documentChunks(documentPath: PathBuilderLike): AsyncIterable<Uint8Array
|
|
|
360
360
|
/**
|
|
361
361
|
* Every document under `inputPath`, where `inputPath` is one GML file or a directory holding them.
|
|
362
362
|
*
|
|
363
|
-
* A directory is read because the download is one national file
|
|
364
|
-
* service document could list more than one entry
|
|
363
|
+
* A directory is read because the download is one national file now and a later
|
|
364
|
+
* service document could list more than one entry.
|
|
365
365
|
* A directory holding no document raises rather than yielding zero rows, so a wrong
|
|
366
366
|
* path reports itself instead of reading as an empty publisher.
|
|
367
367
|
*/
|
|
@@ -54,7 +54,7 @@ export const NL_KADASTER_FEED_URL = "https://service.pdok.nl/kadaster/ad/atom/ad
|
|
|
54
54
|
* The adapter's `inputPath` states this name.
|
|
55
55
|
*
|
|
56
56
|
* The publisher's own name for the file.
|
|
57
|
-
* The adapter reads a `.gml.gz` by inflating it, so the suffix is
|
|
57
|
+
* The adapter reads a `.gml.gz` by inflating it, so the suffix is required.
|
|
58
58
|
*/
|
|
59
59
|
export const NL_KADASTER_DOWNLOAD_FILENAME = "addresses.gml.gz"
|
|
60
60
|
|
package/lib/parquet/shuffle.ts
CHANGED
|
@@ -10,8 +10,8 @@ import { shuffleWith } from "@mailwoman/core/random"
|
|
|
10
10
|
/**
|
|
11
11
|
* Rows held in memory while shuffling, when the caller names no window.
|
|
12
12
|
*
|
|
13
|
-
* Five 50,000-row groups span more than one output row-group and reach `wof-admin`'s
|
|
14
|
-
*
|
|
13
|
+
* Five 50,000-row groups span more than one output row-group and reach `wof-admin`'s whole
|
|
14
|
+
* 13-country set, while a window below `ROW_GROUP_SIZE` draws from no more than one row-group.
|
|
15
15
|
*/
|
|
16
16
|
export const DEFAULT_SHUFFLE_WINDOW = 250_000
|
|
17
17
|
|