@mailwoman/corpus 10.3.0 → 10.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -17
- package/data/PROVENANCE.md +287 -273
- package/data/address-source-register.json +545 -23
- package/data/license-decisions.json +120 -0
- package/data/licenses/PROVENANCE.md +5 -5
- package/data/licenses/cdla-permissive-2.0.md +6 -6
- package/data/licenses/geonames-publication.md +5 -5
- package/data/licenses/licence-ouverte-2.0.md +3 -3
- package/data/licenses/national-address-database.md +12 -12
- package/data/licenses/nlod-2.0.md +3 -3
- package/data/licenses/sirene-publication.md +13 -6
- package/data/licenses/whosonfirst-licenses.md +4 -4
- package/data/source-resolutions.json +46 -0
- package/data/training-manifests/v0.7.2-address-systems.effective.json +1169 -0
- package/data/training-manifests/v0.7.2-address-systems.json +550 -0
- package/lib/adapters/cordis/adapter.ts +9 -9
- package/lib/adapters/gleif/adapter.ts +7 -8
- package/lib/adapters/po-box/adapter.ts +1 -1
- package/lib/adapters/source-id.ts +1 -1
- package/lib/adapters.ts +4 -2
- package/lib/au/adapters/gnaf/adapter.ts +1 -1
- package/lib/be/adapters/brussels/adapter.ts +3 -3
- package/lib/be/adapters/vlaanderen/adapter.ts +1 -1
- package/lib/be/adapters/wallonie/adapter.ts +1 -1
- package/lib/build/align.ts +1 -1
- package/lib/build.ts +1 -1
- package/lib/cz/adapters/cuzk/adapter.ts +2 -2
- package/lib/cz/tools/fetch/cuzk.ts +6 -6
- package/lib/de/recipes/locale.ts +1 -1
- package/lib/dk/adapters/inspire/adapter.ts +1 -1
- package/lib/ee/tools/fetch/ads.ts +2 -2
- package/lib/es/adapters/bizkaia/adapter.ts +1 -1
- package/lib/es/adapters/navarra/adapter.ts +2 -2
- package/lib/es/cadastre.ts +2 -2
- package/lib/es/tools/fetch/bizkaia.ts +8 -8
- package/lib/es/tools/fetch/catastro.ts +6 -6
- package/lib/es/tools/fetch/navarra.ts +4 -4
- package/lib/fingerprints.ts +3 -3
- package/lib/fr/adapters/finess/adapter.ts +6 -5
- package/lib/fr/adapters/finess/voie-type.ts +1 -1
- package/lib/fr/tools/fetch/annuaire-education.ts +3 -3
- package/lib/gb/adapters/os-open/adapter.ts +453 -0
- package/lib/gb/constituent-country.ts +52 -0
- package/lib/gb/os-open/nearest-link.ts +169 -0
- package/lib/gb/os-open/open-names.ts +180 -0
- package/lib/gb/os-open/open-roads.ts +145 -0
- package/lib/gb/tools/fetch/os-open.ts +251 -0
- package/lib/inspire/address.ts +3 -3
- package/lib/international/recipes/locale.ts +41 -7
- package/lib/it/adapters/anac/adapter.ts +4 -3
- package/lib/jp/adapters/wof-admin/adapter.ts +2 -2
- package/lib/kr/adapters/juso/label-rows.ts +1 -1
- package/lib/kr/adapters/localdata/key-index.ts +2 -1
- package/lib/lu/adapters/bd-adresses/adapter.ts +1 -1
- package/lib/nl/adapters/kadaster/adapter.ts +2 -2
- package/lib/nl/tools/fetch/kadaster.ts +2 -2
- package/lib/parquet/shuffle.ts +2 -2
- package/lib/parquet/span-sql.ts +8 -6
- package/lib/parquet/streams.ts +1 -1
- package/lib/parquet/writers.ts +2 -2
- package/lib/pl/tools/fetch/emuia.ts +2 -2
- package/lib/recipes/scaffold.ts +2 -2
- package/lib/recipes/sources.ts +3 -3
- package/lib/recipes/sub/venue/render.ts +1 -1
- package/lib/recipes/sub/venue/sources.ts +4 -3
- package/lib/recipes/sub/venue.ts +4 -3
- package/lib/recipes/venue.ts +11 -11
- package/lib/registers.ts +7 -0
- package/lib/ro/adapters/retea-scolara/adapter.ts +2 -2
- package/lib/runner.ts +3 -3
- package/lib/sk/tools/fetch/inspire.ts +3 -3
- package/lib/source-register/effective-manifest.ts +14 -8
- package/lib/source-register/training-manifest.ts +30 -2
- package/lib/source-register/types.ts +2 -2
- package/lib/source-register.ts +1 -0
- package/lib/subvenue/harvest.ts +6 -5
- package/lib/subvenue/promotions.ts +3 -3
- package/lib/subvenue/surfaces.ts +1 -1
- package/lib/subvenue/table.ts +6 -6
- package/lib/surfaces/locale.ts +1 -1
- package/lib/synthesizers/anchor-absorption.ts +1 -1
- package/lib/synthesizers/boundary-stress.ts +3 -2
- package/lib/synthesizers/no-street.ts +1 -1
- package/lib/synthesizers/po-box.ts +1 -1
- package/lib/synthesizers/tail.ts +2 -2
- package/lib/test-kit/corpus-recipe.ts +1 -1
- package/lib/tools/align-canonical.ts +3 -2
- package/lib/tools/audit.ts +1 -1
- package/lib/tools/corpus-stats.ts +1 -1
- package/lib/tools/fetch/download/network.ts +1 -1
- package/lib/tools/fetch/geonames/dump.ts +1 -1
- package/lib/tools/fetch/gleif.ts +2 -2
- package/lib/tools/fetch/header.ts +1 -1
- package/lib/tools/fetch/inspire-addresses.ts +1 -1
- package/lib/tools/fetch/vlaanderen-ad.ts +1 -1
- package/lib/tools/fetch/wfs-harvest.ts +6 -6
- package/lib/tools/fetch/zip-archive.ts +2 -2
- package/lib/tools/fetch.ts +2 -0
- package/lib/tools/golden/expand.ts +1 -1
- package/lib/tools/golden/promote.ts +3 -3
- package/lib/tools/golden/relabel-street/directory.ts +2 -1
- package/lib/tools/lint/recipe-output/vocab.ts +2 -2
- package/lib/tools/lint/recipe-output.ts +1 -1
- package/lib/tools/migrate/source-names.ts +1 -1
- package/lib/tools/overlay/manifest.ts +62 -10
- package/lib/tools/overlay/split-slice.ts +2 -10
- package/lib/tools/postcode-triples.ts +2 -2
- package/lib/tools/sub/venue/head-nouns.ts +4 -3
- package/lib/types.ts +3 -3
- package/lib/us/adapters/state/tx-notaries/adapter.ts +1 -1
- package/lib/us/adapters/usgov/irs-bmf/adapter.ts +2 -2
- package/lib/us/adapters/usgov/nad/adapter.ts +1 -1
- package/lib/us/adapters/usgov/nppes/adapter.ts +2 -2
- package/lib/us/synthesizers/intersection.ts +4 -4
- package/lib/us/synthesizers/street.ts +3 -3
- package/lib/us/tools/fetch/nad.ts +2 -2
- package/lib/utils/split.ts +59 -6
- package/lib/utils/wof-json.ts +1 -1
- package/out/adapters/cordis/adapter.d.ts +7 -7
- package/out/adapters/cordis/adapter.js +9 -9
- package/out/adapters/cordis/adapter.js.map +1 -1
- package/out/adapters/gleif/adapter.d.ts +7 -8
- package/out/adapters/gleif/adapter.d.ts.map +1 -1
- package/out/adapters/gleif/adapter.js +7 -8
- package/out/adapters/gleif/adapter.js.map +1 -1
- package/out/adapters/po-box/adapter.d.ts +1 -1
- package/out/adapters/source-id.d.ts +1 -1
- package/out/adapters/source-id.js +1 -1
- package/out/adapters.d.ts +2 -2
- package/out/adapters.d.ts.map +1 -1
- package/out/adapters.js +4 -2
- package/out/adapters.js.map +1 -1
- package/out/au/adapters/gnaf/adapter.js +1 -1
- package/out/be/adapters/brussels/adapter.d.ts +2 -2
- package/out/be/adapters/brussels/adapter.js +3 -3
- package/out/be/adapters/vlaanderen/adapter.d.ts +1 -1
- package/out/be/adapters/wallonie/adapter.d.ts +1 -1
- package/out/be/adapters/wallonie/adapter.js +1 -1
- package/out/build/align.d.ts +1 -1
- package/out/build.js +1 -1
- package/out/build.js.map +1 -1
- package/out/cz/adapters/cuzk/adapter.d.ts +2 -2
- package/out/cz/adapters/cuzk/adapter.js +2 -2
- package/out/cz/tools/fetch/cuzk.d.ts +2 -2
- package/out/cz/tools/fetch/cuzk.js +4 -4
- package/out/cz/tools/fetch/cuzk.js.map +1 -1
- package/out/de/recipes/locale.d.ts +1 -1
- package/out/de/recipes/locale.js +1 -1
- package/out/dk/adapters/inspire/adapter.js +1 -1
- package/out/ee/tools/fetch/ads.d.ts +2 -2
- package/out/ee/tools/fetch/ads.js +1 -1
- package/out/es/adapters/bizkaia/adapter.js +1 -1
- package/out/es/adapters/navarra/adapter.d.ts +2 -2
- package/out/es/adapters/navarra/adapter.js +2 -2
- package/out/es/cadastre.d.ts +2 -2
- package/out/es/cadastre.js +2 -2
- package/out/es/tools/fetch/bizkaia.d.ts +5 -5
- package/out/es/tools/fetch/bizkaia.js +6 -6
- package/out/es/tools/fetch/bizkaia.js.map +1 -1
- package/out/es/tools/fetch/catastro.d.ts +4 -4
- package/out/es/tools/fetch/catastro.js +2 -2
- package/out/es/tools/fetch/navarra.d.ts +2 -2
- package/out/es/tools/fetch/navarra.js +2 -2
- package/out/fingerprints.d.ts +3 -3
- package/out/fingerprints.js +3 -3
- package/out/fr/adapters/finess/adapter.d.ts +6 -5
- package/out/fr/adapters/finess/adapter.d.ts.map +1 -1
- package/out/fr/adapters/finess/adapter.js +6 -5
- package/out/fr/adapters/finess/adapter.js.map +1 -1
- package/out/fr/adapters/finess/voie-type.d.ts +1 -1
- package/out/fr/adapters/finess/voie-type.js +1 -1
- package/out/fr/tools/fetch/annuaire-education.d.ts +3 -3
- package/out/fr/tools/fetch/annuaire-education.js +3 -3
- package/out/gb/adapters/os-open/adapter.d.ts +165 -0
- package/out/gb/adapters/os-open/adapter.d.ts.map +1 -0
- package/out/gb/adapters/os-open/adapter.js +298 -0
- package/out/gb/adapters/os-open/adapter.js.map +1 -0
- package/out/gb/constituent-country.d.ts +36 -0
- package/out/gb/constituent-country.d.ts.map +1 -0
- package/out/gb/constituent-country.js +30 -0
- package/out/gb/constituent-country.js.map +1 -0
- package/out/gb/os-open/nearest-link.d.ts +73 -0
- package/out/gb/os-open/nearest-link.d.ts.map +1 -0
- package/out/gb/os-open/nearest-link.js +133 -0
- package/out/gb/os-open/nearest-link.js.map +1 -0
- package/out/gb/os-open/open-names.d.ts +92 -0
- package/out/gb/os-open/open-names.d.ts.map +1 -0
- package/out/gb/os-open/open-names.js +107 -0
- package/out/gb/os-open/open-names.js.map +1 -0
- package/out/gb/os-open/open-roads.d.ts +80 -0
- package/out/gb/os-open/open-roads.d.ts.map +1 -0
- package/out/gb/os-open/open-roads.js +79 -0
- package/out/gb/os-open/open-roads.js.map +1 -0
- package/out/gb/tools/fetch/os-open.d.ts +91 -0
- package/out/gb/tools/fetch/os-open.d.ts.map +1 -0
- package/out/gb/tools/fetch/os-open.js +166 -0
- package/out/gb/tools/fetch/os-open.js.map +1 -0
- package/out/inspire/address.d.ts +3 -3
- package/out/inspire/address.js +3 -3
- package/out/international/recipes/locale.d.ts +6 -0
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +29 -7
- package/out/international/recipes/locale.js.map +1 -1
- package/out/it/adapters/anac/adapter.d.ts +1 -1
- package/out/it/adapters/anac/adapter.d.ts.map +1 -1
- package/out/it/adapters/anac/adapter.js +4 -3
- package/out/it/adapters/anac/adapter.js.map +1 -1
- package/out/jp/adapters/wof-admin/adapter.d.ts +2 -2
- package/out/jp/adapters/wof-admin/adapter.js +2 -2
- package/out/kr/adapters/juso/label-rows.d.ts +1 -1
- package/out/kr/adapters/localdata/key-index.d.ts +2 -1
- package/out/kr/adapters/localdata/key-index.d.ts.map +1 -1
- package/out/kr/adapters/localdata/key-index.js +2 -1
- package/out/kr/adapters/localdata/key-index.js.map +1 -1
- package/out/lu/adapters/bd-adresses/adapter.d.ts +1 -1
- package/out/lu/adapters/bd-adresses/adapter.js +1 -1
- package/out/nl/adapters/kadaster/adapter.js +2 -2
- package/out/nl/tools/fetch/kadaster.d.ts +2 -2
- package/out/nl/tools/fetch/kadaster.js +2 -2
- package/out/parquet/shuffle.d.ts +2 -2
- package/out/parquet/shuffle.js +2 -2
- package/out/parquet/span-sql.d.ts +8 -6
- package/out/parquet/span-sql.d.ts.map +1 -1
- package/out/parquet/span-sql.js +8 -6
- package/out/parquet/span-sql.js.map +1 -1
- package/out/parquet/streams.d.ts +1 -1
- package/out/parquet/writers.d.ts +2 -2
- package/out/parquet/writers.js +1 -1
- package/out/pl/tools/fetch/emuia.d.ts +2 -2
- package/out/recipes/scaffold.d.ts +2 -2
- package/out/recipes/sources.d.ts +2 -2
- package/out/recipes/sources.js +3 -3
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/sub/venue/render.d.ts +1 -1
- package/out/recipes/sub/venue/render.js +1 -1
- package/out/recipes/sub/venue/sources.d.ts +4 -3
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +4 -3
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts +3 -2
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +2 -2
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/venue.d.ts +7 -6
- package/out/recipes/venue.d.ts.map +1 -1
- package/out/recipes/venue.js +11 -10
- package/out/recipes/venue.js.map +1 -1
- package/out/registers.d.ts +7 -0
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +7 -0
- package/out/registers.js.map +1 -1
- package/out/ro/adapters/retea-scolara/adapter.d.ts +2 -2
- package/out/ro/adapters/retea-scolara/adapter.js +2 -2
- package/out/runner.d.ts +3 -3
- package/out/runner.js +2 -2
- package/out/sk/tools/fetch/inspire.d.ts +3 -3
- package/out/sk/tools/fetch/inspire.js +1 -1
- package/out/source-register/effective-manifest.d.ts +11 -5
- package/out/source-register/effective-manifest.d.ts.map +1 -1
- package/out/source-register/effective-manifest.js +9 -4
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/training-manifest.d.ts +27 -0
- package/out/source-register/training-manifest.d.ts.map +1 -1
- package/out/source-register/training-manifest.js +13 -1
- package/out/source-register/training-manifest.js.map +1 -1
- package/out/source-register/types.d.ts +2 -2
- package/out/source-register.d.ts +1 -1
- package/out/source-register.d.ts.map +1 -1
- package/out/source-register.js +1 -1
- package/out/source-register.js.map +1 -1
- package/out/subvenue/harvest.d.ts +4 -3
- package/out/subvenue/harvest.d.ts.map +1 -1
- package/out/subvenue/harvest.js +6 -5
- package/out/subvenue/harvest.js.map +1 -1
- package/out/subvenue/promotions.d.ts +2 -2
- package/out/subvenue/promotions.js +3 -3
- package/out/subvenue/promotions.js.map +1 -1
- package/out/subvenue/surfaces.d.ts +1 -1
- package/out/subvenue/table.d.ts +6 -6
- package/out/subvenue/table.js +1 -1
- package/out/surfaces/locale.d.ts +1 -1
- package/out/surfaces/locale.js +1 -1
- package/out/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/synthesizers/boundary-stress.js +2 -1
- package/out/synthesizers/boundary-stress.js.map +1 -1
- package/out/synthesizers/tail.d.ts +2 -2
- package/out/synthesizers/tail.js +2 -2
- package/out/test-kit/corpus-recipe.d.ts +1 -1
- package/out/test-kit/corpus-recipe.js +1 -1
- package/out/tools/align-canonical.d.ts.map +1 -1
- package/out/tools/align-canonical.js +3 -2
- package/out/tools/align-canonical.js.map +1 -1
- package/out/tools/audit.js +1 -1
- package/out/tools/corpus-stats.d.ts +1 -1
- package/out/tools/fetch/download/network.d.ts +1 -1
- package/out/tools/fetch/download/network.js +1 -1
- package/out/tools/fetch/geonames/dump.d.ts +1 -1
- package/out/tools/fetch/geonames/dump.js +1 -1
- package/out/tools/fetch/gleif.d.ts +2 -2
- package/out/tools/fetch/gleif.js +1 -1
- package/out/tools/fetch/header.d.ts +1 -1
- package/out/tools/fetch/header.js +1 -1
- package/out/tools/fetch/inspire-addresses.d.ts +1 -1
- package/out/tools/fetch/vlaanderen-ad.d.ts +1 -1
- package/out/tools/fetch/wfs-harvest.d.ts +3 -3
- package/out/tools/fetch/wfs-harvest.js +4 -4
- package/out/tools/fetch/wfs-harvest.js.map +1 -1
- package/out/tools/fetch/zip-archive.d.ts +2 -2
- package/out/tools/fetch/zip-archive.js +2 -2
- package/out/tools/fetch.d.ts +2 -0
- package/out/tools/fetch.d.ts.map +1 -1
- package/out/tools/fetch.js +2 -0
- package/out/tools/fetch.js.map +1 -1
- package/out/tools/golden/expand.js +1 -1
- package/out/tools/golden/expand.js.map +1 -1
- package/out/tools/golden/promote.d.ts +1 -1
- package/out/tools/golden/promote.js +3 -3
- package/out/tools/golden/promote.js.map +1 -1
- package/out/tools/golden/relabel-street/directory.d.ts +2 -1
- package/out/tools/golden/relabel-street/directory.d.ts.map +1 -1
- package/out/tools/golden/relabel-street/directory.js.map +1 -1
- package/out/tools/lint/recipe-output/vocab.js +2 -2
- package/out/tools/lint/recipe-output.d.ts +1 -1
- package/out/tools/migrate/source-names.d.ts +1 -1
- package/out/tools/overlay/manifest.d.ts +15 -2
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +41 -10
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/overlay/split-slice.d.ts.map +1 -1
- package/out/tools/overlay/split-slice.js +2 -9
- package/out/tools/overlay/split-slice.js.map +1 -1
- package/out/tools/postcode-triples.d.ts +1 -1
- package/out/tools/postcode-triples.js +2 -2
- package/out/tools/postcode-triples.js.map +1 -1
- package/out/tools/sub/venue/head-nouns.d.ts +1 -1
- package/out/tools/sub/venue/head-nouns.d.ts.map +1 -1
- package/out/tools/sub/venue/head-nouns.js +4 -3
- package/out/tools/sub/venue/head-nouns.js.map +1 -1
- package/out/types.d.ts +3 -3
- package/out/types.js +1 -1
- package/out/us/adapters/state/tx-notaries/adapter.d.ts +1 -1
- package/out/us/adapters/state/tx-notaries/adapter.js +1 -1
- package/out/us/adapters/usgov/irs-bmf/adapter.d.ts +1 -1
- package/out/us/adapters/usgov/irs-bmf/adapter.js +2 -2
- package/out/us/adapters/usgov/nad/adapter.d.ts +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +1 -1
- package/out/us/adapters/usgov/nppes/adapter.d.ts +1 -1
- package/out/us/adapters/usgov/nppes/adapter.js +1 -1
- package/out/us/synthesizers/intersection.js +4 -4
- package/out/us/synthesizers/intersection.js.map +1 -1
- package/out/us/synthesizers/street.js +3 -3
- package/out/us/synthesizers/street.js.map +1 -1
- package/out/us/tools/fetch/nad.d.ts +2 -2
- package/out/us/tools/fetch/nad.js +2 -2
- package/out/utils/split.d.ts +21 -0
- package/out/utils/split.d.ts.map +1 -1
- package/out/utils/split.js +49 -5
- package/out/utils/split.js.map +1 -1
- package/out/utils/wof-json.d.ts +1 -1
- package/out/utils/wof-json.js +1 -1
- package/package.json +15 -14
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Reads the road rows of OS Open Names.
|
|
7
|
+
*
|
|
8
|
+
* The CSV product is 819 files, one per 10 km National Grid square, with no header row. OS ships
|
|
9
|
+
* the 34 column names in `Doc/OS_Open_Names_Header.csv`, and the first data row of each file opens
|
|
10
|
+
* with a byte-order mark, which this reader strips from the `ID` value.
|
|
11
|
+
*
|
|
12
|
+
* A road row's `ID` is the roadNameTOID that OS Open Roads links reference, such as
|
|
13
|
+
* `osgb4000000010927776`. Two `LOCAL_TYPE` values are roads: `Named Road`, and
|
|
14
|
+
* `Section Of Named Road` for a road that OS splits where it crosses a postcode district or a
|
|
15
|
+
* settlement.
|
|
16
|
+
*/
|
|
17
|
+
/**
|
|
18
|
+
* The 34 columns of an OS Open Names CSV, in file order, as `Doc/OS_Open_Names_Header.csv` names them.
|
|
19
|
+
*/
|
|
20
|
+
export declare const OPEN_NAMES_COLUMNS: readonly ["ID", "NAMES_URI", "NAME1", "NAME1_LANG", "NAME2", "NAME2_LANG", "TYPE", "LOCAL_TYPE", "GEOMETRY_X", "GEOMETRY_Y", "MOST_DETAIL_VIEW_RES", "LEAST_DETAIL_VIEW_RES", "MBR_XMIN", "MBR_YMIN", "MBR_XMAX", "MBR_YMAX", "POSTCODE_DISTRICT", "POSTCODE_DISTRICT_URI", "POPULATED_PLACE", "POPULATED_PLACE_URI", "POPULATED_PLACE_TYPE", "DISTRICT_BOROUGH", "DISTRICT_BOROUGH_URI", "DISTRICT_BOROUGH_TYPE", "COUNTY_UNITARY", "COUNTY_UNITARY_URI", "COUNTY_UNITARY_TYPE", "REGION", "REGION_URI", "COUNTRY", "COUNTRY_URI", "RELATED_SPATIAL_OBJECT", "SAME_AS_DBPEDIA", "SAME_AS_GEONAMES"];
|
|
21
|
+
/**
|
|
22
|
+
* The `LOCAL_TYPE` values that mark a road row.
|
|
23
|
+
*/
|
|
24
|
+
export declare const OPEN_NAMES_ROAD_TYPES: ReadonlySet<string>;
|
|
25
|
+
/**
|
|
26
|
+
* One road row of OS Open Names, with the columns the corpus adapter reads.
|
|
27
|
+
*/
|
|
28
|
+
export interface OpenNamesRoad {
|
|
29
|
+
/**
|
|
30
|
+
* The roadNameTOID, such as `osgb4000000010927776`.
|
|
31
|
+
*/
|
|
32
|
+
id: string;
|
|
33
|
+
/**
|
|
34
|
+
* The road's name, in the language OS lists first.
|
|
35
|
+
*/
|
|
36
|
+
name1: string;
|
|
37
|
+
/**
|
|
38
|
+
* The road's other-language name, Welsh or Gaelic, or an empty string.
|
|
39
|
+
*/
|
|
40
|
+
name2: string;
|
|
41
|
+
/**
|
|
42
|
+
* The outward code of the postcode district holding the road, such as `LS6`.
|
|
43
|
+
*/
|
|
44
|
+
postcodeDistrict: string;
|
|
45
|
+
/**
|
|
46
|
+
* The settlement holding the road, or an empty string.
|
|
47
|
+
*
|
|
48
|
+
* A bilingual settlement is written `<NAME1> / <NAME2>` of that settlement,
|
|
49
|
+
* such as `Stornoway / Steòrnabhagh`.
|
|
50
|
+
*/
|
|
51
|
+
populatedPlace: string;
|
|
52
|
+
districtBorough: string;
|
|
53
|
+
countyUnitary: string;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Counts from a read, with a reason for each row that did not yield.
|
|
57
|
+
*/
|
|
58
|
+
export interface OpenNamesReadStats {
|
|
59
|
+
/**
|
|
60
|
+
* Rows read across every file.
|
|
61
|
+
*/
|
|
62
|
+
read: number;
|
|
63
|
+
/**
|
|
64
|
+
* Road rows yielded.
|
|
65
|
+
*/
|
|
66
|
+
roads: number;
|
|
67
|
+
/**
|
|
68
|
+
* Road rows skipped because their `ID` was not among the requested ids.
|
|
69
|
+
*/
|
|
70
|
+
unrequested: number;
|
|
71
|
+
/**
|
|
72
|
+
* Rows with a column count other than 34.
|
|
73
|
+
*/
|
|
74
|
+
malformed: number;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* A stats object with every count at zero.
|
|
78
|
+
*/
|
|
79
|
+
export declare function createOpenNamesReadStats(): OpenNamesReadStats;
|
|
80
|
+
/**
|
|
81
|
+
* Streams the road rows of one Open Names CSV.
|
|
82
|
+
*
|
|
83
|
+
* @param ids When given, only roads whose `ID` is in the set are yielded.
|
|
84
|
+
*/
|
|
85
|
+
export declare function readOpenNamesRoads(csvPath: string, stats: OpenNamesReadStats, ids?: ReadonlySet<string>): AsyncGenerator<OpenNamesRoad>;
|
|
86
|
+
/**
|
|
87
|
+
* The settlement's first-listed name from a `POPULATED_PLACE` value.
|
|
88
|
+
*
|
|
89
|
+
* OS writes a bilingual settlement as `<NAME1> / <NAME2>`, and NAME1 is the form the adapter emits.
|
|
90
|
+
*/
|
|
91
|
+
export declare function populatedPlaceName1(populatedPlace: string): string;
|
|
92
|
+
//# sourceMappingURL=open-names.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"open-names.d.ts","sourceRoot":"","sources":["../../../lib/gb/os-open/open-names.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAIH;;GAEG;AACH,eAAO,MAAM,kBAAkB,YAC9B,IAAI,EACJ,WAAW,EACX,OAAO,EACP,YAAY,EACZ,OAAO,EACP,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,sBAAsB,EACtB,uBAAuB,EACvB,UAAU,EACV,UAAU,EACV,UAAU,EACV,UAAU,EACV,mBAAmB,EACnB,uBAAuB,EACvB,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,gBAAgB,EAChB,oBAAoB,EACpB,qBAAqB,EACrB,QAAQ,EACR,YAAY,EACZ,SAAS,EACT,aAAa,EACb,wBAAwB,EACxB,iBAAiB,EACjB,kBAAkB,CACT,CAAA;AAOV;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,WAAW,CAAC,MAAM,CAAoD,CAAA;AAE1G;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B;;OAEG;IACH,EAAE,EAAE,MAAM,CAAA;IACV;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAA;IACxB;;;;;OAKG;IACH,cAAc,EAAE,MAAM,CAAA;IACtB,eAAe,EAAE,MAAM,CAAA;IACvB,aAAa,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;CACjB;AAED;;GAEG;AACH,wBAAgB,wBAAwB,IAAI,kBAAkB,CAE7D;AAED;;;;GAIG;AACH,wBAAuB,kBAAkB,CACxC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,kBAAkB,EACzB,GAAG,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GACvB,cAAc,CAAC,aAAa,CAAC,CAgC/B;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,cAAc,EAAE,MAAM,GAAG,MAAM,CAElE"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Reads the road rows of OS Open Names.
|
|
7
|
+
*
|
|
8
|
+
* The CSV product is 819 files, one per 10 km National Grid square, with no header row. OS ships
|
|
9
|
+
* the 34 column names in `Doc/OS_Open_Names_Header.csv`, and the first data row of each file opens
|
|
10
|
+
* with a byte-order mark, which this reader strips from the `ID` value.
|
|
11
|
+
*
|
|
12
|
+
* A road row's `ID` is the roadNameTOID that OS Open Roads links reference, such as
|
|
13
|
+
* `osgb4000000010927776`. Two `LOCAL_TYPE` values are roads: `Named Road`, and
|
|
14
|
+
* `Section Of Named Road` for a road that OS splits where it crosses a postcode district or a
|
|
15
|
+
* settlement.
|
|
16
|
+
*/
|
|
17
|
+
import { CSVSpliterator } from "spliterator";
|
|
18
|
+
/**
|
|
19
|
+
* The 34 columns of an OS Open Names CSV, in file order, as `Doc/OS_Open_Names_Header.csv` names them.
|
|
20
|
+
*/
|
|
21
|
+
export const OPEN_NAMES_COLUMNS = [
|
|
22
|
+
"ID",
|
|
23
|
+
"NAMES_URI",
|
|
24
|
+
"NAME1",
|
|
25
|
+
"NAME1_LANG",
|
|
26
|
+
"NAME2",
|
|
27
|
+
"NAME2_LANG",
|
|
28
|
+
"TYPE",
|
|
29
|
+
"LOCAL_TYPE",
|
|
30
|
+
"GEOMETRY_X",
|
|
31
|
+
"GEOMETRY_Y",
|
|
32
|
+
"MOST_DETAIL_VIEW_RES",
|
|
33
|
+
"LEAST_DETAIL_VIEW_RES",
|
|
34
|
+
"MBR_XMIN",
|
|
35
|
+
"MBR_YMIN",
|
|
36
|
+
"MBR_XMAX",
|
|
37
|
+
"MBR_YMAX",
|
|
38
|
+
"POSTCODE_DISTRICT",
|
|
39
|
+
"POSTCODE_DISTRICT_URI",
|
|
40
|
+
"POPULATED_PLACE",
|
|
41
|
+
"POPULATED_PLACE_URI",
|
|
42
|
+
"POPULATED_PLACE_TYPE",
|
|
43
|
+
"DISTRICT_BOROUGH",
|
|
44
|
+
"DISTRICT_BOROUGH_URI",
|
|
45
|
+
"DISTRICT_BOROUGH_TYPE",
|
|
46
|
+
"COUNTY_UNITARY",
|
|
47
|
+
"COUNTY_UNITARY_URI",
|
|
48
|
+
"COUNTY_UNITARY_TYPE",
|
|
49
|
+
"REGION",
|
|
50
|
+
"REGION_URI",
|
|
51
|
+
"COUNTRY",
|
|
52
|
+
"COUNTRY_URI",
|
|
53
|
+
"RELATED_SPATIAL_OBJECT",
|
|
54
|
+
"SAME_AS_DBPEDIA",
|
|
55
|
+
"SAME_AS_GEONAMES",
|
|
56
|
+
];
|
|
57
|
+
const column = Object.fromEntries(OPEN_NAMES_COLUMNS.map((name, index) => [name, index]));
|
|
58
|
+
/**
|
|
59
|
+
* The `LOCAL_TYPE` values that mark a road row.
|
|
60
|
+
*/
|
|
61
|
+
export const OPEN_NAMES_ROAD_TYPES = new Set(["Named Road", "Section Of Named Road"]);
|
|
62
|
+
/**
|
|
63
|
+
* A stats object with every count at zero.
|
|
64
|
+
*/
|
|
65
|
+
export function createOpenNamesReadStats() {
|
|
66
|
+
return { read: 0, roads: 0, unrequested: 0, malformed: 0 };
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Streams the road rows of one Open Names CSV.
|
|
70
|
+
*
|
|
71
|
+
* @param ids When given, only roads whose `ID` is in the set are yielded.
|
|
72
|
+
*/
|
|
73
|
+
export async function* readOpenNamesRoads(csvPath, stats, ids) {
|
|
74
|
+
for await (const row of CSVSpliterator.fromAsync(csvPath, { header: false })) {
|
|
75
|
+
stats.read++;
|
|
76
|
+
if (row.length !== OPEN_NAMES_COLUMNS.length) {
|
|
77
|
+
stats.malformed++;
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
if (!OPEN_NAMES_ROAD_TYPES.has(row[column.LOCAL_TYPE]))
|
|
81
|
+
continue;
|
|
82
|
+
const id = row[column.ID].replace(/^/u, "");
|
|
83
|
+
if (ids && !ids.has(id)) {
|
|
84
|
+
stats.unrequested++;
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
stats.roads++;
|
|
88
|
+
yield {
|
|
89
|
+
id,
|
|
90
|
+
name1: row[column.NAME1].trim(),
|
|
91
|
+
name2: row[column.NAME2].trim(),
|
|
92
|
+
postcodeDistrict: row[column.POSTCODE_DISTRICT].trim(),
|
|
93
|
+
populatedPlace: row[column.POPULATED_PLACE].trim(),
|
|
94
|
+
districtBorough: row[column.DISTRICT_BOROUGH].trim(),
|
|
95
|
+
countyUnitary: row[column.COUNTY_UNITARY].trim(),
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* The settlement's first-listed name from a `POPULATED_PLACE` value.
|
|
101
|
+
*
|
|
102
|
+
* OS writes a bilingual settlement as `<NAME1> / <NAME2>`, and NAME1 is the form the adapter emits.
|
|
103
|
+
*/
|
|
104
|
+
export function populatedPlaceName1(populatedPlace) {
|
|
105
|
+
return populatedPlace.split(" / ")[0].trim();
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=open-names.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"open-names.js","sourceRoot":"","sources":["../../../lib/gb/os-open/open-names.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C;;GAEG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG;IACjC,IAAI;IACJ,WAAW;IACX,OAAO;IACP,YAAY;IACZ,OAAO;IACP,YAAY;IACZ,MAAM;IACN,YAAY;IACZ,YAAY;IACZ,YAAY;IACZ,sBAAsB;IACtB,uBAAuB;IACvB,UAAU;IACV,UAAU;IACV,UAAU;IACV,UAAU;IACV,mBAAmB;IACnB,uBAAuB;IACvB,iBAAiB;IACjB,qBAAqB;IACrB,sBAAsB;IACtB,kBAAkB;IAClB,sBAAsB;IACtB,uBAAuB;IACvB,gBAAgB;IAChB,oBAAoB;IACpB,qBAAqB;IACrB,QAAQ;IACR,YAAY;IACZ,SAAS;IACT,aAAa;IACb,wBAAwB;IACxB,iBAAiB;IACjB,kBAAkB;CACT,CAAA;AAEV,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAGvF,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAwB,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,uBAAuB,CAAC,CAAC,CAAA;AAuD1G;;GAEG;AACH,MAAM,UAAU,wBAAwB;IACvC,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAA;AAC3D,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,kBAAkB,CACxC,OAAe,EACf,KAAyB,EACzB,GAAyB;IAEzB,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,cAAc,CAAC,SAAS,CAAW,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACxF,KAAK,CAAC,IAAI,EAAE,CAAA;QAEZ,IAAI,GAAG,CAAC,MAAM,KAAK,kBAAkB,CAAC,MAAM,EAAE,CAAC;YAC9C,KAAK,CAAC,SAAS,EAAE,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAE,CAAC;YAAE,SAAQ;QAEjE,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;QAE7C,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YACzB,KAAK,CAAC,WAAW,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,KAAK,CAAC,KAAK,EAAE,CAAA;QAEb,MAAM;YACL,EAAE;YACF,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAE,CAAC,IAAI,EAAE;YAChC,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAE,CAAC,IAAI,EAAE;YAChC,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAE,CAAC,IAAI,EAAE;YACvD,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,eAAe,CAAE,CAAC,IAAI,EAAE;YACnD,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,gBAAgB,CAAE,CAAC,IAAI,EAAE;YACrD,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,cAAc,CAAE,CAAC,IAAI,EAAE;SACjD,CAAA;IACF,CAAC;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,cAAsB;IACzD,OAAO,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAA;AAC9C,CAAC"}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Reads the named road links of an OS Open Roads GML tile.
|
|
7
|
+
*
|
|
8
|
+
* The GML product is one file per 100 km National Grid square. A tile holds `road:RoadLink`,
|
|
9
|
+
* `road:RoadNode` and `road:MotorwayJunction` features. Only the links hold a centreline and a
|
|
10
|
+
* name. A named link has a `road:name1` and a `road:roadNameTOID` whose `xlink:href` is `#` plus
|
|
11
|
+
* the roadNameTOID that OS Open Names uses as its `ID`. A link with only a road number carries a
|
|
12
|
+
* `road:roadNumberTOID` instead, and an unnamed link carries neither.
|
|
13
|
+
*
|
|
14
|
+
* The centreline is a `gml:LineString` in EPSG:27700, written as a `gml:posList` of easting and
|
|
15
|
+
* northing pairs. The reader honors the line's `srsDimension` so a 3D tile would still read its
|
|
16
|
+
* first two ordinates.
|
|
17
|
+
*/
|
|
18
|
+
import { type MarkupElement } from "@mailwoman/core/html/elements";
|
|
19
|
+
import type { PathBuilderLike } from "path-ts";
|
|
20
|
+
/**
|
|
21
|
+
* The feature element that carries a centreline.
|
|
22
|
+
*/
|
|
23
|
+
export declare const OPEN_ROADS_LINK_ELEMENT = "road:RoadLink";
|
|
24
|
+
/**
|
|
25
|
+
* One named road link.
|
|
26
|
+
*/
|
|
27
|
+
export interface OpenRoadsLink {
|
|
28
|
+
/**
|
|
29
|
+
* The link's `gml:id`.
|
|
30
|
+
*/
|
|
31
|
+
id: string;
|
|
32
|
+
/**
|
|
33
|
+
* `road:name1` as the tile spells it.
|
|
34
|
+
*/
|
|
35
|
+
name1: string;
|
|
36
|
+
/**
|
|
37
|
+
* The roadNameTOID, without the `#` of the reference, such as `osgb4000000010927776`.
|
|
38
|
+
*/
|
|
39
|
+
roadNameTOID: string;
|
|
40
|
+
/**
|
|
41
|
+
* The centreline as flat `[easting0, northing0, easting1, northing1, …]` in EPSG:27700 meters.
|
|
42
|
+
*/
|
|
43
|
+
vertices: Float64Array;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Counts from a read.
|
|
47
|
+
*/
|
|
48
|
+
export interface OpenRoadsReadStats {
|
|
49
|
+
/**
|
|
50
|
+
* Road links read.
|
|
51
|
+
*/
|
|
52
|
+
links: number;
|
|
53
|
+
/**
|
|
54
|
+
* Named links yielded.
|
|
55
|
+
*/
|
|
56
|
+
named: number;
|
|
57
|
+
/**
|
|
58
|
+
* Links with no `road:name1` or no `road:roadNameTOID`.
|
|
59
|
+
*/
|
|
60
|
+
unnamed: number;
|
|
61
|
+
/**
|
|
62
|
+
* Named links whose centreline held fewer than two vertices or a non-numeric ordinate.
|
|
63
|
+
*/
|
|
64
|
+
malformedGeometry: number;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* A stats object with every count at zero.
|
|
68
|
+
*/
|
|
69
|
+
export declare function createOpenRoadsReadStats(): OpenRoadsReadStats;
|
|
70
|
+
/**
|
|
71
|
+
* The named link a `road:RoadLink` element describes, or a reason it describes none.
|
|
72
|
+
*/
|
|
73
|
+
export declare function readOpenRoadsLink(element: MarkupElement): OpenRoadsLink | "unnamed" | "malformed-geometry";
|
|
74
|
+
/**
|
|
75
|
+
* Streams the named links of one Open Roads GML tile.
|
|
76
|
+
*/
|
|
77
|
+
export declare function readOpenRoadsLinks(gmlPath: PathBuilderLike, stats: OpenRoadsReadStats, options?: {
|
|
78
|
+
signal?: AbortSignal;
|
|
79
|
+
}): AsyncGenerator<OpenRoadsLink>;
|
|
80
|
+
//# sourceMappingURL=open-roads.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"open-roads.d.ts","sourceRoot":"","sources":["../../../lib/gb/os-open/open-roads.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EAAmD,KAAK,aAAa,EAAE,MAAM,+BAA+B,CAAA;AACnH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE9C;;GAEG;AACH,eAAO,MAAM,uBAAuB,kBAAkB,CAAA;AAEtD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B;;OAEG;IACH,EAAE,EAAE,MAAM,CAAA;IACV;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,iBAAiB,EAAE,MAAM,CAAA;CACzB;AAED;;GAEG;AACH,wBAAgB,wBAAwB,IAAI,kBAAkB,CAE7D;AAID;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,aAAa,GAAG,aAAa,GAAG,SAAS,GAAG,oBAAoB,CA2B1G;AAED;;GAEG;AACH,wBAAuB,kBAAkB,CACxC,OAAO,EAAE,eAAe,EACxB,KAAK,EAAE,kBAAkB,EACzB,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAO,GACpC,cAAc,CAAC,aAAa,CAAC,CAyB/B"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Reads the named road links of an OS Open Roads GML tile.
|
|
7
|
+
*
|
|
8
|
+
* The GML product is one file per 100 km National Grid square. A tile holds `road:RoadLink`,
|
|
9
|
+
* `road:RoadNode` and `road:MotorwayJunction` features. Only the links hold a centreline and a
|
|
10
|
+
* name. A named link has a `road:name1` and a `road:roadNameTOID` whose `xlink:href` is `#` plus
|
|
11
|
+
* the roadNameTOID that OS Open Names uses as its `ID`. A link with only a road number carries a
|
|
12
|
+
* `road:roadNumberTOID` instead, and an unnamed link carries neither.
|
|
13
|
+
*
|
|
14
|
+
* The centreline is a `gml:LineString` in EPSG:27700, written as a `gml:posList` of easting and
|
|
15
|
+
* northing pairs. The reader honors the line's `srsDimension` so a 3D tile would still read its
|
|
16
|
+
* first two ordinates.
|
|
17
|
+
*/
|
|
18
|
+
import { openReadStream } from "@mailwoman/core/fs/streams";
|
|
19
|
+
import { elementAtPath, streamMarkupElements, textAtPath } from "@mailwoman/core/html/elements";
|
|
20
|
+
/**
|
|
21
|
+
* The feature element that carries a centreline.
|
|
22
|
+
*/
|
|
23
|
+
export const OPEN_ROADS_LINK_ELEMENT = "road:RoadLink";
|
|
24
|
+
/**
|
|
25
|
+
* A stats object with every count at zero.
|
|
26
|
+
*/
|
|
27
|
+
export function createOpenRoadsReadStats() {
|
|
28
|
+
return { links: 0, named: 0, unnamed: 0, malformedGeometry: 0 };
|
|
29
|
+
}
|
|
30
|
+
const LINE_PATH = ["net:centrelineGeometry", "gml:LineString"];
|
|
31
|
+
/**
|
|
32
|
+
* The named link a `road:RoadLink` element describes, or a reason it describes none.
|
|
33
|
+
*/
|
|
34
|
+
export function readOpenRoadsLink(element) {
|
|
35
|
+
const name1 = (textAtPath(element, "road:name1") ?? "").trim();
|
|
36
|
+
const href = elementAtPath(element, "road:roadNameTOID")?.attributes["xlink:href"] ?? "";
|
|
37
|
+
const roadNameTOID = href.replace(/^#/u, "").trim();
|
|
38
|
+
if (!name1 || !roadNameTOID)
|
|
39
|
+
return "unnamed";
|
|
40
|
+
const dimension = Number(elementAtPath(element, ...LINE_PATH)?.attributes.srsDimension ?? "2") || 2;
|
|
41
|
+
const posList = (textAtPath(element, ...LINE_PATH, "gml:posList") ?? "").trim();
|
|
42
|
+
const ordinates = posList ? posList.split(/\s+/u) : [];
|
|
43
|
+
const vertexCount = Math.floor(ordinates.length / dimension);
|
|
44
|
+
if (vertexCount < 2 || ordinates.length % dimension !== 0)
|
|
45
|
+
return "malformed-geometry";
|
|
46
|
+
const vertices = new Float64Array(vertexCount * 2);
|
|
47
|
+
for (let vertex = 0; vertex < vertexCount; vertex++) {
|
|
48
|
+
const easting = Number(ordinates[vertex * dimension]);
|
|
49
|
+
const northing = Number(ordinates[vertex * dimension + 1]);
|
|
50
|
+
if (!Number.isFinite(easting) || !Number.isFinite(northing))
|
|
51
|
+
return "malformed-geometry";
|
|
52
|
+
vertices[vertex * 2] = easting;
|
|
53
|
+
vertices[vertex * 2 + 1] = northing;
|
|
54
|
+
}
|
|
55
|
+
return { id: element.attributes["gml:id"] ?? "", name1, roadNameTOID, vertices };
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Streams the named links of one Open Roads GML tile.
|
|
59
|
+
*/
|
|
60
|
+
export async function* readOpenRoadsLinks(gmlPath, stats, options = {}) {
|
|
61
|
+
for await (const element of streamMarkupElements(openReadStream(gmlPath), OPEN_ROADS_LINK_ELEMENT, {
|
|
62
|
+
xml: true,
|
|
63
|
+
signal: options.signal,
|
|
64
|
+
})) {
|
|
65
|
+
stats.links++;
|
|
66
|
+
const link = readOpenRoadsLink(element);
|
|
67
|
+
if (link === "unnamed") {
|
|
68
|
+
stats.unnamed++;
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
if (link === "malformed-geometry") {
|
|
72
|
+
stats.malformedGeometry++;
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
stats.named++;
|
|
76
|
+
yield link;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=open-roads.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"open-roads.js","sourceRoot":"","sources":["../../../lib/gb/os-open/open-roads.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAA;AAC3D,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,UAAU,EAAsB,MAAM,+BAA+B,CAAA;AAGnH;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,eAAe,CAAA;AA8CtD;;GAEG;AACH,MAAM,UAAU,wBAAwB;IACvC,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;AAChE,CAAC;AAED,MAAM,SAAS,GAAG,CAAC,wBAAwB,EAAE,gBAAgB,CAAU,CAAA;AAEvE;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAsB;IACvD,MAAM,KAAK,GAAG,CAAC,UAAU,CAAC,OAAO,EAAE,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAC9D,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,EAAE,mBAAmB,CAAC,EAAE,UAAU,CAAC,YAAY,CAAC,IAAI,EAAE,CAAA;IACxF,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAEnD,IAAI,CAAC,KAAK,IAAI,CAAC,YAAY;QAAE,OAAO,SAAS,CAAA;IAE7C,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,EAAE,UAAU,CAAC,YAAY,IAAI,GAAG,CAAC,IAAI,CAAC,CAAA;IACnG,MAAM,OAAO,GAAG,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,SAAS,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAC/E,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACtD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,CAAA;IAE5D,IAAI,WAAW,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,SAAS,KAAK,CAAC;QAAE,OAAO,oBAAoB,CAAA;IAEtF,MAAM,QAAQ,GAAG,IAAI,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC,CAAA;IAElD,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,WAAW,EAAE,MAAM,EAAE,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAA;QACrD,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAA;QAE1D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO,oBAAoB,CAAA;QAExF,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,OAAO,CAAA;QAC9B,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAA;IACpC,CAAC;IAED,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAA;AACjF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,kBAAkB,CACxC,OAAwB,EACxB,KAAyB,EACzB,OAAO,GAA6B,EAAE;IAEtC,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,oBAAoB,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,uBAAuB,EAAE;QAClG,GAAG,EAAE,IAAI;QACT,MAAM,EAAE,OAAO,CAAC,MAAM;KACtB,CAAC,EAAE,CAAC;QACJ,KAAK,CAAC,KAAK,EAAE,CAAA;QAEb,MAAM,IAAI,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;QAEvC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACxB,KAAK,CAAC,OAAO,EAAE,CAAA;YAEf,SAAQ;QACT,CAAC;QAED,IAAI,IAAI,KAAK,oBAAoB,EAAE,CAAC;YACnC,KAAK,CAAC,iBAAiB,EAAE,CAAA;YAEzB,SAAQ;QACT,CAAC;QAED,KAAK,CAAC,KAAK,EAAE,CAAA;QAEb,MAAM,IAAI,CAAA;IACX,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Downloads the three OS OpenData products the `gb-os-open` adapter joins, from the OS Downloads API.
|
|
7
|
+
*
|
|
8
|
+
* The API serves OpenData without a key. `GET /products/<id>/downloads` lists one archive per
|
|
9
|
+
* format and area with its OS-published MD5, and each archive is checked against that MD5 after
|
|
10
|
+
* transfer. A present archive whose MD5 already matches is kept rather than downloaded again.
|
|
11
|
+
*
|
|
12
|
+
* Each archive is unpacked into the subdirectory the adapter reads, `code-point-open/`,
|
|
13
|
+
* `open-names/` or `open-roads/`, so the fetch root is the adapter's `inputPath`. The archive
|
|
14
|
+
* stays beside its members, because its digest is the value the address-source register records.
|
|
15
|
+
*/
|
|
16
|
+
import { APIClient } from "@mailwoman/core/api";
|
|
17
|
+
import { type OSDownload } from "@mailwoman/spatial/gb/os-downloads";
|
|
18
|
+
import { type PathBuilderLike } from "path-ts";
|
|
19
|
+
import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download";
|
|
20
|
+
/**
|
|
21
|
+
* The license all three products are published under.
|
|
22
|
+
*/
|
|
23
|
+
export declare const GB_OS_OPEN_LICENSE_NAME = "Open Government Licence v3.0";
|
|
24
|
+
/**
|
|
25
|
+
* The attribution OGL v3 requires, in the wording OS asks for on Code-Point Open,
|
|
26
|
+
* which also carries Royal Mail and ONS data.
|
|
27
|
+
*/
|
|
28
|
+
export declare const GB_OS_OPEN_ATTRIBUTION = "Contains OS data \u00A9 Crown copyright and database right. Contains Royal Mail data \u00A9 Royal Mail copyright and database right. Contains National Statistics data \u00A9 Crown copyright and database right.";
|
|
29
|
+
/**
|
|
30
|
+
* One product the adapter reads: its OS Downloads API id, the archive format,
|
|
31
|
+
* and the subdirectory it unpacks into.
|
|
32
|
+
*/
|
|
33
|
+
export interface OSOpenProduct {
|
|
34
|
+
productID: string;
|
|
35
|
+
format: string;
|
|
36
|
+
directory: string;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* The three products, in the order they are fetched.
|
|
40
|
+
*/
|
|
41
|
+
export declare const GB_OS_OPEN_PRODUCTS: readonly OSOpenProduct[];
|
|
42
|
+
/**
|
|
43
|
+
* The manifest entry for one product's archive.
|
|
44
|
+
*/
|
|
45
|
+
export interface OSOpenFileManifest extends SourceManifest {
|
|
46
|
+
product_id: string;
|
|
47
|
+
format: string;
|
|
48
|
+
md5: string;
|
|
49
|
+
members: number;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* The archive of `product` among the listed downloads, the Great Britain one where an area is stated.
|
|
53
|
+
*
|
|
54
|
+
* @throws Naming the formats offered, when none matches.
|
|
55
|
+
*/
|
|
56
|
+
export declare function selectOSDownload(product: OSOpenProduct, downloads: readonly OSDownload[]): OSDownload;
|
|
57
|
+
/**
|
|
58
|
+
* The archives the OS Downloads API lists for one product.
|
|
59
|
+
*/
|
|
60
|
+
export declare function readOSDownloads(client: Pick<APIClient, "fetch">, product: OSOpenProduct, options?: {
|
|
61
|
+
signal?: AbortSignal;
|
|
62
|
+
}): Promise<OSDownload[]>;
|
|
63
|
+
/**
|
|
64
|
+
* Downloads and unpacks one product.
|
|
65
|
+
*/
|
|
66
|
+
export declare function downloadOSOpenProduct(client: Pick<APIClient, "fetch">, product: OSOpenProduct, options: {
|
|
67
|
+
outputDir: PathBuilderLike;
|
|
68
|
+
force?: boolean;
|
|
69
|
+
retries?: number;
|
|
70
|
+
retryDelayMs?: number;
|
|
71
|
+
signal?: AbortSignal;
|
|
72
|
+
report?: (line: string) => void;
|
|
73
|
+
}): Promise<{
|
|
74
|
+
entry: OSOpenFileManifest;
|
|
75
|
+
downloaded: boolean;
|
|
76
|
+
}>;
|
|
77
|
+
/**
|
|
78
|
+
* Per-invocation options for the registry entry.
|
|
79
|
+
*/
|
|
80
|
+
export interface FetchGBOSOpenOptions extends BaseFetchOptions {
|
|
81
|
+
force?: boolean;
|
|
82
|
+
retries?: number;
|
|
83
|
+
signal?: AbortSignal;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* The registry entry: fetches all three products and writes `MANIFEST.json` at the adapter's input root.
|
|
87
|
+
*
|
|
88
|
+
* A product that fails is counted and the others are still taken.
|
|
89
|
+
*/
|
|
90
|
+
export declare function fetchGBOSOpen(options: FetchGBOSOpenOptions, report?: (line: string) => void): Promise<FetchSummary>;
|
|
91
|
+
//# sourceMappingURL=os-open.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"os-open.d.ts","sourceRoot":"","sources":["../../../../lib/gb/tools/fetch/os-open.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAK/C,OAAO,EAAyB,KAAK,UAAU,EAAE,MAAM,oCAAoC,CAAA;AAC3F,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAG3D,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAA4B,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAGrH;;GAEG;AACH,eAAO,MAAM,uBAAuB,iCAAiC,CAAA;AAErE;;;GAGG;AACH,eAAO,MAAM,sBAAsB,sNACkK,CAAA;AAErM;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;CACjB;AAED;;GAEG;AACH,eAAO,MAAM,mBAAmB,EAAE,SAAS,aAAa,EAIvD,CAAA;AAED;;GAEG;AACH,MAAM,WAAW,kBAAmB,SAAQ,cAAc;IACzD,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,MAAM,CAAA;IACd,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,SAAS,UAAU,EAAE,GAAG,UAAU,CAYrG;AAED;;GAEG;AACH,wBAAsB,eAAe,CACpC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE,aAAa,EACtB,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAO,GACpC,OAAO,CAAC,UAAU,EAAE,CAAC,CAevB;AAED;;GAEG;AACH,wBAAsB,qBAAqB,CAC1C,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE;IACR,SAAS,EAAE,eAAe,CAAA;IAC1B,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B,GACC,OAAO,CAAC;IAAE,KAAK,EAAE,kBAAkB,CAAC;IAAC,UAAU,EAAE,OAAO,CAAA;CAAE,CAAC,CAsD7D;AAED;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,gBAAgB;IAC7D,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,WAAW,CAAA;CACpB;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAClC,OAAO,EAAE,oBAAoB,EAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAmDvB"}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Downloads the three OS OpenData products the `gb-os-open` adapter joins, from the OS Downloads API.
|
|
7
|
+
*
|
|
8
|
+
* The API serves OpenData without a key. `GET /products/<id>/downloads` lists one archive per
|
|
9
|
+
* format and area with its OS-published MD5, and each archive is checked against that MD5 after
|
|
10
|
+
* transfer. A present archive whose MD5 already matches is kept rather than downloaded again.
|
|
11
|
+
*
|
|
12
|
+
* Each archive is unpacked into the subdirectory the adapter reads, `code-point-open/`,
|
|
13
|
+
* `open-names/` or `open-roads/`, so the fetch root is the adapter's `inputPath`. The archive
|
|
14
|
+
* stays beside its members, because its digest is the value the address-source register records.
|
|
15
|
+
*/
|
|
16
|
+
import { APIClient } from "@mailwoman/core/api";
|
|
17
|
+
import { ByteFormatter } from "@mailwoman/core/fs/formatters";
|
|
18
|
+
import { makeDirectories } from "@mailwoman/core/fs/writers";
|
|
19
|
+
import { extractZipEntries } from "@mailwoman/core/fs/zip";
|
|
20
|
+
import { md5File, sha256File } from "@mailwoman/core/hash";
|
|
21
|
+
import { OS_DOWNLOADS_API_BASE } from "@mailwoman/spatial/gb/os-downloads";
|
|
22
|
+
import { PathBuilder } from "path-ts";
|
|
23
|
+
import { GB_OS_OPEN_ADAPTER_ID, GB_OS_OPEN_PRODUCT_DIRECTORIES } from "#gb/adapters/os-open/adapter";
|
|
24
|
+
import { downloadToFile, writeManifest } from "#tools/fetch/download";
|
|
25
|
+
/**
|
|
26
|
+
* The license all three products are published under.
|
|
27
|
+
*/
|
|
28
|
+
export const GB_OS_OPEN_LICENSE_NAME = "Open Government Licence v3.0";
|
|
29
|
+
/**
|
|
30
|
+
* The attribution OGL v3 requires, in the wording OS asks for on Code-Point Open,
|
|
31
|
+
* which also carries Royal Mail and ONS data.
|
|
32
|
+
*/
|
|
33
|
+
export const GB_OS_OPEN_ATTRIBUTION = "Contains OS data © Crown copyright and database right. Contains Royal Mail data © Royal Mail copyright and database right. Contains National Statistics data © Crown copyright and database right.";
|
|
34
|
+
/**
|
|
35
|
+
* The three products, in the order they are fetched.
|
|
36
|
+
*/
|
|
37
|
+
export const GB_OS_OPEN_PRODUCTS = [
|
|
38
|
+
{ productID: "CodePointOpen", format: "CSV", directory: GB_OS_OPEN_PRODUCT_DIRECTORIES.codePointOpen },
|
|
39
|
+
{ productID: "OpenNames", format: "CSV", directory: GB_OS_OPEN_PRODUCT_DIRECTORIES.openNames },
|
|
40
|
+
{ productID: "OpenRoads", format: "GML", directory: GB_OS_OPEN_PRODUCT_DIRECTORIES.openRoads },
|
|
41
|
+
];
|
|
42
|
+
/**
|
|
43
|
+
* The archive of `product` among the listed downloads, the Great Britain one where an area is stated.
|
|
44
|
+
*
|
|
45
|
+
* @throws Naming the formats offered, when none matches.
|
|
46
|
+
*/
|
|
47
|
+
export function selectOSDownload(product, downloads) {
|
|
48
|
+
const matches = downloads.filter((download) => download.format === product.format);
|
|
49
|
+
const chosen = matches.find((download) => download.area === "GB") ?? (matches.length === 1 ? matches[0] : undefined);
|
|
50
|
+
if (!chosen) {
|
|
51
|
+
throw new Error(`${GB_OS_OPEN_ADAPTER_ID}: OS Downloads API lists no single ${product.format} archive for ${product.productID} ` +
|
|
52
|
+
`(listed: ${downloads.map((download) => `${download.format}/${download.area}`).join(", ") || "nothing"})`);
|
|
53
|
+
}
|
|
54
|
+
return chosen;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* The archives the OS Downloads API lists for one product.
|
|
58
|
+
*/
|
|
59
|
+
export async function readOSDownloads(client, product, options = {}) {
|
|
60
|
+
const { data } = await client.fetch({
|
|
61
|
+
method: "GET",
|
|
62
|
+
url: `${OS_DOWNLOADS_API_BASE}/products/${product.productID}/downloads`,
|
|
63
|
+
timeout: 60_000,
|
|
64
|
+
signal: options.signal,
|
|
65
|
+
});
|
|
66
|
+
if (!Array.isArray(data)) {
|
|
67
|
+
throw new TypeError(`${GB_OS_OPEN_ADAPTER_ID}: OS Downloads API answered ${product.productID} with no download list`);
|
|
68
|
+
}
|
|
69
|
+
return data;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Downloads and unpacks one product.
|
|
73
|
+
*/
|
|
74
|
+
export async function downloadOSOpenProduct(client, product, options) {
|
|
75
|
+
const { report } = options;
|
|
76
|
+
const download = selectOSDownload(product, await readOSDownloads(client, product, { signal: options.signal }));
|
|
77
|
+
const productDir = PathBuilder.from(options.outputDir)(product.directory);
|
|
78
|
+
const archivePath = productDir(download.fileName);
|
|
79
|
+
report?.(`=== ${GB_OS_OPEN_ADAPTER_ID} / ${product.productID} ${download.fileName} (${ByteFormatter.formatIEC(download.size)})`);
|
|
80
|
+
await makeDirectories(productDir);
|
|
81
|
+
const present = options.force ? null : await md5File(archivePath).catch(() => null);
|
|
82
|
+
const downloaded = present !== download.md5;
|
|
83
|
+
if (downloaded) {
|
|
84
|
+
await downloadToFile({
|
|
85
|
+
url: download.url,
|
|
86
|
+
dest: archivePath,
|
|
87
|
+
retries: options.retries,
|
|
88
|
+
retryDelayMs: options.retryDelayMs,
|
|
89
|
+
report,
|
|
90
|
+
});
|
|
91
|
+
const md5 = await md5File(archivePath);
|
|
92
|
+
if (md5 !== download.md5) {
|
|
93
|
+
throw new Error(`${GB_OS_OPEN_ADAPTER_ID}: ${download.fileName} has md5 ${md5}, and OS publishes ${download.md5} for it`);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
report?.(` present, and its md5 matches the published ${download.md5}`);
|
|
98
|
+
}
|
|
99
|
+
const members = await extractZipEntries(archivePath, productDir, { skipExisting: true });
|
|
100
|
+
const sha256 = await sha256File(archivePath);
|
|
101
|
+
report?.(` ${members.length} members unpacked sha256=${sha256}`);
|
|
102
|
+
return {
|
|
103
|
+
entry: {
|
|
104
|
+
source_url: download.url,
|
|
105
|
+
filename: download.fileName,
|
|
106
|
+
product_id: product.productID,
|
|
107
|
+
format: product.format,
|
|
108
|
+
bytes: download.size,
|
|
109
|
+
md5: download.md5,
|
|
110
|
+
sha256,
|
|
111
|
+
members: members.length,
|
|
112
|
+
downloaded_at: new Date().toISOString(),
|
|
113
|
+
},
|
|
114
|
+
downloaded,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* The registry entry: fetches all three products and writes `MANIFEST.json` at the adapter's input root.
|
|
119
|
+
*
|
|
120
|
+
* A product that fails is counted and the others are still taken.
|
|
121
|
+
*/
|
|
122
|
+
export async function fetchGBOSOpen(options, report) {
|
|
123
|
+
const outputDir = options.outRoot(GB_OS_OPEN_ADAPTER_ID);
|
|
124
|
+
const files = [];
|
|
125
|
+
const failedCodes = [];
|
|
126
|
+
let fetched = 0;
|
|
127
|
+
let skipped = 0;
|
|
128
|
+
await using client = new APIClient({ displayName: GB_OS_OPEN_ADAPTER_ID, minRequestIntervalMs: 250, retry: true });
|
|
129
|
+
for (const product of GB_OS_OPEN_PRODUCTS) {
|
|
130
|
+
if (options.signal?.aborted)
|
|
131
|
+
break;
|
|
132
|
+
try {
|
|
133
|
+
const { entry, downloaded } = await downloadOSOpenProduct(client, product, {
|
|
134
|
+
outputDir,
|
|
135
|
+
force: options.force,
|
|
136
|
+
retries: options.retries,
|
|
137
|
+
retryDelayMs: options.retryDelayMs,
|
|
138
|
+
signal: options.signal,
|
|
139
|
+
report,
|
|
140
|
+
});
|
|
141
|
+
files.push(entry);
|
|
142
|
+
if (downloaded) {
|
|
143
|
+
fetched += 1;
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
skipped += 1;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
report?.(` ✗ ${product.productID}: ${error instanceof Error ? error.message : String(error)}`);
|
|
151
|
+
failedCodes.push(product.productID);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
const manifest = {
|
|
155
|
+
source: GB_OS_OPEN_ADAPTER_ID,
|
|
156
|
+
source_url: OS_DOWNLOADS_API_BASE,
|
|
157
|
+
license: GB_OS_OPEN_LICENSE_NAME,
|
|
158
|
+
attribution: GB_OS_OPEN_ATTRIBUTION,
|
|
159
|
+
downloaded_at: new Date().toISOString(),
|
|
160
|
+
files,
|
|
161
|
+
};
|
|
162
|
+
await writeManifest(outputDir("MANIFEST.json"), manifest);
|
|
163
|
+
report?.(` MANIFEST written to ${outputDir("MANIFEST.json")}`);
|
|
164
|
+
return { fetched, skipped, failed: failedCodes.length, failedCodes };
|
|
165
|
+
}
|
|
166
|
+
//# sourceMappingURL=os-open.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"os-open.js","sourceRoot":"","sources":["../../../../lib/gb/tools/fetch/os-open.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,qBAAqB,EAAmB,MAAM,oCAAoC,CAAA;AAC3F,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAE3D,OAAO,EAAE,qBAAqB,EAAE,8BAA8B,EAAE,MAAM,8BAA8B,CAAA;AAEpG,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAErE;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,8BAA8B,CAAA;AAErE;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAClC,oMAAoM,CAAA;AAYrM;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAA6B;IAC5D,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,8BAA8B,CAAC,aAAa,EAAE;IACtG,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,8BAA8B,CAAC,SAAS,EAAE;IAC9F,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,8BAA8B,CAAC,SAAS,EAAE;CAC9F,CAAA;AAYD;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAsB,EAAE,SAAgC;IACxF,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAA;IAClF,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAA;IAEpH,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACd,GAAG,qBAAqB,sCAAsC,OAAO,CAAC,MAAM,gBAAgB,OAAO,CAAC,SAAS,GAAG;YAC/G,YAAY,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,GAAG,CAC1G,CAAA;IACF,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,MAAgC,EAChC,OAAsB,EACtB,OAAO,GAA6B,EAAE;IAEtC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAAe;QACjD,MAAM,EAAE,KAAK;QACb,GAAG,EAAE,GAAG,qBAAqB,aAAa,OAAO,CAAC,SAAS,YAAY;QACvE,OAAO,EAAE,MAAM;QACf,MAAM,EAAE,OAAO,CAAC,MAAM;KACtB,CAAC,CAAA;IAEF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,SAAS,CAClB,GAAG,qBAAqB,+BAA+B,OAAO,CAAC,SAAS,wBAAwB,CAChG,CAAA;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAC1C,MAAgC,EAChC,OAAsB,EACtB,OAOC;IAED,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IAC1B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,EAAE,MAAM,eAAe,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;IAC9G,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACzE,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAEjD,MAAM,EAAE,CACP,OAAO,qBAAqB,MAAM,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC,QAAQ,KAAK,aAAa,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CACtH,CAAA;IAED,MAAM,eAAe,CAAC,UAAU,CAAC,CAAA;IAEjC,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;IACnF,MAAM,UAAU,GAAG,OAAO,KAAK,QAAQ,CAAC,GAAG,CAAA;IAE3C,IAAI,UAAU,EAAE,CAAC;QAChB,MAAM,cAAc,CAAC;YACpB,GAAG,EAAE,QAAQ,CAAC,GAAG;YACjB,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,MAAM;SACN,CAAC,CAAA;QAEF,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,CAAA;QAEtC,IAAI,GAAG,KAAK,QAAQ,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CACd,GAAG,qBAAqB,KAAK,QAAQ,CAAC,QAAQ,YAAY,GAAG,sBAAsB,QAAQ,CAAC,GAAG,SAAS,CACxG,CAAA;QACF,CAAC;IACF,CAAC;SAAM,CAAC;QACP,MAAM,EAAE,CAAC,gDAAgD,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAA;IACzE,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,WAAW,EAAE,UAAU,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAA;IACxF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,CAAA;IAE5C,MAAM,EAAE,CAAC,KAAK,OAAO,CAAC,MAAM,6BAA6B,MAAM,EAAE,CAAC,CAAA;IAElE,OAAO;QACN,KAAK,EAAE;YACN,UAAU,EAAE,QAAQ,CAAC,GAAG;YACxB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,QAAQ,CAAC,IAAI;YACpB,GAAG,EAAE,QAAQ,CAAC,GAAG;YACjB,MAAM;YACN,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACvC;QACD,UAAU;KACV,CAAA;AACF,CAAC;AAWD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,OAA6B,EAC7B,MAA+B;IAE/B,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAA;IACxD,MAAM,KAAK,GAAyB,EAAE,CAAA;IACtC,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,YAAY,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAElH,KAAK,MAAM,OAAO,IAAI,mBAAmB,EAAE,CAAC;QAC3C,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAK;QAElC,IAAI,CAAC;YACJ,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,MAAM,qBAAqB,CAAC,MAAM,EAAE,OAAO,EAAE;gBAC1E,SAAS;gBACT,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM;aACN,CAAC,CAAA;YAEF,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEjB,IAAI,UAAU,EAAE,CAAC;gBAChB,OAAO,IAAI,CAAC,CAAA;YACb,CAAC;iBAAM,CAAC;gBACP,OAAO,IAAI,CAAC,CAAA;YACb,CAAC;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,OAAO,OAAO,CAAC,SAAS,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YAE/F,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACpC,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAA6B;QAC1C,MAAM,EAAE,qBAAqB;QAC7B,UAAU,EAAE,qBAAqB;QACjC,OAAO,EAAE,uBAAuB;QAChC,WAAW,EAAE,sBAAsB;QACnC,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK;KACL,CAAA;IAED,MAAM,aAAa,CAAC,SAAS,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAEzD,MAAM,EAAE,CAAC,yBAAyB,SAAS,CAAC,eAAe,CAAC,EAAE,CAAC,CAAA;IAE/D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,CAAA;AACrE,CAAC"}
|