@mailwoman/corpus 10.6.0 → 10.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/PROVENANCE.md +24 -0
- package/data/address-source-register.json +18948 -8757
- package/data/corpus-source-inputs.json +4756 -0
- package/data/license-decisions.json +7260 -0
- package/data/licenses/whosonfirst-licenses.md +3 -2
- package/data/reviewed-ve-postcode-tuples.json +5 -72
- package/data/source-additions.json +2450 -0
- package/data/source-resolutions.json +87 -0
- package/data/training-manifests/v0.7.10-gb-county.effective.json +2335 -0
- package/data/training-manifests/v0.7.10-gb-county.json +1132 -0
- package/data/training-manifests/v0.7.2-address-systems.effective.json +3 -2
- package/data/training-manifests/v0.7.7-no-ppd.json +550 -0
- package/data/training-manifests/v0.7.8-rights-clean.json +538 -0
- package/data/training-manifests/v0.7.9-country-gap.json +1132 -0
- package/data/us-situs-sources.json +128 -0
- package/lib/adapters/cordis/adapter.ts +1 -1
- package/lib/adapters/libpostal-dictionary.ts +23 -3
- package/lib/adapters/osm/adapter.ts +42 -11
- package/lib/adapters.ts +14 -0
- package/lib/az/adapters/opendata/adapter.ts +389 -0
- package/lib/bs/adapters/gbpa/adapter.ts +248 -0
- package/lib/build/types.ts +16 -0
- package/lib/build.ts +21 -0
- package/lib/ee/adapters/ads/adapter.ts +18 -0
- package/lib/fo/adapters/adressur/adapter.ts +242 -0
- package/lib/gb/adapters/os-open/adapter.ts +12 -0
- package/lib/gb/county.ts +59 -0
- package/lib/gb/os-open/open/names.ts +41 -0
- package/lib/gu/adapters/bsp/adapter.ts +434 -0
- package/lib/im/adapters/land-registry/adapter.ts +340 -0
- package/lib/international/recipes/locale.ts +164 -19
- package/lib/li/adapters/llv/adapter.ts +246 -0
- package/lib/recipes/bare/postcode.ts +10 -34
- package/lib/recipes/country-balanced.ts +95 -78
- package/lib/recipes/intersection.ts +22 -30
- package/lib/recipes/po/box/cedex/recipe.ts +8 -5
- package/lib/recipes/po/box/cedex/sources.ts +2 -23
- package/lib/recipes/scaffold.ts +28 -3
- package/lib/recipes/sources.ts +112 -1
- package/lib/recipes/street/affix.ts +17 -29
- package/lib/recipes/sub/venue/context.ts +33 -9
- package/lib/recipes/sub/venue/sources.ts +54 -13
- package/lib/recipes/sub/venue.ts +68 -23
- package/lib/recipes/unit.ts +20 -7
- package/lib/registers.ts +33 -0
- package/lib/source-register/corpus-inputs.ts +528 -0
- package/lib/source-register/effective-manifest.ts +37 -5
- package/lib/source-register/types.ts +20 -0
- package/lib/south-asia/recipes/register.ts +1 -1
- package/lib/tools/overlay/manifest.ts +109 -1
- package/lib/tools/source-register/build.ts +47 -1
- package/lib/tools/variety-sample.ts +633 -0
- package/lib/us/adapters/usgov/nad/adapter.ts +39 -10
- package/lib/us/recipes/skeleton-sources.ts +156 -0
- package/lib/us/tools/nad-skeletons.ts +395 -0
- package/lib/uy/adapters/ide/adapter.ts +275 -0
- package/lib/ve/recipes/reviewed-postcode-tail.ts +29 -6
- package/out/adapters/cordis/adapter.d.ts +1 -1
- package/out/adapters/cordis/adapter.js +1 -1
- package/out/adapters/libpostal-dictionary.d.ts +9 -0
- package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
- package/out/adapters/libpostal-dictionary.js +19 -3
- package/out/adapters/libpostal-dictionary.js.map +1 -1
- package/out/adapters/osm/adapter.d.ts +12 -1
- package/out/adapters/osm/adapter.d.ts.map +1 -1
- package/out/adapters/osm/adapter.js +30 -13
- package/out/adapters/osm/adapter.js.map +1 -1
- package/out/adapters.d.ts.map +1 -1
- package/out/adapters.js +14 -0
- package/out/adapters.js.map +1 -1
- package/out/az/adapters/opendata/adapter.d.ts +88 -0
- package/out/az/adapters/opendata/adapter.d.ts.map +1 -0
- package/out/az/adapters/opendata/adapter.js +282 -0
- package/out/az/adapters/opendata/adapter.js.map +1 -0
- package/out/bs/adapters/gbpa/adapter.d.ts +64 -0
- package/out/bs/adapters/gbpa/adapter.d.ts.map +1 -0
- package/out/bs/adapters/gbpa/adapter.js +187 -0
- package/out/bs/adapters/gbpa/adapter.js.map +1 -0
- package/out/build/types.d.ts +15 -0
- package/out/build/types.d.ts.map +1 -1
- package/out/build/types.js.map +1 -1
- package/out/build.d.ts.map +1 -1
- package/out/build.js +17 -0
- package/out/build.js.map +1 -1
- package/out/ee/adapters/ads/adapter.d.ts +6 -0
- package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
- package/out/ee/adapters/ads/adapter.js +16 -0
- package/out/ee/adapters/ads/adapter.js.map +1 -1
- package/out/fo/adapters/adressur/adapter.d.ts +63 -0
- package/out/fo/adapters/adressur/adapter.d.ts.map +1 -0
- package/out/fo/adapters/adressur/adapter.js +177 -0
- package/out/fo/adapters/adressur/adapter.js.map +1 -0
- package/out/gb/adapters/os-open/adapter.d.ts +8 -0
- package/out/gb/adapters/os-open/adapter.d.ts.map +1 -1
- package/out/gb/adapters/os-open/adapter.js +11 -0
- package/out/gb/adapters/os-open/adapter.js.map +1 -1
- package/out/gb/county.d.ts +30 -0
- package/out/gb/county.d.ts.map +1 -0
- package/out/gb/county.js +51 -0
- package/out/gb/county.js.map +1 -0
- package/out/gb/os-open/open/names.d.ts +17 -0
- package/out/gb/os-open/open/names.d.ts.map +1 -1
- package/out/gb/os-open/open/names.js +19 -0
- package/out/gb/os-open/open/names.js.map +1 -1
- package/out/gu/adapters/bsp/adapter.d.ts +83 -0
- package/out/gu/adapters/bsp/adapter.d.ts.map +1 -0
- package/out/gu/adapters/bsp/adapter.js +346 -0
- package/out/gu/adapters/bsp/adapter.js.map +1 -0
- package/out/im/adapters/land-registry/adapter.d.ts +68 -0
- package/out/im/adapters/land-registry/adapter.d.ts.map +1 -0
- package/out/im/adapters/land-registry/adapter.js +261 -0
- package/out/im/adapters/land-registry/adapter.js.map +1 -0
- package/out/international/recipes/locale.d.ts +24 -4
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +107 -15
- package/out/international/recipes/locale.js.map +1 -1
- package/out/li/adapters/llv/adapter.d.ts +69 -0
- package/out/li/adapters/llv/adapter.d.ts.map +1 -0
- package/out/li/adapters/llv/adapter.js +181 -0
- package/out/li/adapters/llv/adapter.js.map +1 -0
- package/out/recipes/bare/postcode.d.ts.map +1 -1
- package/out/recipes/bare/postcode.js +9 -30
- package/out/recipes/bare/postcode.js.map +1 -1
- package/out/recipes/country-balanced.d.ts.map +1 -1
- package/out/recipes/country-balanced.js +82 -73
- package/out/recipes/country-balanced.js.map +1 -1
- package/out/recipes/intersection.d.ts +3 -2
- package/out/recipes/intersection.d.ts.map +1 -1
- package/out/recipes/intersection.js +20 -26
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/po/box/cedex/recipe.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/recipe.js +5 -4
- package/out/recipes/po/box/cedex/recipe.js.map +1 -1
- package/out/recipes/po/box/cedex/sources.d.ts +2 -45
- package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/sources.js +0 -20
- package/out/recipes/po/box/cedex/sources.js.map +1 -1
- package/out/recipes/scaffold.d.ts +15 -2
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +21 -1
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/sources.d.ts.map +1 -1
- package/out/recipes/sources.js +112 -1
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/street/affix.d.ts.map +1 -1
- package/out/recipes/street/affix.js +8 -20
- package/out/recipes/street/affix.js.map +1 -1
- package/out/recipes/sub/venue/context.d.ts +14 -3
- package/out/recipes/sub/venue/context.d.ts.map +1 -1
- package/out/recipes/sub/venue/context.js +29 -9
- package/out/recipes/sub/venue/context.js.map +1 -1
- package/out/recipes/sub/venue/sources.d.ts +41 -8
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +26 -7
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +35 -14
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/unit.d.ts +4 -0
- package/out/recipes/unit.d.ts.map +1 -1
- package/out/recipes/unit.js +7 -6
- package/out/recipes/unit.js.map +1 -1
- package/out/registers.d.ts +33 -0
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +33 -0
- package/out/registers.js.map +1 -1
- package/out/source-register/corpus-inputs.d.ts +240 -0
- package/out/source-register/corpus-inputs.d.ts.map +1 -0
- package/out/source-register/corpus-inputs.js +275 -0
- package/out/source-register/corpus-inputs.js.map +1 -0
- package/out/source-register/effective-manifest.d.ts +2 -0
- package/out/source-register/effective-manifest.d.ts.map +1 -1
- package/out/source-register/effective-manifest.js +33 -4
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/types.d.ts +21 -1
- package/out/source-register/types.d.ts.map +1 -1
- package/out/source-register/types.js +20 -0
- package/out/source-register/types.js.map +1 -1
- package/out/south-asia/recipes/register.js +1 -1
- package/out/south-asia/recipes/register.js.map +1 -1
- package/out/tools/overlay/manifest.d.ts +7 -0
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +72 -1
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/source-register/build.d.ts +9 -0
- package/out/tools/source-register/build.d.ts.map +1 -1
- package/out/tools/source-register/build.js +25 -3
- package/out/tools/source-register/build.js.map +1 -1
- package/out/tools/variety-sample.d.ts +196 -0
- package/out/tools/variety-sample.d.ts.map +1 -0
- package/out/tools/variety-sample.js +417 -0
- package/out/tools/variety-sample.js.map +1 -0
- package/out/us/adapters/usgov/nad/adapter.d.ts +80 -0
- package/out/us/adapters/usgov/nad/adapter.d.ts.map +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +27 -8
- package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
- package/out/us/recipes/skeleton-sources.d.ts +92 -0
- package/out/us/recipes/skeleton-sources.d.ts.map +1 -0
- package/out/us/recipes/skeleton-sources.js +121 -0
- package/out/us/recipes/skeleton-sources.js.map +1 -0
- package/out/us/tools/nad-skeletons.d.ts +120 -0
- package/out/us/tools/nad-skeletons.d.ts.map +1 -0
- package/out/us/tools/nad-skeletons.js +262 -0
- package/out/us/tools/nad-skeletons.js.map +1 -0
- package/out/uy/adapters/ide/adapter.d.ts +64 -0
- package/out/uy/adapters/ide/adapter.d.ts.map +1 -0
- package/out/uy/adapters/ide/adapter.js +206 -0
- package/out/uy/adapters/ide/adapter.js.map +1 -0
- package/out/ve/recipes/reviewed-postcode-tail.d.ts +6 -2
- package/out/ve/recipes/reviewed-postcode-tail.d.ts.map +1 -1
- package/out/ve/recipes/reviewed-postcode-tail.js +16 -5
- package/out/ve/recipes/reviewed-postcode-tail.js.map +1 -1
- package/package.json +14 -14
- package/lib/recipes/unit/sources.ts +0 -44
- package/lib/tools/fetch/ppd.ts +0 -121
- package/out/recipes/unit/sources.d.ts +0 -28
- package/out/recipes/unit/sources.d.ts.map +0 -1
- package/out/recipes/unit/sources.js +0 -30
- package/out/recipes/unit/sources.js.map +0 -1
- package/out/tools/fetch/ppd.d.ts +0 -15
- package/out/tools/fetch/ppd.d.ts.map +0 -1
- package/out/tools/fetch/ppd.js +0 -86
- package/out/tools/fetch/ppd.js.map +0 -1
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `llv-li`: the official index of building addresses of Liechtenstein, `Verzeichnis der
|
|
7
|
+
* Gebäudeadressen`, which the Liechtensteinische Landesverwaltung publishes on
|
|
8
|
+
* `service.geo.llv.li/download/` as one ZIP (`getfile.php?theme=officialAdress`).
|
|
9
|
+
*
|
|
10
|
+
* The ZIP holds `dataset.csv`, a shapefile of the same records and the license PDF. The CSV is
|
|
11
|
+
* Windows-1252 (`Äussere Wiesen` reads `\xc4ussere Wiesen`), CRLF-terminated and `;`-separated, with no
|
|
12
|
+
* quoting. The adapter reads `dataset.csv` from the ZIP, or from a path to the extracted CSV.
|
|
13
|
+
*
|
|
14
|
+
* The adapter reads `STN_NAME` as the street, `ADR_NUMBER` as the house number (it carries its own
|
|
15
|
+
* lower-case letter suffix, `12c`), `ZIP_ZIP4` as the postcode and `ZIP_NAME` as the postal locality.
|
|
16
|
+
* `COM_FOSNR` is the municipality's number and no column holds its name, so no region is emitted. The LI
|
|
17
|
+
* layout prints no region in any case.
|
|
18
|
+
*
|
|
19
|
+
* `STATUS` reads `real` or `planned`. A planned address is one assigned to a building not yet built, so
|
|
20
|
+
* the adapter refuses it. `OFFICIAL` reads `TRUE` or `FALSE`; an address the register marks as not
|
|
21
|
+
* official is refused too, because its text is not the designation the municipality assigned.
|
|
22
|
+
*
|
|
23
|
+
* The publisher grants reproduction, redistribution, enrichment, editing, combination and commercial use
|
|
24
|
+
* under its `ABI/OGD-Version 2 vom 5.2.2021` terms, on condition that the source
|
|
25
|
+
* `Liechtensteinische Landesverwaltung` is stated and the terms are enclosed or linked. Every row
|
|
26
|
+
* records that license.
|
|
27
|
+
*/
|
|
28
|
+
import { type AdapterOptions, type CanonicalRow, type CorpusAdapter } from "#types";
|
|
29
|
+
/**
|
|
30
|
+
* Registry id for this adapter, stamped into every row it emits.
|
|
31
|
+
*/
|
|
32
|
+
export declare const LLV_LI_ADAPTER_ID = "llv-li";
|
|
33
|
+
/**
|
|
34
|
+
* The Landesverwaltung's own terms, `ABI/OGD-Version 2 vom 5.2.2021`.
|
|
35
|
+
* They have no SPDX identifier.
|
|
36
|
+
*/
|
|
37
|
+
export declare const LLV_LI_LICENSE = "LicenseRef-LI-ABI-OGD-2";
|
|
38
|
+
/**
|
|
39
|
+
* Every jurisdiction this adapter emits, checked against a caller's `--country`.
|
|
40
|
+
*/
|
|
41
|
+
export declare const LLV_LI_COUNTRIES: readonly string[];
|
|
42
|
+
/**
|
|
43
|
+
* The ZIP member that holds the address table.
|
|
44
|
+
*/
|
|
45
|
+
export declare const LLV_LI_MEMBER = "dataset.csv";
|
|
46
|
+
/**
|
|
47
|
+
* The columns the adapter reads from `dataset.csv`.
|
|
48
|
+
*/
|
|
49
|
+
export interface LLVLiechtensteinRecord {
|
|
50
|
+
ADR_EGAID: string;
|
|
51
|
+
STN_NAME: string;
|
|
52
|
+
ADR_NUMBER: string;
|
|
53
|
+
ZIP_ZIP4: string;
|
|
54
|
+
ZIP_NAME: string;
|
|
55
|
+
STATUS: string;
|
|
56
|
+
OFFICIAL: string;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Maps one record onto the components the corpus asserts, or returns null when the row is refused.
|
|
60
|
+
*
|
|
61
|
+
* Each refusal increments its reason in `dropped`.
|
|
62
|
+
*/
|
|
63
|
+
export declare function componentsForLLVRecord(record: LLVLiechtensteinRecord, dropped?: Pick<AdapterOptions, "dropped">): CanonicalRow["components"] | null;
|
|
64
|
+
export declare function createLLVLiechtensteinAdapter(): CorpusAdapter;
|
|
65
|
+
/**
|
|
66
|
+
* The configured adapter instance registered with the corpus builder.
|
|
67
|
+
*/
|
|
68
|
+
export declare const llvLiechtensteinAdapter: CorpusAdapter;
|
|
69
|
+
//# sourceMappingURL=adapter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../lib/li/adapters/llv/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAaH,OAAO,EAEN,KAAK,cAAc,EACnB,KAAK,YAAY,EACjB,KAAK,aAAa,EAGlB,MAAM,QAAQ,CAAA;AAEf;;GAEG;AACH,eAAO,MAAM,iBAAiB,WAAW,CAAA;AAEzC;;;GAGG;AACH,eAAO,MAAM,cAAc,4BAA4B,CAAA;AAEvD;;GAEG;AACH,eAAO,MAAM,gBAAgB,EAAE,SAAS,MAAM,EAAW,CAAA;AAEzD;;GAEG;AACH,eAAO,MAAM,aAAa,gBAAgB,CAAA;AAE1C;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;CAChB;AAQD;;;;GAIG;AACH,wBAAgB,sBAAsB,CACrC,MAAM,EAAE,sBAAsB,EAC9B,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,SAAS,CAAC,GACvC,YAAY,CAAC,YAAY,CAAC,GAAG,IAAI,CA0DnC;AAmBD,wBAAgB,6BAA6B,IAAI,aAAa,CAmE7D;AAED;;GAEG;AACH,eAAO,MAAM,uBAAuB,eAAkC,CAAA"}
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `llv-li`: the official index of building addresses of Liechtenstein, `Verzeichnis der
|
|
7
|
+
* Gebäudeadressen`, which the Liechtensteinische Landesverwaltung publishes on
|
|
8
|
+
* `service.geo.llv.li/download/` as one ZIP (`getfile.php?theme=officialAdress`).
|
|
9
|
+
*
|
|
10
|
+
* The ZIP holds `dataset.csv`, a shapefile of the same records and the license PDF. The CSV is
|
|
11
|
+
* Windows-1252 (`Äussere Wiesen` reads `\xc4ussere Wiesen`), CRLF-terminated and `;`-separated, with no
|
|
12
|
+
* quoting. The adapter reads `dataset.csv` from the ZIP, or from a path to the extracted CSV.
|
|
13
|
+
*
|
|
14
|
+
* The adapter reads `STN_NAME` as the street, `ADR_NUMBER` as the house number (it carries its own
|
|
15
|
+
* lower-case letter suffix, `12c`), `ZIP_ZIP4` as the postcode and `ZIP_NAME` as the postal locality.
|
|
16
|
+
* `COM_FOSNR` is the municipality's number and no column holds its name, so no region is emitted. The LI
|
|
17
|
+
* layout prints no region in any case.
|
|
18
|
+
*
|
|
19
|
+
* `STATUS` reads `real` or `planned`. A planned address is one assigned to a building not yet built, so
|
|
20
|
+
* the adapter refuses it. `OFFICIAL` reads `TRUE` or `FALSE`; an address the register marks as not
|
|
21
|
+
* official is refused too, because its text is not the designation the municipality assigned.
|
|
22
|
+
*
|
|
23
|
+
* The publisher grants reproduction, redistribution, enrichment, editing, combination and commercial use
|
|
24
|
+
* under its `ABI/OGD-Version 2 vom 5.2.2021` terms, on condition that the source
|
|
25
|
+
* `Liechtensteinische Landesverwaltung` is stated and the terms are enclosed or linked. Every row
|
|
26
|
+
* records that license.
|
|
27
|
+
*/
|
|
28
|
+
import { formatAddressRow } from "@mailwoman/codex/address/format";
|
|
29
|
+
import { matchCountryPostcode } from "@mailwoman/codex/postcode/country-formats";
|
|
30
|
+
import { readLocalBuffer } from "@mailwoman/core/fs/readers";
|
|
31
|
+
import { decodeBytes } from "@mailwoman/core/fs/streams";
|
|
32
|
+
import { readZipEntry } from "@mailwoman/core/fs/zip";
|
|
33
|
+
import { resolvePath } from "path-ts";
|
|
34
|
+
import { CSVSpliterator } from "spliterator";
|
|
35
|
+
import { UnsupportedCountryError } from "#adapters/errors";
|
|
36
|
+
import { stableSourceIDFromParts } from "#adapters/source-id";
|
|
37
|
+
import { SourceRegister } from "#registers";
|
|
38
|
+
import { AddressRole, countDropped, SurfaceOrigin, } from "#types";
|
|
39
|
+
/**
|
|
40
|
+
* Registry id for this adapter, stamped into every row it emits.
|
|
41
|
+
*/
|
|
42
|
+
export const LLV_LI_ADAPTER_ID = "llv-li";
|
|
43
|
+
/**
|
|
44
|
+
* The Landesverwaltung's own terms, `ABI/OGD-Version 2 vom 5.2.2021`.
|
|
45
|
+
* They have no SPDX identifier.
|
|
46
|
+
*/
|
|
47
|
+
export const LLV_LI_LICENSE = "LicenseRef-LI-ABI-OGD-2";
|
|
48
|
+
/**
|
|
49
|
+
* Every jurisdiction this adapter emits, checked against a caller's `--country`.
|
|
50
|
+
*/
|
|
51
|
+
export const LLV_LI_COUNTRIES = ["LI"];
|
|
52
|
+
/**
|
|
53
|
+
* The ZIP member that holds the address table.
|
|
54
|
+
*/
|
|
55
|
+
export const LLV_LI_MEMBER = "dataset.csv";
|
|
56
|
+
function present(value) {
|
|
57
|
+
const trimmed = value?.trim() ?? "";
|
|
58
|
+
return trimmed || null;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Maps one record onto the components the corpus asserts, or returns null when the row is refused.
|
|
62
|
+
*
|
|
63
|
+
* Each refusal increments its reason in `dropped`.
|
|
64
|
+
*/
|
|
65
|
+
export function componentsForLLVRecord(record, dropped) {
|
|
66
|
+
const tally = dropped ?? {};
|
|
67
|
+
if (record.STATUS?.trim() !== "real") {
|
|
68
|
+
countDropped(tally, record.STATUS?.trim() === "planned" ? "row:status-planned" : "row:status-unknown");
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
if (record.OFFICIAL?.trim().toUpperCase() !== "TRUE") {
|
|
72
|
+
countDropped(tally, "row:not-official");
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
const street = present(record.STN_NAME);
|
|
76
|
+
if (!street) {
|
|
77
|
+
countDropped(tally, "row:no-street");
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
const number = present(record.ADR_NUMBER);
|
|
81
|
+
if (!number) {
|
|
82
|
+
countDropped(tally, "row:no-house-number");
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
const components = {
|
|
86
|
+
street,
|
|
87
|
+
house_number: number,
|
|
88
|
+
};
|
|
89
|
+
const postcode = present(record.ZIP_ZIP4);
|
|
90
|
+
const matched = postcode ? matchCountryPostcode(postcode, "LI") : null;
|
|
91
|
+
if (matched) {
|
|
92
|
+
components.postcode = matched;
|
|
93
|
+
}
|
|
94
|
+
else if (postcode) {
|
|
95
|
+
countDropped(tally, "component:postcode:not-country-format");
|
|
96
|
+
}
|
|
97
|
+
const locality = present(record.ZIP_NAME);
|
|
98
|
+
if (locality) {
|
|
99
|
+
components.locality = locality;
|
|
100
|
+
}
|
|
101
|
+
if (!components.locality && !components.postcode) {
|
|
102
|
+
countDropped(tally, "row:no-locality-or-postcode");
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
return components;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* The bytes of `dataset.csv`, read from the publisher's ZIP or from the extracted file.
|
|
109
|
+
*
|
|
110
|
+
* The table is about 2 MB, so it is held whole.
|
|
111
|
+
*/
|
|
112
|
+
async function datasetBytes(inputPath) {
|
|
113
|
+
if (!inputPath.toLowerCase().endsWith(".zip"))
|
|
114
|
+
return readLocalBuffer(inputPath);
|
|
115
|
+
const chunks = [];
|
|
116
|
+
for await (const chunk of readZipEntry(inputPath, LLV_LI_MEMBER)) {
|
|
117
|
+
chunks.push(chunk);
|
|
118
|
+
}
|
|
119
|
+
return Buffer.concat(chunks);
|
|
120
|
+
}
|
|
121
|
+
export function createLLVLiechtensteinAdapter() {
|
|
122
|
+
return {
|
|
123
|
+
id: LLV_LI_ADAPTER_ID,
|
|
124
|
+
defaultLicense: LLV_LI_LICENSE,
|
|
125
|
+
addressRole: AddressRole.Premise,
|
|
126
|
+
register: SourceRegister.LiechtensteinGebaeudeadressen,
|
|
127
|
+
surface: SurfaceOrigin.Rendered,
|
|
128
|
+
description: "Verzeichnis der Gebäudeadressen (Liechtensteinische Landesverwaltung): national building addresses.",
|
|
129
|
+
async *rows(opts) {
|
|
130
|
+
if (opts.country && !LLV_LI_COUNTRIES.includes(opts.country)) {
|
|
131
|
+
throw new UnsupportedCountryError(LLV_LI_ADAPTER_ID, LLV_LI_COUNTRIES, opts.country);
|
|
132
|
+
}
|
|
133
|
+
const text = decodeBytes(await datasetBytes(resolvePath(opts.inputPath).toString()), "windows-1252");
|
|
134
|
+
const records = CSVSpliterator.from(Buffer.from(text, "utf8"), {
|
|
135
|
+
normalizeKeys: false,
|
|
136
|
+
columnDelimiter: ";",
|
|
137
|
+
});
|
|
138
|
+
let emitted = 0;
|
|
139
|
+
for (const record of records) {
|
|
140
|
+
if (opts.signal?.aborted)
|
|
141
|
+
return;
|
|
142
|
+
if (opts.limit !== undefined && emitted >= opts.limit)
|
|
143
|
+
return;
|
|
144
|
+
const components = componentsForLLVRecord(record, opts);
|
|
145
|
+
if (!components)
|
|
146
|
+
continue;
|
|
147
|
+
const rendered = formatAddressRow(components, "LI", {
|
|
148
|
+
singleLine: true,
|
|
149
|
+
});
|
|
150
|
+
if (!rendered) {
|
|
151
|
+
countDropped(opts, "row:render-failed");
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
for (const tag of rendered.unplaced) {
|
|
155
|
+
countDropped(opts, `component:${tag}:not-rendered`);
|
|
156
|
+
}
|
|
157
|
+
const { raw, components: aligned } = rendered;
|
|
158
|
+
yield {
|
|
159
|
+
raw,
|
|
160
|
+
components: aligned,
|
|
161
|
+
country: "LI",
|
|
162
|
+
locale: "de-LI",
|
|
163
|
+
source: LLV_LI_ADAPTER_ID,
|
|
164
|
+
// `ADR_EGAID` is the federal address identifier, unique per row of the register.
|
|
165
|
+
source_id: stableSourceIDFromParts(LLV_LI_ADAPTER_ID, {
|
|
166
|
+
...aligned,
|
|
167
|
+
egaid: record.ADR_EGAID ?? "",
|
|
168
|
+
}),
|
|
169
|
+
corpus_version: "",
|
|
170
|
+
license: LLV_LI_LICENSE,
|
|
171
|
+
};
|
|
172
|
+
emitted++;
|
|
173
|
+
}
|
|
174
|
+
},
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* The configured adapter instance registered with the corpus builder.
|
|
179
|
+
*/
|
|
180
|
+
export const llvLiechtensteinAdapter = createLLVLiechtensteinAdapter();
|
|
181
|
+
//# sourceMappingURL=adapter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../lib/li/adapters/llv/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAA;AAClE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAA;AAChF,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAA;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAA;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EACN,WAAW,EAIX,YAAY,EACZ,aAAa,GACb,MAAM,QAAQ,CAAA;AAEf;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,QAAQ,CAAA;AAEzC;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,yBAAyB,CAAA;AAEvD;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAsB,CAAC,IAAI,CAAC,CAAA;AAEzD;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,aAAa,CAAA;AAe1C,SAAS,OAAO,CAAC,KAAyB;IACzC,MAAM,OAAO,GAAG,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;IAEnC,OAAO,OAAO,IAAI,IAAI,CAAA;AACvB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CACrC,MAA8B,EAC9B,OAAyC;IAEzC,MAAM,KAAK,GAAG,OAAO,IAAI,EAAE,CAAA;IAE3B,IAAI,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,MAAM,EAAE,CAAC;QACtC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAA;QAEtG,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,MAAM,EAAE,CAAC;QACtD,YAAY,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAAA;QAEvC,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAEvC,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,CAAA;QAEpC,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAEzC,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,YAAY,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAA;QAE1C,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,MAAM,UAAU,GAA+B;QAC9C,MAAM;QACN,YAAY,EAAE,MAAM;KACpB,CAAA;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,oBAAoB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAEtE,IAAI,OAAO,EAAE,CAAC;QACb,UAAU,CAAC,QAAQ,GAAG,OAAO,CAAA;IAC9B,CAAC;SAAM,IAAI,QAAQ,EAAE,CAAC;QACrB,YAAY,CAAC,KAAK,EAAE,uCAAuC,CAAC,CAAA;IAC7D,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAEzC,IAAI,QAAQ,EAAE,CAAC;QACd,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;IAC/B,CAAC;IAED,IAAI,CAAC,UAAU,CAAC,QAAQ,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;QAClD,YAAY,CAAC,KAAK,EAAE,6BAA6B,CAAC,CAAA;QAElD,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,OAAO,UAAU,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,YAAY,CAAC,SAAiB;IAC5C,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,eAAe,CAAC,SAAS,CAAC,CAAA;IAEhF,MAAM,MAAM,GAAiB,EAAE,CAAA;IAE/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,YAAY,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,CAAC;QAClE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACnB,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;AAC7B,CAAC;AAED,MAAM,UAAU,6BAA6B;IAC5C,OAAO;QACN,EAAE,EAAE,iBAAiB;QACrB,cAAc,EAAE,cAAc;QAC9B,WAAW,EAAE,WAAW,CAAC,OAAO;QAChC,QAAQ,EAAE,cAAc,CAAC,6BAA6B;QACtD,OAAO,EAAE,aAAa,CAAC,QAAQ;QAC/B,WAAW,EAAE,qGAAqG;QAElH,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9D,MAAM,IAAI,uBAAuB,CAAC,iBAAiB,EAAE,gBAAgB,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YACrF,CAAC;YAED,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAC,EAAE,cAAc,CAAC,CAAA;YAEpG,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAyB,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE;gBACtF,aAAa,EAAE,KAAK;gBACpB,eAAe,EAAE,GAAG;aACpB,CAAC,CAAA;YAEF,IAAI,OAAO,GAAG,CAAC,CAAA;YAEf,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,OAAM;gBAEhC,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;oBAAE,OAAM;gBAE7D,MAAM,UAAU,GAAG,sBAAsB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;gBAEvD,IAAI,CAAC,UAAU;oBAAE,SAAQ;gBAEzB,MAAM,QAAQ,GAAG,gBAAgB,CAAC,UAAU,EAAE,IAAI,EAAE;oBACnD,UAAU,EAAE,IAAI;iBAChB,CAAC,CAAA;gBAEF,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACf,YAAY,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAA;oBAEvC,SAAQ;gBACT,CAAC;gBAED,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBACrC,YAAY,CAAC,IAAI,EAAE,aAAa,GAAG,eAAe,CAAC,CAAA;gBACpD,CAAC;gBAED,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAA;gBAE7C,MAAM;oBACL,GAAG;oBACH,UAAU,EAAE,OAAO;oBACnB,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,OAAO;oBACf,MAAM,EAAE,iBAAiB;oBACzB,iFAAiF;oBACjF,SAAS,EAAE,uBAAuB,CAAC,iBAAiB,EAAE;wBACrD,GAAG,OAAO;wBACV,KAAK,EAAE,MAAM,CAAC,SAAS,IAAI,EAAE;qBAC7B,CAAC;oBACF,cAAc,EAAE,EAAE;oBAClB,OAAO,EAAE,cAAc;iBACvB,CAAA;gBAED,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,6BAA6B,EAAE,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"postcode.d.ts","sourceRoot":"","sources":["../../../lib/recipes/bare/postcode.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAQH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAI9C,OAAO,EAAiB,KAAK,YAAY,EAAkC,MAAM,mBAAmB,CAAA;
|
|
1
|
+
{"version":3,"file":"postcode.d.ts","sourceRoot":"","sources":["../../../lib/recipes/bare/postcode.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAQH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAI9C,OAAO,EAAiB,KAAK,YAAY,EAAkC,MAAM,mBAAmB,CAAA;AAuFpG;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAM9F;AAED;;GAEG;AACH,wBAAsB,0BAA0B,CAAC,CAAC,SAAS,eAAe,EACzE,KAAK,EAAE,SAAS,CAAC,EAAE,EACnB,MAAM,GAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,OAAO,CAAc,GAChD,OAAO,CAAC,CAAC,EAAE,CAAC,CAId;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAM9E;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAI3D;AAED;;GAEG;AACH,eAAO,MAAM,kBAAkB,EAAE,YA+JhC,CAAA"}
|
|
@@ -39,39 +39,18 @@ const BARE_POSTCODE_PROVENANCE = {
|
|
|
39
39
|
};
|
|
40
40
|
const EXTRACTED = dataRootPath("openaddresses", "extracted");
|
|
41
41
|
/**
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
|
|
45
|
-
const SWEDISH_MUNICIPALITIES = [
|
|
46
|
-
"alingsas",
|
|
47
|
-
"gislaved",
|
|
48
|
-
"goteborg",
|
|
49
|
-
"helsingborg",
|
|
50
|
-
"hoganas",
|
|
51
|
-
"kalmar",
|
|
52
|
-
"kristinehamn",
|
|
53
|
-
"malmo",
|
|
54
|
-
"nacka",
|
|
55
|
-
"savsjö",
|
|
56
|
-
"stockholm",
|
|
57
|
-
"uppsala",
|
|
58
|
-
"vasteras",
|
|
59
|
-
"vaxholm",
|
|
60
|
-
"vaxjo",
|
|
61
|
-
"Österåker",
|
|
62
|
-
];
|
|
63
|
-
/**
|
|
42
|
+
* Each file's publisher has a register row whose elected decision permits transform
|
|
43
|
+
* and train, and `corpus-source-inputs.json` lists that row under `codex-bare-postcode`.
|
|
44
|
+
*
|
|
64
45
|
* Greece is deliberately absent: its only member declares a `postcode` column with no values over
|
|
65
|
-
* 10,877 rows, though `gr_postcode` shares `NNN NN` with CZ/SK
|
|
46
|
+
* 10,877 rows, though `gr_postcode` shares `NNN NN` with CZ/SK, so no row here claims to be Greek.
|
|
47
|
+
* Sweden's sixteen `se/municipality_of_*` files are absent because their register row,
|
|
48
|
+
* `se-property-building-2`, records no decision permitting training.
|
|
66
49
|
*/
|
|
67
50
|
const SOURCES = [
|
|
68
51
|
{ csv: EXTRACTED("cz", "countrywide.csv"), country: "CZ" },
|
|
69
52
|
{ csv: EXTRACTED("sk", "countrywide.csv"), country: "SK" },
|
|
70
53
|
{ csv: EXTRACTED("nl", "countrywide.csv"), country: "NL" },
|
|
71
|
-
...SWEDISH_MUNICIPALITIES.map((name) => ({
|
|
72
|
-
csv: EXTRACTED("se", `municipality_of_${name}.csv`),
|
|
73
|
-
country: "SE",
|
|
74
|
-
})),
|
|
75
54
|
];
|
|
76
55
|
/**
|
|
77
56
|
* The postcode format to use on its own and the locale to stamp.
|
|
@@ -153,7 +132,7 @@ export function detectedAsPostcode(surface) {
|
|
|
153
132
|
*/
|
|
154
133
|
export const barePostcodeRecipe = {
|
|
155
134
|
name: "bare-postcode",
|
|
156
|
-
description: "A postcode alone, in its country's written form — the shape no other recipe output carries (CZ/SK/
|
|
135
|
+
description: "A postcode alone, in its country's written form — the shape no other recipe output carries (CZ/SK/NL)",
|
|
157
136
|
mode: "generate",
|
|
158
137
|
options: [
|
|
159
138
|
{
|
|
@@ -167,8 +146,8 @@ export const barePostcodeRecipe = {
|
|
|
167
146
|
let skipped = 0;
|
|
168
147
|
let unrecognized = 0;
|
|
169
148
|
// Preflight the complete input set before writing a row.
|
|
170
|
-
// A missing
|
|
171
|
-
//
|
|
149
|
+
// A missing file would otherwise produce a plausible artifact with less
|
|
150
|
+
// coverage than the recipe declares.
|
|
172
151
|
const missing = await findMissingPostcodeSources(SOURCES.map(({ csv }) => csv));
|
|
173
152
|
if (missing.length) {
|
|
174
153
|
throw new Error(`bare-postcode is missing ${missing.length} of ${SOURCES.length} required OpenAddresses files, ` +
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"postcode.js","sourceRoot":"","sources":["../../../lib/recipes/bare/postcode.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAA;AAGvE,OAAO,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC/E,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAA;AACpE,OAAO,EAAE,aAAa,EAAqB,cAAc,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAA;AACpG,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAA;AAEtC;;;;;GAKG;AACH;;;GAGG;AACH,MAAM,oBAAoB,GAAG,mBAAmB,CAAC,qBAAqB,CAAC,CAAA;AAEvE,MAAM,wBAAwB,GAAG;IAChC,QAAQ,EAAE,cAAc,CAAC,KAAK;IAC9B,OAAO,EAAE,aAAa,CAAC,QAAQ;IAC/B,iCAAiC;IACjC,6CAA6C;IAC7C,YAAY,EAAE,IAAI;CAClB,CAAA;
|
|
1
|
+
{"version":3,"file":"postcode.js","sourceRoot":"","sources":["../../../lib/recipes/bare/postcode.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAA;AAGvE,OAAO,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC/E,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAA;AACpE,OAAO,EAAE,aAAa,EAAqB,cAAc,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAA;AACpG,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAA;AAEtC;;;;;GAKG;AACH;;;GAGG;AACH,MAAM,oBAAoB,GAAG,mBAAmB,CAAC,qBAAqB,CAAC,CAAA;AAEvE,MAAM,wBAAwB,GAAG;IAChC,QAAQ,EAAE,cAAc,CAAC,KAAK;IAC9B,OAAO,EAAE,aAAa,CAAC,QAAQ;IAC/B,iCAAiC;IACjC,6CAA6C;IAC7C,YAAY,EAAE,IAAI;CAClB,CAAA;AAUD,MAAM,SAAS,GAAG,YAAY,CAAC,eAAe,EAAE,WAAW,CAAC,CAAA;AAE5D;;;;;;;;GAQG;AACH,MAAM,OAAO,GAAqB;IACjC,EAAE,GAAG,EAAE,SAAS,CAAC,IAAI,EAAE,iBAAiB,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE;IAC1D,EAAE,GAAG,EAAE,SAAS,CAAC,IAAI,EAAE,iBAAiB,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE;IAC1D,EAAE,GAAG,EAAE,SAAS,CAAC,IAAI,EAAE,iBAAiB,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE;CAC1D,CAAA;AAED;;;;;;GAMG;AACH,MAAM,aAAa,GAAmF,IAAI,GAAG,CAAC;IAC7G,oCAAoC;IACpC,wFAAwF;IACxF,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAChD,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAChD,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAChD,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAChD,4CAA4C;IAC5C,uCAAuC;IACvC;QACC,IAAI;QACJ;YACC,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9G;KACD;CACD,CAAC,CAAA;AAEF;;;;;GAKG;AACH,SAAS,WAAW,CAAC,OAAe;IACnC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAA;IAEvC,OAAO,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,CAAA;AAC/D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,KAAuB,EAAE,KAAa,EAAE,IAAY;IACnF,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;IAE3C,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IAErC,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAA;AAC/D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC/C,KAAmB,EACnB,MAAM,GAAkC,UAAU;IAElD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAEpG,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;AAC1E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAe,EAAE,QAAgB;IACnE,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAA;IAE5D,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAA;IAEpB,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;AAC1E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAe;IACjD,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,IAAI,CAClD,CAAC,GAAG,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,OAAO,CAAC,MAAM,CAChG,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAiB;IAC/C,IAAI,EAAE,eAAe;IACrB,WAAW,EAAE,uGAAuG;IACpH,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE;QACR;YACC,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,sFAAsF;SACnG;KACD;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,YAAY,GAAG,CAAC,CAAA;QAEpB,yDAAyD;QACzD,wEAAwE;QACxE,qCAAqC;QACrC,MAAM,OAAO,GAAG,MAAM,0BAA0B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAA;QAE/E,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACd,4BAA4B,OAAO,CAAC,MAAM,OAAO,OAAO,CAAC,MAAM,iCAAiC;gBAC/F,iBAAiB,OAAO,CAAC,CAAC,CAAC,gEAAgE;gBAC3F,6CAA6C,CAC9C,CAAA;QACF,CAAC;QAED,mFAAmF;QACnF,wFAAwF;QACxF,uFAAuF;QACvF,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;QACvE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAA;QAC/F,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QACpE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,GAAG,EAAU,CAAC,CAAC,CAAC,CAAA;QAExF,oEAAoE;QACpE,iFAAiF;QACjF,MAAM,WAAW,GAAG,MAAM,kBAAkB,EAAE,CAAA;QAC9C,IAAI,YAAY,GAAG,CAAC,CAAA;QAEpB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YAE9C,IAAI,CAAC,IAAI;gBAAE,SAAQ;YAEnB,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBACpD,IAAI,EAAE,CAAA;gBAEN,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC;qBACxC,IAAI,EAAE;qBACN,WAAW,EAAE;qBACb,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;gBAEzB,MAAM,YAAY,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAE,CAAA;gBAExD,IAAI,CAAC,OAAO,IAAI,sBAAsB,CAAC,OAAO,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC9E,OAAO,EAAE,CAAA;oBAET,SAAQ;gBACT,CAAC;gBAED,6EAA6E;gBAC7E,iFAAiF;gBACjF,yCAAyC;gBACzC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;oBAChG,YAAY,EAAE,CAAA;oBAEd,OAAO,EAAE,CAAA;oBAET,SAAQ;gBACT,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;oBAClC,OAAO,EAAE,CAAA;oBAET,SAAQ;gBACT,CAAC;gBAED,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;YAC1B,CAAC;QACF,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAE,CAAA;YACxC,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAA;YAC/F,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;YACpF,MAAM,QAAQ,GAAG,eAAe,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAE,EAAE,YAAY,EAAE,WAAW,CAAC,CAAA;YAEzF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;gBAErC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;oBAChC,IAAI,UAAU,CAAC,GAAG,CAAC,OAAO,CAAE,IAAI,MAAM;wBAAE,MAAK;oBAE7C,uEAAuE;oBACvE,6CAA6C;oBAC7C,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC;wBAClC,YAAY,EAAE,CAAA;wBAEd,SAAQ;oBACT,CAAC;oBAED,MAAM,UAAU,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAA;oBAExC,MAAM,SAAS,GAAG;wBACjB,GAAG,EAAE,OAAO;wBACZ,UAAU;wBACV,OAAO;wBACP,MAAM,EAAE,IAAI,CAAC,MAAM;wBACnB,MAAM,EAAE,oBAAoB;wBAC5B,SAAS,EAAE,cAAc,CAAC,oBAAoB,EAAE;4BAC/C,GAAG,UAAU;4BACb,CAAC,EAAE,OAAO;yBACV,CAAC;wBACF,cAAc,EAAE,QAAQ;wBACxB,OAAO,EACN,oGAAoG;qBACrG,CAAA;oBAED,IAAI,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,wBAAwB,CAAC,EAAE,CAAC;wBAChF,OAAO,EAAE,CAAA;wBACT,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,CAAC,OAAO,CAAE,GAAG,CAAC,CAAC,CAAA;oBACtD,CAAC;yBAAM,CAAC;wBACP,OAAO,EAAE,CAAA;oBACV,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;QAED,sFAAsF;QACtF,sFAAsF;QACtF,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CACd,GAAG,YAAY,6EAA6E;gBAC3F,+FAA+F,CAChG,CAAA;QACF,CAAC;QAED,wEAAwE;QACxE,0EAA0E;QAC1E,2DAA2D;QAC3D,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CACd,sCAAsC,OAAO,CAAC,MAAM,qBAAqB;gBACxE,yDAAyD,CAC1D,CAAA;QACF,CAAC;QAED,4EAA4E;QAC5E,yFAAyF;QACzF,OAAO,CAAC,KAAK,CACZ,oBAAoB,YAAY,CAAC,cAAc,EAAE,yCAAyC;YACzF,IAAI,WAAW,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,CACrD,CAAA;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAClC,CAAC;CACD,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"country-balanced.d.ts","sourceRoot":"","sources":["../../lib/recipes/country-balanced.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAYH,OAAO,EAAiC,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAA;
|
|
1
|
+
{"version":3,"file":"country-balanced.d.ts","sourceRoot":"","sources":["../../lib/recipes/country-balanced.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAYH,OAAO,EAAiC,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAqYpF;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,YA+FnC,CAAA"}
|
|
@@ -16,6 +16,7 @@ import { stableSourceID } from "#adapters/source-id";
|
|
|
16
16
|
import { readOATuples, requireRegister } from "#recipes/scaffold";
|
|
17
17
|
import { defaultRecipeSource } from "#recipes/sources";
|
|
18
18
|
import { SurfaceOrigin } from "#types";
|
|
19
|
+
import { US_SKELETON_EVAL_SOURCE, US_SKELETON_TRAIN_SOURCES } from "#us/recipes/skeleton-sources";
|
|
19
20
|
import { alignRow } from "#utils";
|
|
20
21
|
// Country names are drawn without regard to the address's own country.
|
|
21
22
|
const COUNTRY_FORM_POOL = (() => {
|
|
@@ -30,72 +31,67 @@ const COUNTRY_FORM_POOL = (() => {
|
|
|
30
31
|
*/
|
|
31
32
|
const COUNTRY_ABSENT_PROB = 0.3;
|
|
32
33
|
/**
|
|
33
|
-
* The training sources.
|
|
34
|
+
* The training sources, in two groups that {@linkcode readSourcePool} gives equal halves of the read budget.
|
|
35
|
+
*
|
|
36
|
+
* The first group is the shared US skeletons.
|
|
37
|
+
* They take the half that the four OpenAddresses US state files held before `us/il/cook` was withdrawn.
|
|
38
|
+
*
|
|
39
|
+
* The number of US files therefore leaves the pool's country mix unchanged.
|
|
34
40
|
*/
|
|
35
|
-
const
|
|
36
|
-
{
|
|
37
|
-
zip
|
|
38
|
-
csv
|
|
39
|
-
iso2: "US",
|
|
40
|
-
region: "IA",
|
|
41
|
-
order: "us",
|
|
42
|
-
},
|
|
43
|
-
{ zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", iso2: "US", region: "IL", order: "us" },
|
|
44
|
-
{
|
|
45
|
-
zip: dataRootPath("oa-cache", "us__mt__statewide.zip"),
|
|
46
|
-
csv: "us/mt/statewide.csv",
|
|
47
|
-
iso2: "US",
|
|
48
|
-
region: "MT",
|
|
49
|
-
order: "us",
|
|
50
|
-
},
|
|
51
|
-
{
|
|
52
|
-
zip: dataRootPath("oa-cache", "us__sd__statewide.zip"),
|
|
53
|
-
csv: "us/sd/statewide.csv",
|
|
41
|
+
const SOURCE_GROUPS = [
|
|
42
|
+
US_SKELETON_TRAIN_SOURCES.map(({ zip, csv, region, register }) => ({
|
|
43
|
+
zip,
|
|
44
|
+
csv,
|
|
54
45
|
iso2: "US",
|
|
55
|
-
region
|
|
46
|
+
region,
|
|
56
47
|
order: "us",
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
48
|
+
register,
|
|
49
|
+
})),
|
|
50
|
+
[
|
|
51
|
+
{
|
|
52
|
+
zip: dataRootPath("oa-cache", "de__sn__statewide.zip"),
|
|
53
|
+
csv: "de/sn/statewide.csv",
|
|
54
|
+
iso2: "DE",
|
|
55
|
+
region: "",
|
|
56
|
+
order: "eu",
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
zip: dataRootPath("oa-cache", "fr__countrywide.zip"),
|
|
60
|
+
csv: "fr/countrywide.csv",
|
|
61
|
+
iso2: "FR",
|
|
62
|
+
region: "",
|
|
63
|
+
order: "fr",
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
zip: dataRootPath("oa-cache", "it__countrywide.zip"),
|
|
67
|
+
csv: "it/countrywide.csv",
|
|
68
|
+
iso2: "IT",
|
|
69
|
+
region: "",
|
|
70
|
+
order: "eu",
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
zip: dataRootPath("oa-cache", "nl__countrywide.zip"),
|
|
74
|
+
csv: "nl/countrywide.csv",
|
|
75
|
+
iso2: "NL",
|
|
76
|
+
region: "",
|
|
77
|
+
order: "eu",
|
|
78
|
+
},
|
|
79
|
+
],
|
|
86
80
|
];
|
|
87
81
|
/**
|
|
88
|
-
* The held-out sources that `--golden` reads.
|
|
82
|
+
* The held-out sources that `--golden` reads, one per group.
|
|
89
83
|
*/
|
|
90
|
-
const
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
84
|
+
const EVAL_SOURCE_GROUPS = [
|
|
85
|
+
[
|
|
86
|
+
{
|
|
87
|
+
zip: US_SKELETON_EVAL_SOURCE.zip,
|
|
88
|
+
csv: US_SKELETON_EVAL_SOURCE.csv,
|
|
89
|
+
iso2: "US",
|
|
90
|
+
region: US_SKELETON_EVAL_SOURCE.region,
|
|
91
|
+
order: "us",
|
|
92
|
+
},
|
|
93
|
+
],
|
|
94
|
+
[{ zip: dataRootPath("oa-cache", "de__berlin.zip"), csv: "de/berlin.csv", iso2: "DE", region: "", order: "eu" }],
|
|
99
95
|
];
|
|
100
96
|
/**
|
|
101
97
|
* Reads up to `limit` tuples from one source archive.
|
|
@@ -108,9 +104,30 @@ async function readTuples(source, limit) {
|
|
|
108
104
|
region: row.region || source.region,
|
|
109
105
|
iso2: source.iso2,
|
|
110
106
|
order: source.order,
|
|
107
|
+
...(source.register ? { register: source.register } : {}),
|
|
111
108
|
}),
|
|
112
109
|
});
|
|
113
110
|
}
|
|
111
|
+
/**
|
|
112
|
+
* Reads the tuple pool for `count` rows: a threefold read of the target,
|
|
113
|
+
* which leaves room for skipped rows.
|
|
114
|
+
*
|
|
115
|
+
* Each group takes an equal share of that read, split evenly between its sources.
|
|
116
|
+
*/
|
|
117
|
+
async function readSourcePool(groups, count) {
|
|
118
|
+
const pool = [];
|
|
119
|
+
for (const group of groups) {
|
|
120
|
+
const perSource = Math.ceil((count * 3) / groups.length / group.length);
|
|
121
|
+
for (const s of group) {
|
|
122
|
+
const t = await readTuples(s, perSource);
|
|
123
|
+
console.error(` ${s.csv} (${s.iso2}): ${t.length} tuples`);
|
|
124
|
+
for (const x of t) {
|
|
125
|
+
pool.push(x);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return pool;
|
|
130
|
+
}
|
|
114
131
|
// The share of country draws taken from the curated surface forms instead of canonical names.
|
|
115
132
|
const SURFACE_FORM_SHARE = 0.6;
|
|
116
133
|
/**
|
|
@@ -297,17 +314,7 @@ export const countryBalancedRecipe = {
|
|
|
297
314
|
const count = opts.count;
|
|
298
315
|
const random = makeMulberry32(opts.seed);
|
|
299
316
|
const source = opts.sourceName ?? defaultRecipeSource("synth-country");
|
|
300
|
-
const
|
|
301
|
-
// A threefold read of the target leaves room for skipped rows.
|
|
302
|
-
const perSource = Math.ceil((count * 3) / sources.length);
|
|
303
|
-
const pool = [];
|
|
304
|
-
for (const s of sources) {
|
|
305
|
-
const t = await readTuples(s, perSource);
|
|
306
|
-
console.error(` ${s.csv} (${s.iso2}): ${t.length} tuples`);
|
|
307
|
-
for (const x of t) {
|
|
308
|
-
pool.push(x);
|
|
309
|
-
}
|
|
310
|
-
}
|
|
317
|
+
const pool = await readSourcePool(opts.golden ? EVAL_SOURCE_GROUPS : SOURCE_GROUPS, count);
|
|
311
318
|
if (!pool.length) {
|
|
312
319
|
throw new Error(`No tuples — are the cached OA zips present in ${dataRootPath("oa-cache")}?`);
|
|
313
320
|
}
|
|
@@ -319,6 +326,7 @@ export const countryBalancedRecipe = {
|
|
|
319
326
|
const roll = random();
|
|
320
327
|
let rendered;
|
|
321
328
|
let rowISO2;
|
|
329
|
+
let rowRegister;
|
|
322
330
|
if (roll < HOMOGRAPH_FRAC) {
|
|
323
331
|
const h = renderHomograph(random);
|
|
324
332
|
rendered = h;
|
|
@@ -334,6 +342,7 @@ export const countryBalancedRecipe = {
|
|
|
334
342
|
const country = pickCountry(random);
|
|
335
343
|
rendered = renderCountry(random, t, country);
|
|
336
344
|
rowISO2 = t.iso2;
|
|
345
|
+
rowRegister = t.register;
|
|
337
346
|
if (country && !rendered.raw.includes(country)) {
|
|
338
347
|
skipped++;
|
|
339
348
|
continue;
|
|
@@ -354,7 +363,7 @@ export const countryBalancedRecipe = {
|
|
|
354
363
|
source,
|
|
355
364
|
source_id: stableSourceID(source, components),
|
|
356
365
|
corpus_version: "0.4.0",
|
|
357
|
-
license: "OpenAddresses multi-locale skeletons + injected ISO-3166 country surface forms (codex)",
|
|
366
|
+
license: "OpenAddresses multi-locale and National Address Database US skeletons + injected ISO-3166 country surface forms (codex)",
|
|
358
367
|
};
|
|
359
368
|
const aligned = alignRow(canonical);
|
|
360
369
|
if (aligned.kind !== "labeled" || !aligned.row) {
|
|
@@ -365,7 +374,7 @@ export const countryBalancedRecipe = {
|
|
|
365
374
|
...aligned.row,
|
|
366
375
|
recipe: "country",
|
|
367
376
|
base_source_id: null,
|
|
368
|
-
register: requireRegister(opts, "country-balanced"),
|
|
377
|
+
register: rowRegister ?? requireRegister(opts, "country-balanced"),
|
|
369
378
|
surface: SurfaceOrigin.Composed,
|
|
370
379
|
}));
|
|
371
380
|
emitted++;
|