@mailwoman/corpus 10.6.0 → 10.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/PROVENANCE.md +24 -0
- package/data/address-source-register.json +18948 -8757
- package/data/corpus-source-inputs.json +4756 -0
- package/data/license-decisions.json +7260 -0
- package/data/licenses/whosonfirst-licenses.md +3 -2
- package/data/reviewed-ve-postcode-tuples.json +5 -72
- package/data/source-additions.json +2450 -0
- package/data/source-resolutions.json +87 -0
- package/data/training-manifests/v0.7.10-gb-county.effective.json +2335 -0
- package/data/training-manifests/v0.7.10-gb-county.json +1132 -0
- package/data/training-manifests/v0.7.2-address-systems.effective.json +3 -2
- package/data/training-manifests/v0.7.7-no-ppd.json +550 -0
- package/data/training-manifests/v0.7.8-rights-clean.json +538 -0
- package/data/training-manifests/v0.7.9-country-gap.json +1132 -0
- package/data/us-situs-sources.json +128 -0
- package/lib/adapters/cordis/adapter.ts +1 -1
- package/lib/adapters/libpostal-dictionary.ts +23 -3
- package/lib/adapters/osm/adapter.ts +42 -11
- package/lib/adapters.ts +14 -0
- package/lib/az/adapters/opendata/adapter.ts +389 -0
- package/lib/bs/adapters/gbpa/adapter.ts +248 -0
- package/lib/build/types.ts +16 -0
- package/lib/build.ts +21 -0
- package/lib/ee/adapters/ads/adapter.ts +18 -0
- package/lib/fo/adapters/adressur/adapter.ts +242 -0
- package/lib/gb/adapters/os-open/adapter.ts +12 -0
- package/lib/gb/county.ts +59 -0
- package/lib/gb/os-open/open/names.ts +41 -0
- package/lib/gu/adapters/bsp/adapter.ts +434 -0
- package/lib/im/adapters/land-registry/adapter.ts +340 -0
- package/lib/international/recipes/locale.ts +164 -19
- package/lib/li/adapters/llv/adapter.ts +246 -0
- package/lib/recipes/bare/postcode.ts +10 -34
- package/lib/recipes/country-balanced.ts +95 -78
- package/lib/recipes/intersection.ts +22 -30
- package/lib/recipes/po/box/cedex/recipe.ts +8 -5
- package/lib/recipes/po/box/cedex/sources.ts +2 -23
- package/lib/recipes/scaffold.ts +28 -3
- package/lib/recipes/sources.ts +112 -1
- package/lib/recipes/street/affix.ts +17 -29
- package/lib/recipes/sub/venue/context.ts +33 -9
- package/lib/recipes/sub/venue/sources.ts +54 -13
- package/lib/recipes/sub/venue.ts +68 -23
- package/lib/recipes/unit.ts +20 -7
- package/lib/registers.ts +33 -0
- package/lib/source-register/corpus-inputs.ts +528 -0
- package/lib/source-register/effective-manifest.ts +37 -5
- package/lib/source-register/types.ts +20 -0
- package/lib/south-asia/recipes/register.ts +1 -1
- package/lib/tools/overlay/manifest.ts +109 -1
- package/lib/tools/source-register/build.ts +47 -1
- package/lib/tools/variety-sample.ts +633 -0
- package/lib/us/adapters/usgov/nad/adapter.ts +39 -10
- package/lib/us/recipes/skeleton-sources.ts +156 -0
- package/lib/us/tools/nad-skeletons.ts +395 -0
- package/lib/uy/adapters/ide/adapter.ts +275 -0
- package/lib/ve/recipes/reviewed-postcode-tail.ts +29 -6
- package/out/adapters/cordis/adapter.d.ts +1 -1
- package/out/adapters/cordis/adapter.js +1 -1
- package/out/adapters/libpostal-dictionary.d.ts +9 -0
- package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
- package/out/adapters/libpostal-dictionary.js +19 -3
- package/out/adapters/libpostal-dictionary.js.map +1 -1
- package/out/adapters/osm/adapter.d.ts +12 -1
- package/out/adapters/osm/adapter.d.ts.map +1 -1
- package/out/adapters/osm/adapter.js +30 -13
- package/out/adapters/osm/adapter.js.map +1 -1
- package/out/adapters.d.ts.map +1 -1
- package/out/adapters.js +14 -0
- package/out/adapters.js.map +1 -1
- package/out/az/adapters/opendata/adapter.d.ts +88 -0
- package/out/az/adapters/opendata/adapter.d.ts.map +1 -0
- package/out/az/adapters/opendata/adapter.js +282 -0
- package/out/az/adapters/opendata/adapter.js.map +1 -0
- package/out/bs/adapters/gbpa/adapter.d.ts +64 -0
- package/out/bs/adapters/gbpa/adapter.d.ts.map +1 -0
- package/out/bs/adapters/gbpa/adapter.js +187 -0
- package/out/bs/adapters/gbpa/adapter.js.map +1 -0
- package/out/build/types.d.ts +15 -0
- package/out/build/types.d.ts.map +1 -1
- package/out/build/types.js.map +1 -1
- package/out/build.d.ts.map +1 -1
- package/out/build.js +17 -0
- package/out/build.js.map +1 -1
- package/out/ee/adapters/ads/adapter.d.ts +6 -0
- package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
- package/out/ee/adapters/ads/adapter.js +16 -0
- package/out/ee/adapters/ads/adapter.js.map +1 -1
- package/out/fo/adapters/adressur/adapter.d.ts +63 -0
- package/out/fo/adapters/adressur/adapter.d.ts.map +1 -0
- package/out/fo/adapters/adressur/adapter.js +177 -0
- package/out/fo/adapters/adressur/adapter.js.map +1 -0
- package/out/gb/adapters/os-open/adapter.d.ts +8 -0
- package/out/gb/adapters/os-open/adapter.d.ts.map +1 -1
- package/out/gb/adapters/os-open/adapter.js +11 -0
- package/out/gb/adapters/os-open/adapter.js.map +1 -1
- package/out/gb/county.d.ts +30 -0
- package/out/gb/county.d.ts.map +1 -0
- package/out/gb/county.js +51 -0
- package/out/gb/county.js.map +1 -0
- package/out/gb/os-open/open/names.d.ts +17 -0
- package/out/gb/os-open/open/names.d.ts.map +1 -1
- package/out/gb/os-open/open/names.js +19 -0
- package/out/gb/os-open/open/names.js.map +1 -1
- package/out/gu/adapters/bsp/adapter.d.ts +83 -0
- package/out/gu/adapters/bsp/adapter.d.ts.map +1 -0
- package/out/gu/adapters/bsp/adapter.js +346 -0
- package/out/gu/adapters/bsp/adapter.js.map +1 -0
- package/out/im/adapters/land-registry/adapter.d.ts +68 -0
- package/out/im/adapters/land-registry/adapter.d.ts.map +1 -0
- package/out/im/adapters/land-registry/adapter.js +261 -0
- package/out/im/adapters/land-registry/adapter.js.map +1 -0
- package/out/international/recipes/locale.d.ts +24 -4
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +107 -15
- package/out/international/recipes/locale.js.map +1 -1
- package/out/li/adapters/llv/adapter.d.ts +69 -0
- package/out/li/adapters/llv/adapter.d.ts.map +1 -0
- package/out/li/adapters/llv/adapter.js +181 -0
- package/out/li/adapters/llv/adapter.js.map +1 -0
- package/out/recipes/bare/postcode.d.ts.map +1 -1
- package/out/recipes/bare/postcode.js +9 -30
- package/out/recipes/bare/postcode.js.map +1 -1
- package/out/recipes/country-balanced.d.ts.map +1 -1
- package/out/recipes/country-balanced.js +82 -73
- package/out/recipes/country-balanced.js.map +1 -1
- package/out/recipes/intersection.d.ts +3 -2
- package/out/recipes/intersection.d.ts.map +1 -1
- package/out/recipes/intersection.js +20 -26
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/po/box/cedex/recipe.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/recipe.js +5 -4
- package/out/recipes/po/box/cedex/recipe.js.map +1 -1
- package/out/recipes/po/box/cedex/sources.d.ts +2 -45
- package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/sources.js +0 -20
- package/out/recipes/po/box/cedex/sources.js.map +1 -1
- package/out/recipes/scaffold.d.ts +15 -2
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +21 -1
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/sources.d.ts.map +1 -1
- package/out/recipes/sources.js +112 -1
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/street/affix.d.ts.map +1 -1
- package/out/recipes/street/affix.js +8 -20
- package/out/recipes/street/affix.js.map +1 -1
- package/out/recipes/sub/venue/context.d.ts +14 -3
- package/out/recipes/sub/venue/context.d.ts.map +1 -1
- package/out/recipes/sub/venue/context.js +29 -9
- package/out/recipes/sub/venue/context.js.map +1 -1
- package/out/recipes/sub/venue/sources.d.ts +41 -8
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +26 -7
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +35 -14
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/unit.d.ts +4 -0
- package/out/recipes/unit.d.ts.map +1 -1
- package/out/recipes/unit.js +7 -6
- package/out/recipes/unit.js.map +1 -1
- package/out/registers.d.ts +33 -0
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +33 -0
- package/out/registers.js.map +1 -1
- package/out/source-register/corpus-inputs.d.ts +240 -0
- package/out/source-register/corpus-inputs.d.ts.map +1 -0
- package/out/source-register/corpus-inputs.js +275 -0
- package/out/source-register/corpus-inputs.js.map +1 -0
- package/out/source-register/effective-manifest.d.ts +2 -0
- package/out/source-register/effective-manifest.d.ts.map +1 -1
- package/out/source-register/effective-manifest.js +33 -4
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/types.d.ts +21 -1
- package/out/source-register/types.d.ts.map +1 -1
- package/out/source-register/types.js +20 -0
- package/out/source-register/types.js.map +1 -1
- package/out/south-asia/recipes/register.js +1 -1
- package/out/south-asia/recipes/register.js.map +1 -1
- package/out/tools/overlay/manifest.d.ts +7 -0
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +72 -1
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/source-register/build.d.ts +9 -0
- package/out/tools/source-register/build.d.ts.map +1 -1
- package/out/tools/source-register/build.js +25 -3
- package/out/tools/source-register/build.js.map +1 -1
- package/out/tools/variety-sample.d.ts +196 -0
- package/out/tools/variety-sample.d.ts.map +1 -0
- package/out/tools/variety-sample.js +417 -0
- package/out/tools/variety-sample.js.map +1 -0
- package/out/us/adapters/usgov/nad/adapter.d.ts +80 -0
- package/out/us/adapters/usgov/nad/adapter.d.ts.map +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +27 -8
- package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
- package/out/us/recipes/skeleton-sources.d.ts +92 -0
- package/out/us/recipes/skeleton-sources.d.ts.map +1 -0
- package/out/us/recipes/skeleton-sources.js +121 -0
- package/out/us/recipes/skeleton-sources.js.map +1 -0
- package/out/us/tools/nad-skeletons.d.ts +120 -0
- package/out/us/tools/nad-skeletons.d.ts.map +1 -0
- package/out/us/tools/nad-skeletons.js +262 -0
- package/out/us/tools/nad-skeletons.js.map +1 -0
- package/out/uy/adapters/ide/adapter.d.ts +64 -0
- package/out/uy/adapters/ide/adapter.d.ts.map +1 -0
- package/out/uy/adapters/ide/adapter.js +206 -0
- package/out/uy/adapters/ide/adapter.js.map +1 -0
- package/out/ve/recipes/reviewed-postcode-tail.d.ts +6 -2
- package/out/ve/recipes/reviewed-postcode-tail.d.ts.map +1 -1
- package/out/ve/recipes/reviewed-postcode-tail.js +16 -5
- package/out/ve/recipes/reviewed-postcode-tail.js.map +1 -1
- package/package.json +14 -14
- package/lib/recipes/unit/sources.ts +0 -44
- package/lib/tools/fetch/ppd.ts +0 -121
- package/out/recipes/unit/sources.d.ts +0 -28
- package/out/recipes/unit/sources.d.ts.map +0 -1
- package/out/recipes/unit/sources.js +0 -30
- package/out/recipes/unit/sources.js.map +0 -1
- package/out/tools/fetch/ppd.d.ts +0 -15
- package/out/tools/fetch/ppd.d.ts.map +0 -1
- package/out/tools/fetch/ppd.js +0 -86
- package/out/tools/fetch/ppd.js.map +0 -1
|
@@ -20,8 +20,6 @@ import {
|
|
|
20
20
|
readFrTuples,
|
|
21
21
|
readPostalTuples,
|
|
22
22
|
readUsTuples,
|
|
23
|
-
US_EVAL_SOURCE,
|
|
24
|
-
US_TRAIN_SOURCES,
|
|
25
23
|
} from "#recipes/po/box/cedex/sources"
|
|
26
24
|
import type { Rendered, USTuple } from "#recipes/po/box/cedex/types"
|
|
27
25
|
import { renderPmbUs, renderPoBoxUs } from "#recipes/po/box/cedex/us"
|
|
@@ -30,6 +28,11 @@ import { recipeSourceID, type CanonicalRecipeRow, type CorpusRecipe } from "#rec
|
|
|
30
28
|
import { defaultRecipeSource } from "#recipes/sources"
|
|
31
29
|
import { synthesizeMilitaryPoBoxRow } from "#synthesizers/po-box"
|
|
32
30
|
import { SurfaceOrigin } from "#types"
|
|
31
|
+
import {
|
|
32
|
+
NAD_SKELETON_DIRECTORY,
|
|
33
|
+
US_SKELETON_EVAL_SOURCE,
|
|
34
|
+
US_SKELETON_TRAIN_SOURCES,
|
|
35
|
+
} from "#us/recipes/skeleton-sources"
|
|
33
36
|
import { alignRow } from "#utils"
|
|
34
37
|
|
|
35
38
|
const COMPONENT_ORDER = ["house_number", "street", "po_box", "venue", "locality", "postcode", "region", "cedex"]
|
|
@@ -63,7 +66,7 @@ export const poBoxCedexRecipe: CorpusRecipe = {
|
|
|
63
66
|
const source = opts.sourceName ?? defaultRecipeSource("synth-po-box-cedex")
|
|
64
67
|
const usPool: USTuple[] = []
|
|
65
68
|
|
|
66
|
-
for (const sourceEntry of opts.golden ? [
|
|
69
|
+
for (const sourceEntry of opts.golden ? [US_SKELETON_EVAL_SOURCE] : US_SKELETON_TRAIN_SOURCES) {
|
|
67
70
|
const tuples = await readUsTuples(sourceEntry)
|
|
68
71
|
|
|
69
72
|
console.error(` ${sourceEntry.csv}: ${tuples.length} tuples`)
|
|
@@ -96,7 +99,7 @@ export const poBoxCedexRecipe: CorpusRecipe = {
|
|
|
96
99
|
|
|
97
100
|
if (!usPool.length || !frPool.length || !qcPool.length || !onPool.length || !auPool.length || !nzPool.length)
|
|
98
101
|
throw new Error(
|
|
99
|
-
`A base pool is empty — check ${dataRootPath("oa-cache")}, ${GEONAMES_CA}, ${GEONAMES_POSTAL_AU.zip}, and ${GEONAMES_POSTAL_NZ.zip}.`
|
|
102
|
+
`A base pool is empty — check ${dataRootPath("oa-cache")}, ${dataRootPath(NAD_SKELETON_DIRECTORY)}, ${GEONAMES_CA}, ${GEONAMES_POSTAL_AU.zip}, and ${GEONAMES_POSTAL_NZ.zip}.`
|
|
100
103
|
)
|
|
101
104
|
|
|
102
105
|
let emitted = 0,
|
|
@@ -185,7 +188,7 @@ export const poBoxCedexRecipe: CorpusRecipe = {
|
|
|
185
188
|
? "GeoNames AU postal dump (CC-BY 4.0) locality/state/postcode tails + Australia Post Postal Delivery Type designators (@mailwoman/codex/au)"
|
|
186
189
|
: country === "NZ"
|
|
187
190
|
? "GeoNames NZ postal dump (CC-BY 4.0) locality/postcode tails + NZ Post ADV358 Delivery Service Types (@mailwoman/codex/nz)"
|
|
188
|
-
: "OpenAddresses US (non-VT) skeletons + USPS Pub-28 §29 PO-box designators (codex/corpus templates)",
|
|
191
|
+
: "OpenAddresses and National Address Database US (non-VT) skeletons + USPS Pub-28 §29 PO-box designators (codex/corpus templates)",
|
|
189
192
|
}
|
|
190
193
|
|
|
191
194
|
const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
|
|
@@ -12,28 +12,7 @@ import { TextSpliterator, TSVSpliterator } from "spliterator"
|
|
|
12
12
|
|
|
13
13
|
import type { AUTuple, FRTuple, NZTuple, USTuple } from "#recipes/po/box/cedex/types"
|
|
14
14
|
import { readCSVRecords, readZippedCSVRecords } from "#recipes/scaffold"
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* Non-Vermont OpenAddresses files used for training rows.
|
|
18
|
-
*/
|
|
19
|
-
export const US_TRAIN_SOURCES = [
|
|
20
|
-
{ zip: dataRootPath("oa-cache", "us__ca__berkeley.zip"), csv: "us/ca/berkeley.csv", region: "CA" },
|
|
21
|
-
{ zip: dataRootPath("oa-cache", "us__ca__marin.zip"), csv: "us/ca/marin.csv", region: "CA" },
|
|
22
|
-
{ zip: dataRootPath("oa-cache", "us__dc__statewide.zip"), csv: "us/dc/statewide.csv", region: "DC" },
|
|
23
|
-
{ zip: dataRootPath("oa-cache", "us__ia__statewide.zip"), csv: "us/ia/statewide.csv", region: "IA" },
|
|
24
|
-
{ zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", region: "IL" },
|
|
25
|
-
{ zip: dataRootPath("oa-cache", "us__mt__statewide.zip"), csv: "us/mt/statewide.csv", region: "MT" },
|
|
26
|
-
{ zip: dataRootPath("oa-cache", "us__sd__statewide.zip"), csv: "us/sd/statewide.csv", region: "SD" },
|
|
27
|
-
]
|
|
28
|
-
|
|
29
|
-
/**
|
|
30
|
-
* Vermont OpenAddresses file reserved for golden rows.
|
|
31
|
-
*/
|
|
32
|
-
export const US_EVAL_SOURCE = {
|
|
33
|
-
zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
|
|
34
|
-
csv: "us/vt/statewide.csv",
|
|
35
|
-
region: "VT",
|
|
36
|
-
}
|
|
15
|
+
import type { USSkeletonSource } from "#us/recipes/skeleton-sources"
|
|
37
16
|
|
|
38
17
|
/**
|
|
39
18
|
* Countrywide French OpenAddresses source.
|
|
@@ -72,7 +51,7 @@ export const isHoldoutLocality = (name: string): boolean => localityHash(name) %
|
|
|
72
51
|
const cleanLocality = (locality: string) =>
|
|
73
52
|
locality && locality.length <= MAX_LOCALITY_LENGTH && !/\d|,/.test(locality) && !/cedex/i.test(locality)
|
|
74
53
|
|
|
75
|
-
export async function readUsTuples(source:
|
|
54
|
+
export async function readUsTuples(source: USSkeletonSource): Promise<USTuple[]> {
|
|
76
55
|
const tuples: USTuple[] = []
|
|
77
56
|
const seen = new Set<string>()
|
|
78
57
|
|
package/lib/recipes/scaffold.ts
CHANGED
|
@@ -11,12 +11,13 @@
|
|
|
11
11
|
|
|
12
12
|
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
13
13
|
import { readZipEntry } from "@mailwoman/core/fs/zip"
|
|
14
|
-
import { tryParsingJSON, stringifyJSON } from "@mailwoman/core/json"
|
|
14
|
+
import { parseJSONStrict, tryParsingJSON, stringifyJSON } from "@mailwoman/core/json"
|
|
15
15
|
import type { PathBuilderLike } from "path-ts"
|
|
16
16
|
import type { AsyncChunkIterator, AsyncDataResource } from "spliterator"
|
|
17
17
|
import { AsyncSequence, CSVSpliterator, TextSpliterator } from "spliterator"
|
|
18
18
|
|
|
19
19
|
import { stableSourceIDFromParts } from "#adapters/source-id"
|
|
20
|
+
import type { SourceRightsAdmission } from "#source-register/corpus-inputs"
|
|
20
21
|
import type { SurfaceOrigin } from "#types"
|
|
21
22
|
import { alignRow } from "#utils"
|
|
22
23
|
|
|
@@ -284,6 +285,29 @@ export function createRecipeLineWriter(sink: RecipeLineSink): WriteRecipeLine {
|
|
|
284
285
|
}
|
|
285
286
|
}
|
|
286
287
|
|
|
288
|
+
/**
|
|
289
|
+
* Wraps a recipe's writer so every line's `source` passes the source-rights check
|
|
290
|
+
* before the line is written.
|
|
291
|
+
*
|
|
292
|
+
* A recipe's `source` is its table default or whatever `--source-name` supplied,
|
|
293
|
+
* so the line itself is the only place the value is certain.
|
|
294
|
+
* The check caches each answer, so the cost after the first line of a source is one parse.
|
|
295
|
+
*
|
|
296
|
+
* @throws From the returned writer, when a line carries no string `source` or its source is refused.
|
|
297
|
+
*/
|
|
298
|
+
export function admittingRecipeLineWriter(write: WriteRecipeLine, rights: SourceRightsAdmission): WriteRecipeLine {
|
|
299
|
+
return (line) => {
|
|
300
|
+
const { source } = parseJSONStrict(line) as { source?: unknown }
|
|
301
|
+
|
|
302
|
+
if (typeof source !== "string") {
|
|
303
|
+
throw new TypeError(`a recipe wrote a row with no string \`source\`, so its rights cannot be checked: ${line}`)
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
rights.admit(source)
|
|
307
|
+
write(line)
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
287
311
|
/**
|
|
288
312
|
* What a recipe records about the rows it writes.
|
|
289
313
|
*
|
|
@@ -478,8 +502,9 @@ export interface RecipeOptions {
|
|
|
478
502
|
/**
|
|
479
503
|
* `sub-venue`: GB/US/FR address-context tuples jsonl.
|
|
480
504
|
*
|
|
481
|
-
* @defaultValue the house-venue
|
|
482
|
-
*
|
|
505
|
+
* @defaultValue the house-venue v6 tuples (`$MAILWOMAN_DATA_ROOT/corpus/intermediate/house-venue-tuples-v6.jsonl`),
|
|
506
|
+
* whose GB rows come from OS OpenData and whose US rows come from NAD and from the bodies whose own decisions
|
|
507
|
+
* permit training. DE and ES read OpenAddresses directly.
|
|
483
508
|
*/
|
|
484
509
|
subVenueTuples?: string
|
|
485
510
|
/**
|
package/lib/recipes/sources.ts
CHANGED
|
@@ -135,7 +135,18 @@ export const RECIPE_SOURCES: ReadonlyArray<RecipeSource> = [
|
|
|
135
135
|
["es", LOCALE_RECIPE],
|
|
136
136
|
["es-pedania", LOCALE_RECIPE, "The ES source run with `--district-as-locality`, under `--source-name`."],
|
|
137
137
|
["nz", LOCALE_RECIPE],
|
|
138
|
-
[
|
|
138
|
+
[
|
|
139
|
+
"gb",
|
|
140
|
+
LOCALE_RECIPE,
|
|
141
|
+
"Rendered from HM Land Registry Price Paid Data address fields, which Royal Mail and Ordnance Survey " +
|
|
142
|
+
"license for non-commercial use only. No recipe writes it now; `rendered-gb-os-open` replaces it (#2635).",
|
|
143
|
+
],
|
|
144
|
+
[
|
|
145
|
+
"gb-os-open",
|
|
146
|
+
LOCALE_RECIPE,
|
|
147
|
+
"The GB part reads the `gb-os-open` adapter's canonical rows (OS OpenData, OGL v3.0). " +
|
|
148
|
+
"No corpus stores the retired spelling.",
|
|
149
|
+
],
|
|
139
150
|
["fr-order", "fr/recipes/order.ts"],
|
|
140
151
|
["fr-admin-split", "fr/recipes/admin-split.ts", "The département is derived from the record's own postcode."],
|
|
141
152
|
["fr-lieudit", "fr/recipes/lieudit.ts"],
|
|
@@ -283,6 +294,106 @@ export const CARRIED_SOURCES: ReadonlyArray<string> = [
|
|
|
283
294
|
"osm-vn",
|
|
284
295
|
"osm-pk",
|
|
285
296
|
"osm-bd",
|
|
297
|
+
// The 2026-10-16 and 2026-10-17 extracts for the countries the corpus held under 1,000 street rows of,
|
|
298
|
+
// where no national address register with an electable grant was downloadable (country-gap-v1).
|
|
299
|
+
"osm-eg",
|
|
300
|
+
"osm-rs",
|
|
301
|
+
"osm-ar",
|
|
302
|
+
"osm-ua",
|
|
303
|
+
"osm-kz",
|
|
304
|
+
"osm-by",
|
|
305
|
+
"osm-md",
|
|
306
|
+
"osm-cl",
|
|
307
|
+
"osm-uz",
|
|
308
|
+
"osm-pe",
|
|
309
|
+
"osm-my",
|
|
310
|
+
"osm-bn",
|
|
311
|
+
"osm-ba",
|
|
312
|
+
"osm-ae",
|
|
313
|
+
"osm-kw",
|
|
314
|
+
"osm-ge",
|
|
315
|
+
"osm-am",
|
|
316
|
+
"osm-pr",
|
|
317
|
+
"osm-kg",
|
|
318
|
+
"osm-ph",
|
|
319
|
+
"osm-ci",
|
|
320
|
+
// The 2026-10-16 to 2026-10-18 extracts for the second set of countries under 1,000 street rows,
|
|
321
|
+
// where no national address register with an electable grant was downloadable (country-gap-v2).
|
|
322
|
+
// The Bahamas, Azerbaijan and French Polynesia also carry a national source that covers part
|
|
323
|
+
// of the country (`gbpa-bs` Freeport, `opendata-az` three cities, `ban-pf` three communes).
|
|
324
|
+
"osm-al",
|
|
325
|
+
"osm-ao",
|
|
326
|
+
"osm-aw",
|
|
327
|
+
"osm-az",
|
|
328
|
+
"osm-bb",
|
|
329
|
+
"osm-bh",
|
|
330
|
+
"osm-bm",
|
|
331
|
+
"osm-bs",
|
|
332
|
+
"osm-bt",
|
|
333
|
+
"osm-bw",
|
|
334
|
+
"osm-bz",
|
|
335
|
+
"osm-cd",
|
|
336
|
+
"osm-co",
|
|
337
|
+
"osm-cr",
|
|
338
|
+
"osm-cu",
|
|
339
|
+
"osm-cw",
|
|
340
|
+
"osm-do",
|
|
341
|
+
"osm-dz",
|
|
342
|
+
"osm-ec",
|
|
343
|
+
"osm-gg",
|
|
344
|
+
"osm-gh",
|
|
345
|
+
"osm-gl",
|
|
346
|
+
"osm-gt",
|
|
347
|
+
"osm-hk",
|
|
348
|
+
"osm-ht",
|
|
349
|
+
"osm-iq",
|
|
350
|
+
"osm-ir",
|
|
351
|
+
"osm-je",
|
|
352
|
+
"osm-jm",
|
|
353
|
+
"osm-jo",
|
|
354
|
+
"osm-ke",
|
|
355
|
+
"osm-kh",
|
|
356
|
+
"osm-lb",
|
|
357
|
+
"osm-lk",
|
|
358
|
+
"osm-lr",
|
|
359
|
+
"osm-ma",
|
|
360
|
+
"osm-me",
|
|
361
|
+
"osm-mg",
|
|
362
|
+
"osm-mk",
|
|
363
|
+
"osm-mm",
|
|
364
|
+
"osm-mn",
|
|
365
|
+
"osm-mo",
|
|
366
|
+
"osm-mt",
|
|
367
|
+
"osm-mv",
|
|
368
|
+
"osm-mz",
|
|
369
|
+
"osm-na",
|
|
370
|
+
"osm-np",
|
|
371
|
+
"osm-om",
|
|
372
|
+
"osm-pa",
|
|
373
|
+
"osm-pf",
|
|
374
|
+
"osm-py",
|
|
375
|
+
"osm-qa",
|
|
376
|
+
"osm-rw",
|
|
377
|
+
"osm-sa",
|
|
378
|
+
"osm-sm",
|
|
379
|
+
"osm-sn",
|
|
380
|
+
"osm-sr",
|
|
381
|
+
"osm-sv",
|
|
382
|
+
"osm-th",
|
|
383
|
+
"osm-tj",
|
|
384
|
+
"osm-tm",
|
|
385
|
+
"osm-tn",
|
|
386
|
+
"osm-tt",
|
|
387
|
+
"osm-tz",
|
|
388
|
+
"osm-ug",
|
|
389
|
+
"osm-ve",
|
|
390
|
+
"osm-vi",
|
|
391
|
+
"osm-zm",
|
|
392
|
+
"osm-zw",
|
|
393
|
+
// One id per dependency a national register covers beside its state, written by
|
|
394
|
+
// `corpus run <adapter> --country <cc> --source-name`, so the dependency's rows hold their own weight.
|
|
395
|
+
"ryhti-ax",
|
|
396
|
+
"matrikkelen-sj",
|
|
286
397
|
// One id per French overseas jurisdiction for the BAN department files,
|
|
287
398
|
// written by `corpus run ban --source-name`.
|
|
288
399
|
// Metropolitan rows keep `ban`, so its weight does not decide the rate at which a territory is drawn.
|
|
@@ -26,30 +26,14 @@ import { stableSourceID } from "#adapters/source-id"
|
|
|
26
26
|
import { readCSVRecords, readOATuples, recipeSourceID, type CorpusRecipe } from "#recipes/scaffold"
|
|
27
27
|
import { defaultRecipeSource } from "#recipes/sources"
|
|
28
28
|
import type { CanonicalRow } from "#types"
|
|
29
|
+
import {
|
|
30
|
+
NAD_SKELETON_DIRECTORY,
|
|
31
|
+
US_SKELETON_EVAL_SOURCE,
|
|
32
|
+
US_SKELETON_TRAIN_SOURCES,
|
|
33
|
+
type USSkeletonSource,
|
|
34
|
+
} from "#us/recipes/skeleton-sources"
|
|
29
35
|
import { alignRow } from "#utils"
|
|
30
36
|
|
|
31
|
-
interface USSource {
|
|
32
|
-
zip: PathBuilderLike
|
|
33
|
-
csv: string
|
|
34
|
-
region: string
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
const TRAIN_SOURCES: readonly USSource[] = [
|
|
38
|
-
{ zip: dataRootPath("oa-cache", "us__ca__berkeley.zip"), csv: "us/ca/berkeley.csv", region: "CA" },
|
|
39
|
-
{ zip: dataRootPath("oa-cache", "us__ca__marin.zip"), csv: "us/ca/marin.csv", region: "CA" },
|
|
40
|
-
{ zip: dataRootPath("oa-cache", "us__dc__statewide.zip"), csv: "us/dc/statewide.csv", region: "DC" },
|
|
41
|
-
{ zip: dataRootPath("oa-cache", "us__ia__statewide.zip"), csv: "us/ia/statewide.csv", region: "IA" },
|
|
42
|
-
{ zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", region: "IL" },
|
|
43
|
-
{ zip: dataRootPath("oa-cache", "us__mt__statewide.zip"), csv: "us/mt/statewide.csv", region: "MT" },
|
|
44
|
-
{ zip: dataRootPath("oa-cache", "us__sd__statewide.zip"), csv: "us/sd/statewide.csv", region: "SD" },
|
|
45
|
-
]
|
|
46
|
-
|
|
47
|
-
const EVAL_SOURCE: USSource = {
|
|
48
|
-
zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
|
|
49
|
-
csv: "us/vt/statewide.csv",
|
|
50
|
-
region: "VT",
|
|
51
|
-
}
|
|
52
|
-
|
|
53
37
|
interface BalanceSource {
|
|
54
38
|
zip: PathBuilderLike
|
|
55
39
|
csv: string
|
|
@@ -118,12 +102,12 @@ interface BalanceTuple {
|
|
|
118
102
|
|
|
119
103
|
type Prefix = Pick<NonNullable<ReturnType<typeof matchLeadingDirectional>>, "canonical" | "abbreviation">
|
|
120
104
|
|
|
121
|
-
async function readTuples(source:
|
|
105
|
+
async function readTuples(source: USSkeletonSource): Promise<USTuple[]> {
|
|
122
106
|
return readOATuples(source, {
|
|
123
107
|
extra: (fields, _row, key) => ({
|
|
124
108
|
...fields,
|
|
125
109
|
region: source.region,
|
|
126
|
-
base_source_id:
|
|
110
|
+
base_source_id: `${source.register}:${source.csv}:${key}`,
|
|
127
111
|
}),
|
|
128
112
|
})
|
|
129
113
|
}
|
|
@@ -372,7 +356,7 @@ export const streetAffixRecipe: CorpusRecipe = {
|
|
|
372
356
|
const count = opts.count ?? 50_000
|
|
373
357
|
const source = opts.sourceName ?? defaultRecipeSource("synth-affix")
|
|
374
358
|
const multilocaleCount = opts.multilocaleCount ?? 0
|
|
375
|
-
const sources = opts.golden ? [
|
|
359
|
+
const sources = opts.golden ? [US_SKELETON_EVAL_SOURCE] : US_SKELETON_TRAIN_SOURCES
|
|
376
360
|
|
|
377
361
|
const pool: USTuple[] = []
|
|
378
362
|
|
|
@@ -387,7 +371,10 @@ export const streetAffixRecipe: CorpusRecipe = {
|
|
|
387
371
|
}
|
|
388
372
|
|
|
389
373
|
if (!pool.length) {
|
|
390
|
-
throw new Error(
|
|
374
|
+
throw new Error(
|
|
375
|
+
`No US tuples found — are the cached OA zips present in ${dataRootPath("oa-cache")} and the NAD ` +
|
|
376
|
+
`skeletons in ${dataRootPath(NAD_SKELETON_DIRECTORY)}?`
|
|
377
|
+
)
|
|
391
378
|
}
|
|
392
379
|
|
|
393
380
|
let emitted = 0
|
|
@@ -448,7 +435,8 @@ export const streetAffixRecipe: CorpusRecipe = {
|
|
|
448
435
|
source,
|
|
449
436
|
source_id: stableSourceID(source, components),
|
|
450
437
|
corpus_version: "0.4.0",
|
|
451
|
-
license:
|
|
438
|
+
license:
|
|
439
|
+
"OpenAddresses and National Address Database US (non-VT) skeletons, street split via USPS Pub-28 C1/C2 (codex)",
|
|
452
440
|
}
|
|
453
441
|
|
|
454
442
|
const aligned = alignRow(canonical)
|
|
@@ -564,7 +552,7 @@ export const suffixBoundaryRecipe: CorpusRecipe = {
|
|
|
564
552
|
const random = makeMulberry32(opts.seed)
|
|
565
553
|
const count = opts.count ?? 30_000
|
|
566
554
|
const source = opts.sourceName ?? defaultRecipeSource("synth-suffix-boundary")
|
|
567
|
-
const sources = opts.golden ? [
|
|
555
|
+
const sources = opts.golden ? [US_SKELETON_EVAL_SOURCE] : US_SKELETON_TRAIN_SOURCES
|
|
568
556
|
|
|
569
557
|
const venuePool = await readVenuePool(VENUE_POOL_CSV)
|
|
570
558
|
|
|
@@ -630,7 +618,7 @@ export const suffixBoundaryRecipe: CorpusRecipe = {
|
|
|
630
618
|
source_id: recipeSourceID(source, { ...components, class: rowClass, n: String(emitted) }),
|
|
631
619
|
corpus_version: "0.19.0",
|
|
632
620
|
license:
|
|
633
|
-
"OpenAddresses US (non-VT) skeletons; terminal suffix labels via USPS Pub-28 codex; venue shell from HRSA site names (US public domain)",
|
|
621
|
+
"OpenAddresses and National Address Database US (non-VT) skeletons; terminal suffix labels via USPS Pub-28 codex; venue shell from HRSA site names (US public domain)",
|
|
634
622
|
}
|
|
635
623
|
|
|
636
624
|
const aligned = alignRow(canonical)
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
9
|
+
import { stringifyJSON } from "@mailwoman/core/json"
|
|
9
10
|
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
10
11
|
import type { PathBuilderLike } from "path-ts"
|
|
11
12
|
|
|
@@ -22,24 +23,37 @@ import type { LocaleBaseTuple } from "#surfaces/locale"
|
|
|
22
23
|
* so the CSV handling (quoted fields, crlf, city-noise cleaning, the DE per-part region fallback)
|
|
23
24
|
* has exactly one implementation.
|
|
24
25
|
*
|
|
25
|
-
* The DE
|
|
26
|
-
*
|
|
26
|
+
* The DE parts read the extracted `de/berlin.csv` and `de/sn/statewide.csv`.
|
|
27
|
+
* They are the `europe.zip` members the recipe read before that archive left the data root,
|
|
28
|
+
* with the md5s `9903d438…` and `623b099b…` that the note "a file's mtime is not
|
|
29
|
+
* its data's vintage" in `corpus/agents.md` records.
|
|
27
30
|
*
|
|
28
|
-
*
|
|
29
|
-
* The note "a file's mtime is not its data's vintage" in `corpus/agents.md` records this comparison.
|
|
31
|
+
* A missing part throws in {@link readLocaleTuples}, so the DE leg cannot emit zero rows silently.
|
|
30
32
|
*/
|
|
31
33
|
const CONTEXT_PARTS: Readonly<Record<string, readonly LocalePart[]>> = {
|
|
32
34
|
DE: [
|
|
33
|
-
{
|
|
34
|
-
{
|
|
35
|
+
{ path: dataRootPath("openaddresses", "extracted", "de", "berlin.csv"), region: "Berlin" },
|
|
36
|
+
{ path: dataRootPath("openaddresses", "extracted", "de", "sn", "statewide.csv"), region: "Sachsen" },
|
|
35
37
|
],
|
|
36
38
|
ES: [{ path: dataRootPath("openaddresses", "extracted", "es", "countrywide.csv") }],
|
|
37
39
|
}
|
|
38
40
|
|
|
39
41
|
/**
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
42
|
+
* The `source` a Great Britain house-venue tuple must carry.
|
|
43
|
+
*
|
|
44
|
+
* The v3 tuples took their GB rows from HM Land Registry Price Paid Data, whose address
|
|
45
|
+
* fields Royal Mail and Ordnance Survey license for non-commercial use only (#2635).
|
|
46
|
+
* Their GB rows carry no `source`.
|
|
47
|
+
*/
|
|
48
|
+
export const GB_CONTEXT_TUPLE_SOURCE = "gb-os-open"
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Load the address skeletons every leg renders onto: GB / US / FR from the house-venue tuples
|
|
52
|
+
* (`corpora/house_venue_tuples.py`, whose GB rows come from the `gb-os-open` adapter's OS OpenData rows),
|
|
53
|
+
* DE and ES streamed out of OpenAddresses.
|
|
54
|
+
*
|
|
55
|
+
* @throws When a GB tuple does not carry {@linkcode GB_CONTEXT_TUPLE_SOURCE},
|
|
56
|
+
* so a tuples file built from Price Paid Data is refused rather than rendered.
|
|
43
57
|
*/
|
|
44
58
|
export async function loadContextTuples(
|
|
45
59
|
tuplesPath: PathBuilderLike,
|
|
@@ -50,6 +64,16 @@ export async function loadContextTuples(
|
|
|
50
64
|
for await (const tuple of readRecipeTuples(tuplesPath)) {
|
|
51
65
|
const country = String(tuple.country ?? "")
|
|
52
66
|
|
|
67
|
+
// country-branch: GB is the one country whose earlier tuples came from a refused source (#2635).
|
|
68
|
+
// The tuples file records provenance for GB rows only, so the check reads GB rows only.
|
|
69
|
+
if (country === "GB" && tuple.source !== GB_CONTEXT_TUPLE_SOURCE) {
|
|
70
|
+
throw new Error(
|
|
71
|
+
`${tuplesPath.toString()} holds a GB tuple from source ${stringifyJSON(tuple.source ?? null)}; ` +
|
|
72
|
+
`GB address skeletons must come from ${GB_CONTEXT_TUPLE_SOURCE}. Rebuild the tuples with ` +
|
|
73
|
+
"`python -m mailwoman_train.corpora.house_venue_tuples`."
|
|
74
|
+
)
|
|
75
|
+
}
|
|
76
|
+
|
|
53
77
|
if (!country || !tuple.locality || !tuple.street) continue
|
|
54
78
|
|
|
55
79
|
const mapped: LocaleBaseTuple = {
|
|
@@ -12,6 +12,7 @@ import { upperFirst } from "@mailwoman/normalize/case"
|
|
|
12
12
|
import type { PathBuilderLike } from "path-ts"
|
|
13
13
|
|
|
14
14
|
import { escapeSQLString, openDuckDB } from "#parquet/duckdb"
|
|
15
|
+
import { SourceRegister } from "#registers"
|
|
15
16
|
import { classifyIdentifier, readSubVenueJSONL } from "#subvenue/harvest"
|
|
16
17
|
import { SUBVENUE_PROMOTIONS, type SubVenuePromotion } from "#subvenue/promotions"
|
|
17
18
|
import type { SubVenueLexiconTable } from "#subvenue/table"
|
|
@@ -378,11 +379,34 @@ export function sampleIdentifier(model: IdentifierModel, designatorID: string, r
|
|
|
378
379
|
}
|
|
379
380
|
|
|
380
381
|
/**
|
|
381
|
-
* The
|
|
382
|
+
* The publication a pooled name was read from.
|
|
383
|
+
*
|
|
384
|
+
* Each row records the sources of the names it carries, because the two publications are
|
|
385
|
+
* licensed differently: OpenStreetMap under ODbL 1.0, Overture Places under CDLA-Permissive-2.0.
|
|
382
386
|
*/
|
|
383
|
-
export
|
|
384
|
-
|
|
387
|
+
export const NameSource = {
|
|
388
|
+
OpenStreetMap: SourceRegister.OpenStreetMap,
|
|
389
|
+
OverturePlaces: "overture-places",
|
|
390
|
+
} as const
|
|
385
391
|
|
|
392
|
+
export type NameSource = (typeof NameSource)[keyof typeof NameSource]
|
|
393
|
+
|
|
394
|
+
/**
|
|
395
|
+
* A pooled name with the publication it was read from.
|
|
396
|
+
*
|
|
397
|
+
* A name both sources hold appears once per source, so its draw still records the pool it came from.
|
|
398
|
+
*/
|
|
399
|
+
export interface SourcedName {
|
|
400
|
+
name: string
|
|
401
|
+
source: NameSource
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/**
|
|
405
|
+
* The name pools that one source contributes to a recipe leg.
|
|
406
|
+
*
|
|
407
|
+
* Only OSM extracts fill `attested` and `unpromotedShapes`, because POI data has no tier or localized names.
|
|
408
|
+
*/
|
|
409
|
+
export interface NamePools {
|
|
386
410
|
/**
|
|
387
411
|
* Real venue-tier names that fit the venue slot, such as stations, airports, campuses and hospitals.
|
|
388
412
|
*/
|
|
@@ -410,11 +434,19 @@ export interface LegPools {
|
|
|
410
434
|
}
|
|
411
435
|
|
|
412
436
|
/**
|
|
413
|
-
* The name pools
|
|
437
|
+
* The address context and name pools for one recipe leg.
|
|
414
438
|
*
|
|
415
|
-
*
|
|
439
|
+
* `attested` and `unpromotedShapes` hold OpenStreetMap names only, as {@linkcode NamePools} states.
|
|
440
|
+
* The other three pools merge both sources and keep each name's source.
|
|
416
441
|
*/
|
|
417
|
-
export
|
|
442
|
+
export interface LegPools {
|
|
443
|
+
context: LocaleBaseTuple[]
|
|
444
|
+
venues: SourcedName[]
|
|
445
|
+
attested: string[]
|
|
446
|
+
rejectedVenues: SourcedName[]
|
|
447
|
+
longerNames: SourcedName[]
|
|
448
|
+
unpromotedShapes: string[]
|
|
449
|
+
}
|
|
418
450
|
|
|
419
451
|
/**
|
|
420
452
|
* Empty name pools for a source that a leg lacks.
|
|
@@ -571,14 +603,23 @@ export async function readOvertureNamePools(
|
|
|
571
603
|
}
|
|
572
604
|
|
|
573
605
|
/**
|
|
574
|
-
* Concatenates
|
|
606
|
+
* Concatenates an OpenStreetMap extract's pools and the Overture Places pools,
|
|
607
|
+
* OpenStreetMap first, and tags each merged name with its source.
|
|
608
|
+
*
|
|
609
|
+
* The order and length of every pool match a plain concatenation, so a seeded run
|
|
610
|
+
* draws the same names as one that recorded no source.
|
|
575
611
|
*/
|
|
576
|
-
export function mergeNamePools(
|
|
612
|
+
export function mergeNamePools(osm: NamePools, overture: NamePools): Omit<LegPools, "context"> {
|
|
613
|
+
const tag = (a: readonly string[], b: readonly string[]): SourcedName[] => [
|
|
614
|
+
...a.map((name) => ({ name, source: NameSource.OpenStreetMap })),
|
|
615
|
+
...b.map((name) => ({ name, source: NameSource.OverturePlaces })),
|
|
616
|
+
]
|
|
617
|
+
|
|
577
618
|
return {
|
|
578
|
-
venues:
|
|
579
|
-
attested: [...
|
|
580
|
-
rejectedVenues:
|
|
581
|
-
longerNames:
|
|
582
|
-
unpromotedShapes: [...
|
|
619
|
+
venues: tag(osm.venues, overture.venues),
|
|
620
|
+
attested: [...osm.attested, ...overture.attested],
|
|
621
|
+
rejectedVenues: tag(osm.rejectedVenues, overture.rejectedVenues),
|
|
622
|
+
longerNames: tag(osm.longerNames, overture.longerNames),
|
|
623
|
+
unpromotedShapes: [...osm.unpromotedShapes, ...overture.unpromotedShapes],
|
|
583
624
|
}
|
|
584
625
|
}
|