@mailwoman/corpus 10.5.0 → 10.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. package/data/PROVENANCE.md +24 -0
  2. package/data/address-source-register.json +18948 -8757
  3. package/data/corpus-source-inputs.json +4756 -0
  4. package/data/license-decisions.json +7260 -0
  5. package/data/licenses/whosonfirst-licenses.md +3 -2
  6. package/data/reviewed-ve-postcode-tuples.json +5 -72
  7. package/data/source-additions.json +2450 -0
  8. package/data/source-resolutions.json +87 -0
  9. package/data/training-manifests/v0.7.10-gb-county.effective.json +2335 -0
  10. package/data/training-manifests/v0.7.10-gb-county.json +1132 -0
  11. package/data/training-manifests/v0.7.2-address-systems.effective.json +3 -2
  12. package/data/training-manifests/v0.7.7-no-ppd.json +550 -0
  13. package/data/training-manifests/v0.7.8-rights-clean.json +538 -0
  14. package/data/training-manifests/v0.7.9-country-gap.json +1132 -0
  15. package/data/us-situs-sources.json +128 -0
  16. package/lib/adapters/cordis/adapter.ts +1 -1
  17. package/lib/adapters/libpostal-dictionary.ts +23 -3
  18. package/lib/adapters/osm/adapter.ts +42 -11
  19. package/lib/adapters.ts +14 -0
  20. package/lib/az/adapters/opendata/adapter.ts +389 -0
  21. package/lib/bs/adapters/gbpa/adapter.ts +248 -0
  22. package/lib/build/types.ts +16 -0
  23. package/lib/build.ts +21 -0
  24. package/lib/ee/adapters/ads/adapter.ts +18 -0
  25. package/lib/fo/adapters/adressur/adapter.ts +242 -0
  26. package/lib/gb/adapters/os-open/adapter.ts +14 -2
  27. package/lib/gb/county.ts +59 -0
  28. package/lib/gb/os-open/{open-names.ts → open/names.ts} +41 -0
  29. package/lib/gu/adapters/bsp/adapter.ts +434 -0
  30. package/lib/im/adapters/land-registry/adapter.ts +340 -0
  31. package/lib/international/recipes/locale.ts +164 -19
  32. package/lib/li/adapters/llv/adapter.ts +246 -0
  33. package/lib/recipes/bare/postcode.ts +10 -34
  34. package/lib/recipes/country-balanced.ts +95 -78
  35. package/lib/recipes/intersection.ts +22 -30
  36. package/lib/recipes/po/box/cedex/recipe.ts +8 -5
  37. package/lib/recipes/po/box/cedex/sources.ts +2 -23
  38. package/lib/recipes/scaffold.ts +28 -3
  39. package/lib/recipes/sources.ts +112 -1
  40. package/lib/recipes/street/affix.ts +17 -29
  41. package/lib/recipes/sub/venue/context.ts +33 -9
  42. package/lib/recipes/sub/venue/sources.ts +54 -13
  43. package/lib/recipes/sub/venue.ts +68 -23
  44. package/lib/recipes/unit.ts +20 -7
  45. package/lib/registers.ts +33 -0
  46. package/lib/source-register/corpus-inputs.ts +528 -0
  47. package/lib/source-register/effective-manifest.ts +37 -5
  48. package/lib/source-register/types.ts +20 -0
  49. package/lib/south-asia/recipes/register.ts +1 -1
  50. package/lib/tools/overlay/manifest.ts +109 -1
  51. package/lib/tools/source-register/build.ts +47 -1
  52. package/lib/tools/variety-sample.ts +633 -0
  53. package/lib/us/adapters/usgov/nad/adapter.ts +39 -10
  54. package/lib/us/recipes/skeleton-sources.ts +156 -0
  55. package/lib/us/tools/nad-skeletons.ts +395 -0
  56. package/lib/uy/adapters/ide/adapter.ts +275 -0
  57. package/lib/ve/recipes/reviewed-postcode-tail.ts +29 -6
  58. package/out/adapters/cordis/adapter.d.ts +1 -1
  59. package/out/adapters/cordis/adapter.js +1 -1
  60. package/out/adapters/libpostal-dictionary.d.ts +9 -0
  61. package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
  62. package/out/adapters/libpostal-dictionary.js +19 -3
  63. package/out/adapters/libpostal-dictionary.js.map +1 -1
  64. package/out/adapters/osm/adapter.d.ts +12 -1
  65. package/out/adapters/osm/adapter.d.ts.map +1 -1
  66. package/out/adapters/osm/adapter.js +30 -13
  67. package/out/adapters/osm/adapter.js.map +1 -1
  68. package/out/adapters.d.ts.map +1 -1
  69. package/out/adapters.js +14 -0
  70. package/out/adapters.js.map +1 -1
  71. package/out/az/adapters/opendata/adapter.d.ts +88 -0
  72. package/out/az/adapters/opendata/adapter.d.ts.map +1 -0
  73. package/out/az/adapters/opendata/adapter.js +282 -0
  74. package/out/az/adapters/opendata/adapter.js.map +1 -0
  75. package/out/bs/adapters/gbpa/adapter.d.ts +64 -0
  76. package/out/bs/adapters/gbpa/adapter.d.ts.map +1 -0
  77. package/out/bs/adapters/gbpa/adapter.js +187 -0
  78. package/out/bs/adapters/gbpa/adapter.js.map +1 -0
  79. package/out/build/types.d.ts +15 -0
  80. package/out/build/types.d.ts.map +1 -1
  81. package/out/build/types.js.map +1 -1
  82. package/out/build.d.ts.map +1 -1
  83. package/out/build.js +17 -0
  84. package/out/build.js.map +1 -1
  85. package/out/ee/adapters/ads/adapter.d.ts +6 -0
  86. package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
  87. package/out/ee/adapters/ads/adapter.js +16 -0
  88. package/out/ee/adapters/ads/adapter.js.map +1 -1
  89. package/out/fo/adapters/adressur/adapter.d.ts +63 -0
  90. package/out/fo/adapters/adressur/adapter.d.ts.map +1 -0
  91. package/out/fo/adapters/adressur/adapter.js +177 -0
  92. package/out/fo/adapters/adressur/adapter.js.map +1 -0
  93. package/out/gb/adapters/os-open/adapter.d.ts +10 -2
  94. package/out/gb/adapters/os-open/adapter.d.ts.map +1 -1
  95. package/out/gb/adapters/os-open/adapter.js +13 -2
  96. package/out/gb/adapters/os-open/adapter.js.map +1 -1
  97. package/out/gb/county.d.ts +30 -0
  98. package/out/gb/county.d.ts.map +1 -0
  99. package/out/gb/county.js +51 -0
  100. package/out/gb/county.js.map +1 -0
  101. package/out/gb/os-open/{open-names.d.ts → open/names.d.ts} +18 -1
  102. package/out/gb/os-open/open/names.d.ts.map +1 -0
  103. package/out/gb/os-open/{open-names.js → open/names.js} +20 -1
  104. package/out/gb/os-open/open/names.js.map +1 -0
  105. package/out/gb/os-open/{open-roads.d.ts → open/roads.d.ts} +1 -1
  106. package/out/gb/os-open/open/roads.d.ts.map +1 -0
  107. package/out/gb/os-open/{open-roads.js → open/roads.js} +1 -1
  108. package/out/gb/os-open/open/roads.js.map +1 -0
  109. package/out/gu/adapters/bsp/adapter.d.ts +83 -0
  110. package/out/gu/adapters/bsp/adapter.d.ts.map +1 -0
  111. package/out/gu/adapters/bsp/adapter.js +346 -0
  112. package/out/gu/adapters/bsp/adapter.js.map +1 -0
  113. package/out/im/adapters/land-registry/adapter.d.ts +68 -0
  114. package/out/im/adapters/land-registry/adapter.d.ts.map +1 -0
  115. package/out/im/adapters/land-registry/adapter.js +261 -0
  116. package/out/im/adapters/land-registry/adapter.js.map +1 -0
  117. package/out/international/recipes/locale.d.ts +24 -4
  118. package/out/international/recipes/locale.d.ts.map +1 -1
  119. package/out/international/recipes/locale.js +107 -15
  120. package/out/international/recipes/locale.js.map +1 -1
  121. package/out/li/adapters/llv/adapter.d.ts +69 -0
  122. package/out/li/adapters/llv/adapter.d.ts.map +1 -0
  123. package/out/li/adapters/llv/adapter.js +181 -0
  124. package/out/li/adapters/llv/adapter.js.map +1 -0
  125. package/out/recipes/bare/postcode.d.ts.map +1 -1
  126. package/out/recipes/bare/postcode.js +9 -30
  127. package/out/recipes/bare/postcode.js.map +1 -1
  128. package/out/recipes/country-balanced.d.ts.map +1 -1
  129. package/out/recipes/country-balanced.js +82 -73
  130. package/out/recipes/country-balanced.js.map +1 -1
  131. package/out/recipes/intersection.d.ts +3 -2
  132. package/out/recipes/intersection.d.ts.map +1 -1
  133. package/out/recipes/intersection.js +20 -26
  134. package/out/recipes/intersection.js.map +1 -1
  135. package/out/recipes/po/box/cedex/recipe.d.ts.map +1 -1
  136. package/out/recipes/po/box/cedex/recipe.js +5 -4
  137. package/out/recipes/po/box/cedex/recipe.js.map +1 -1
  138. package/out/recipes/po/box/cedex/sources.d.ts +2 -45
  139. package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
  140. package/out/recipes/po/box/cedex/sources.js +0 -20
  141. package/out/recipes/po/box/cedex/sources.js.map +1 -1
  142. package/out/recipes/scaffold.d.ts +15 -2
  143. package/out/recipes/scaffold.d.ts.map +1 -1
  144. package/out/recipes/scaffold.js +21 -1
  145. package/out/recipes/scaffold.js.map +1 -1
  146. package/out/recipes/sources.d.ts.map +1 -1
  147. package/out/recipes/sources.js +112 -1
  148. package/out/recipes/sources.js.map +1 -1
  149. package/out/recipes/street/affix.d.ts.map +1 -1
  150. package/out/recipes/street/affix.js +8 -20
  151. package/out/recipes/street/affix.js.map +1 -1
  152. package/out/recipes/sub/venue/context.d.ts +14 -3
  153. package/out/recipes/sub/venue/context.d.ts.map +1 -1
  154. package/out/recipes/sub/venue/context.js +29 -9
  155. package/out/recipes/sub/venue/context.js.map +1 -1
  156. package/out/recipes/sub/venue/sources.d.ts +41 -8
  157. package/out/recipes/sub/venue/sources.d.ts.map +1 -1
  158. package/out/recipes/sub/venue/sources.js +26 -7
  159. package/out/recipes/sub/venue/sources.js.map +1 -1
  160. package/out/recipes/sub/venue.d.ts.map +1 -1
  161. package/out/recipes/sub/venue.js +35 -14
  162. package/out/recipes/sub/venue.js.map +1 -1
  163. package/out/recipes/unit.d.ts +4 -0
  164. package/out/recipes/unit.d.ts.map +1 -1
  165. package/out/recipes/unit.js +7 -6
  166. package/out/recipes/unit.js.map +1 -1
  167. package/out/registers.d.ts +33 -0
  168. package/out/registers.d.ts.map +1 -1
  169. package/out/registers.js +33 -0
  170. package/out/registers.js.map +1 -1
  171. package/out/source-register/corpus-inputs.d.ts +240 -0
  172. package/out/source-register/corpus-inputs.d.ts.map +1 -0
  173. package/out/source-register/corpus-inputs.js +275 -0
  174. package/out/source-register/corpus-inputs.js.map +1 -0
  175. package/out/source-register/effective-manifest.d.ts +2 -0
  176. package/out/source-register/effective-manifest.d.ts.map +1 -1
  177. package/out/source-register/effective-manifest.js +33 -4
  178. package/out/source-register/effective-manifest.js.map +1 -1
  179. package/out/source-register/types.d.ts +21 -1
  180. package/out/source-register/types.d.ts.map +1 -1
  181. package/out/source-register/types.js +20 -0
  182. package/out/source-register/types.js.map +1 -1
  183. package/out/south-asia/recipes/register.js +1 -1
  184. package/out/south-asia/recipes/register.js.map +1 -1
  185. package/out/tools/overlay/manifest.d.ts +7 -0
  186. package/out/tools/overlay/manifest.d.ts.map +1 -1
  187. package/out/tools/overlay/manifest.js +72 -1
  188. package/out/tools/overlay/manifest.js.map +1 -1
  189. package/out/tools/source-register/build.d.ts +9 -0
  190. package/out/tools/source-register/build.d.ts.map +1 -1
  191. package/out/tools/source-register/build.js +25 -3
  192. package/out/tools/source-register/build.js.map +1 -1
  193. package/out/tools/variety-sample.d.ts +196 -0
  194. package/out/tools/variety-sample.d.ts.map +1 -0
  195. package/out/tools/variety-sample.js +417 -0
  196. package/out/tools/variety-sample.js.map +1 -0
  197. package/out/us/adapters/usgov/nad/adapter.d.ts +80 -0
  198. package/out/us/adapters/usgov/nad/adapter.d.ts.map +1 -1
  199. package/out/us/adapters/usgov/nad/adapter.js +27 -8
  200. package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
  201. package/out/us/recipes/skeleton-sources.d.ts +92 -0
  202. package/out/us/recipes/skeleton-sources.d.ts.map +1 -0
  203. package/out/us/recipes/skeleton-sources.js +121 -0
  204. package/out/us/recipes/skeleton-sources.js.map +1 -0
  205. package/out/us/tools/nad-skeletons.d.ts +120 -0
  206. package/out/us/tools/nad-skeletons.d.ts.map +1 -0
  207. package/out/us/tools/nad-skeletons.js +262 -0
  208. package/out/us/tools/nad-skeletons.js.map +1 -0
  209. package/out/uy/adapters/ide/adapter.d.ts +64 -0
  210. package/out/uy/adapters/ide/adapter.d.ts.map +1 -0
  211. package/out/uy/adapters/ide/adapter.js +206 -0
  212. package/out/uy/adapters/ide/adapter.js.map +1 -0
  213. package/out/ve/recipes/reviewed-postcode-tail.d.ts +6 -2
  214. package/out/ve/recipes/reviewed-postcode-tail.d.ts.map +1 -1
  215. package/out/ve/recipes/reviewed-postcode-tail.js +16 -5
  216. package/out/ve/recipes/reviewed-postcode-tail.js.map +1 -1
  217. package/package.json +14 -14
  218. package/lib/recipes/unit/sources.ts +0 -44
  219. package/lib/tools/fetch/ppd.ts +0 -121
  220. package/out/gb/os-open/open-names.d.ts.map +0 -1
  221. package/out/gb/os-open/open-names.js.map +0 -1
  222. package/out/gb/os-open/open-roads.d.ts.map +0 -1
  223. package/out/gb/os-open/open-roads.js.map +0 -1
  224. package/out/recipes/unit/sources.d.ts +0 -28
  225. package/out/recipes/unit/sources.d.ts.map +0 -1
  226. package/out/recipes/unit/sources.js +0 -30
  227. package/out/recipes/unit/sources.js.map +0 -1
  228. package/out/tools/fetch/ppd.d.ts +0 -15
  229. package/out/tools/fetch/ppd.d.ts.map +0 -1
  230. package/out/tools/fetch/ppd.js +0 -86
  231. package/out/tools/fetch/ppd.js.map +0 -1
  232. /package/lib/gb/os-open/{open-roads.ts → open/roads.ts} +0 -0
@@ -20,8 +20,6 @@ import {
20
20
  readFrTuples,
21
21
  readPostalTuples,
22
22
  readUsTuples,
23
- US_EVAL_SOURCE,
24
- US_TRAIN_SOURCES,
25
23
  } from "#recipes/po/box/cedex/sources"
26
24
  import type { Rendered, USTuple } from "#recipes/po/box/cedex/types"
27
25
  import { renderPmbUs, renderPoBoxUs } from "#recipes/po/box/cedex/us"
@@ -30,6 +28,11 @@ import { recipeSourceID, type CanonicalRecipeRow, type CorpusRecipe } from "#rec
30
28
  import { defaultRecipeSource } from "#recipes/sources"
31
29
  import { synthesizeMilitaryPoBoxRow } from "#synthesizers/po-box"
32
30
  import { SurfaceOrigin } from "#types"
31
+ import {
32
+ NAD_SKELETON_DIRECTORY,
33
+ US_SKELETON_EVAL_SOURCE,
34
+ US_SKELETON_TRAIN_SOURCES,
35
+ } from "#us/recipes/skeleton-sources"
33
36
  import { alignRow } from "#utils"
34
37
 
35
38
  const COMPONENT_ORDER = ["house_number", "street", "po_box", "venue", "locality", "postcode", "region", "cedex"]
@@ -63,7 +66,7 @@ export const poBoxCedexRecipe: CorpusRecipe = {
63
66
  const source = opts.sourceName ?? defaultRecipeSource("synth-po-box-cedex")
64
67
  const usPool: USTuple[] = []
65
68
 
66
- for (const sourceEntry of opts.golden ? [US_EVAL_SOURCE] : US_TRAIN_SOURCES) {
69
+ for (const sourceEntry of opts.golden ? [US_SKELETON_EVAL_SOURCE] : US_SKELETON_TRAIN_SOURCES) {
67
70
  const tuples = await readUsTuples(sourceEntry)
68
71
 
69
72
  console.error(` ${sourceEntry.csv}: ${tuples.length} tuples`)
@@ -96,7 +99,7 @@ export const poBoxCedexRecipe: CorpusRecipe = {
96
99
 
97
100
  if (!usPool.length || !frPool.length || !qcPool.length || !onPool.length || !auPool.length || !nzPool.length)
98
101
  throw new Error(
99
- `A base pool is empty — check ${dataRootPath("oa-cache")}, ${GEONAMES_CA}, ${GEONAMES_POSTAL_AU.zip}, and ${GEONAMES_POSTAL_NZ.zip}.`
102
+ `A base pool is empty — check ${dataRootPath("oa-cache")}, ${dataRootPath(NAD_SKELETON_DIRECTORY)}, ${GEONAMES_CA}, ${GEONAMES_POSTAL_AU.zip}, and ${GEONAMES_POSTAL_NZ.zip}.`
100
103
  )
101
104
 
102
105
  let emitted = 0,
@@ -185,7 +188,7 @@ export const poBoxCedexRecipe: CorpusRecipe = {
185
188
  ? "GeoNames AU postal dump (CC-BY 4.0) locality/state/postcode tails + Australia Post Postal Delivery Type designators (@mailwoman/codex/au)"
186
189
  : country === "NZ"
187
190
  ? "GeoNames NZ postal dump (CC-BY 4.0) locality/postcode tails + NZ Post ADV358 Delivery Service Types (@mailwoman/codex/nz)"
188
- : "OpenAddresses US (non-VT) skeletons + USPS Pub-28 §29 PO-box designators (codex/corpus templates)",
191
+ : "OpenAddresses and National Address Database US (non-VT) skeletons + USPS Pub-28 §29 PO-box designators (codex/corpus templates)",
189
192
  }
190
193
 
191
194
  const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
@@ -12,28 +12,7 @@ import { TextSpliterator, TSVSpliterator } from "spliterator"
12
12
 
13
13
  import type { AUTuple, FRTuple, NZTuple, USTuple } from "#recipes/po/box/cedex/types"
14
14
  import { readCSVRecords, readZippedCSVRecords } from "#recipes/scaffold"
15
-
16
- /**
17
- * Non-Vermont OpenAddresses files used for training rows.
18
- */
19
- export const US_TRAIN_SOURCES = [
20
- { zip: dataRootPath("oa-cache", "us__ca__berkeley.zip"), csv: "us/ca/berkeley.csv", region: "CA" },
21
- { zip: dataRootPath("oa-cache", "us__ca__marin.zip"), csv: "us/ca/marin.csv", region: "CA" },
22
- { zip: dataRootPath("oa-cache", "us__dc__statewide.zip"), csv: "us/dc/statewide.csv", region: "DC" },
23
- { zip: dataRootPath("oa-cache", "us__ia__statewide.zip"), csv: "us/ia/statewide.csv", region: "IA" },
24
- { zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", region: "IL" },
25
- { zip: dataRootPath("oa-cache", "us__mt__statewide.zip"), csv: "us/mt/statewide.csv", region: "MT" },
26
- { zip: dataRootPath("oa-cache", "us__sd__statewide.zip"), csv: "us/sd/statewide.csv", region: "SD" },
27
- ]
28
-
29
- /**
30
- * Vermont OpenAddresses file reserved for golden rows.
31
- */
32
- export const US_EVAL_SOURCE = {
33
- zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
34
- csv: "us/vt/statewide.csv",
35
- region: "VT",
36
- }
15
+ import type { USSkeletonSource } from "#us/recipes/skeleton-sources"
37
16
 
38
17
  /**
39
18
  * Countrywide French OpenAddresses source.
@@ -72,7 +51,7 @@ export const isHoldoutLocality = (name: string): boolean => localityHash(name) %
72
51
  const cleanLocality = (locality: string) =>
73
52
  locality && locality.length <= MAX_LOCALITY_LENGTH && !/\d|,/.test(locality) && !/cedex/i.test(locality)
74
53
 
75
- export async function readUsTuples(source: { zip: PathBuilderLike; csv: string; region: string }): Promise<USTuple[]> {
54
+ export async function readUsTuples(source: USSkeletonSource): Promise<USTuple[]> {
76
55
  const tuples: USTuple[] = []
77
56
  const seen = new Set<string>()
78
57
 
@@ -11,12 +11,13 @@
11
11
 
12
12
  import { pathExists } from "@mailwoman/core/fs/readers/stat"
13
13
  import { readZipEntry } from "@mailwoman/core/fs/zip"
14
- import { tryParsingJSON, stringifyJSON } from "@mailwoman/core/json"
14
+ import { parseJSONStrict, tryParsingJSON, stringifyJSON } from "@mailwoman/core/json"
15
15
  import type { PathBuilderLike } from "path-ts"
16
16
  import type { AsyncChunkIterator, AsyncDataResource } from "spliterator"
17
17
  import { AsyncSequence, CSVSpliterator, TextSpliterator } from "spliterator"
18
18
 
19
19
  import { stableSourceIDFromParts } from "#adapters/source-id"
20
+ import type { SourceRightsAdmission } from "#source-register/corpus-inputs"
20
21
  import type { SurfaceOrigin } from "#types"
21
22
  import { alignRow } from "#utils"
22
23
 
@@ -284,6 +285,29 @@ export function createRecipeLineWriter(sink: RecipeLineSink): WriteRecipeLine {
284
285
  }
285
286
  }
286
287
 
288
+ /**
289
+ * Wraps a recipe's writer so every line's `source` passes the source-rights check
290
+ * before the line is written.
291
+ *
292
+ * A recipe's `source` is its table default or whatever `--source-name` supplied,
293
+ * so the line itself is the only place the value is certain.
294
+ * The check caches each answer, so the cost after the first line of a source is one parse.
295
+ *
296
+ * @throws From the returned writer, when a line carries no string `source` or its source is refused.
297
+ */
298
+ export function admittingRecipeLineWriter(write: WriteRecipeLine, rights: SourceRightsAdmission): WriteRecipeLine {
299
+ return (line) => {
300
+ const { source } = parseJSONStrict(line) as { source?: unknown }
301
+
302
+ if (typeof source !== "string") {
303
+ throw new TypeError(`a recipe wrote a row with no string \`source\`, so its rights cannot be checked: ${line}`)
304
+ }
305
+
306
+ rights.admit(source)
307
+ write(line)
308
+ }
309
+ }
310
+
287
311
  /**
288
312
  * What a recipe records about the rows it writes.
289
313
  *
@@ -478,8 +502,9 @@ export interface RecipeOptions {
478
502
  /**
479
503
  * `sub-venue`: GB/US/FR address-context tuples jsonl.
480
504
  *
481
- * @defaultValue the house-venue v3 tuples (`$MAILWOMAN_DATA_ROOT/corpus/intermediate/house-venue-tuples-v3.jsonl`);
482
- * DE and ES read OpenAddresses directly.
505
+ * @defaultValue the house-venue v6 tuples (`$MAILWOMAN_DATA_ROOT/corpus/intermediate/house-venue-tuples-v6.jsonl`),
506
+ * whose GB rows come from OS OpenData and whose US rows come from NAD and from the bodies whose own decisions
507
+ * permit training. DE and ES read OpenAddresses directly.
483
508
  */
484
509
  subVenueTuples?: string
485
510
  /**
@@ -135,7 +135,18 @@ export const RECIPE_SOURCES: ReadonlyArray<RecipeSource> = [
135
135
  ["es", LOCALE_RECIPE],
136
136
  ["es-pedania", LOCALE_RECIPE, "The ES source run with `--district-as-locality`, under `--source-name`."],
137
137
  ["nz", LOCALE_RECIPE],
138
- ["gb", LOCALE_RECIPE],
138
+ [
139
+ "gb",
140
+ LOCALE_RECIPE,
141
+ "Rendered from HM Land Registry Price Paid Data address fields, which Royal Mail and Ordnance Survey " +
142
+ "license for non-commercial use only. No recipe writes it now; `rendered-gb-os-open` replaces it (#2635).",
143
+ ],
144
+ [
145
+ "gb-os-open",
146
+ LOCALE_RECIPE,
147
+ "The GB part reads the `gb-os-open` adapter's canonical rows (OS OpenData, OGL v3.0). " +
148
+ "No corpus stores the retired spelling.",
149
+ ],
139
150
  ["fr-order", "fr/recipes/order.ts"],
140
151
  ["fr-admin-split", "fr/recipes/admin-split.ts", "The département is derived from the record's own postcode."],
141
152
  ["fr-lieudit", "fr/recipes/lieudit.ts"],
@@ -283,6 +294,106 @@ export const CARRIED_SOURCES: ReadonlyArray<string> = [
283
294
  "osm-vn",
284
295
  "osm-pk",
285
296
  "osm-bd",
297
+ // The 2026-10-16 and 2026-10-17 extracts for the countries the corpus held under 1,000 street rows of,
298
+ // where no national address register with an electable grant was downloadable (country-gap-v1).
299
+ "osm-eg",
300
+ "osm-rs",
301
+ "osm-ar",
302
+ "osm-ua",
303
+ "osm-kz",
304
+ "osm-by",
305
+ "osm-md",
306
+ "osm-cl",
307
+ "osm-uz",
308
+ "osm-pe",
309
+ "osm-my",
310
+ "osm-bn",
311
+ "osm-ba",
312
+ "osm-ae",
313
+ "osm-kw",
314
+ "osm-ge",
315
+ "osm-am",
316
+ "osm-pr",
317
+ "osm-kg",
318
+ "osm-ph",
319
+ "osm-ci",
320
+ // The 2026-10-16 to 2026-10-18 extracts for the second set of countries under 1,000 street rows,
321
+ // where no national address register with an electable grant was downloadable (country-gap-v2).
322
+ // The Bahamas, Azerbaijan and French Polynesia also carry a national source that covers part
323
+ // of the country (`gbpa-bs` Freeport, `opendata-az` three cities, `ban-pf` three communes).
324
+ "osm-al",
325
+ "osm-ao",
326
+ "osm-aw",
327
+ "osm-az",
328
+ "osm-bb",
329
+ "osm-bh",
330
+ "osm-bm",
331
+ "osm-bs",
332
+ "osm-bt",
333
+ "osm-bw",
334
+ "osm-bz",
335
+ "osm-cd",
336
+ "osm-co",
337
+ "osm-cr",
338
+ "osm-cu",
339
+ "osm-cw",
340
+ "osm-do",
341
+ "osm-dz",
342
+ "osm-ec",
343
+ "osm-gg",
344
+ "osm-gh",
345
+ "osm-gl",
346
+ "osm-gt",
347
+ "osm-hk",
348
+ "osm-ht",
349
+ "osm-iq",
350
+ "osm-ir",
351
+ "osm-je",
352
+ "osm-jm",
353
+ "osm-jo",
354
+ "osm-ke",
355
+ "osm-kh",
356
+ "osm-lb",
357
+ "osm-lk",
358
+ "osm-lr",
359
+ "osm-ma",
360
+ "osm-me",
361
+ "osm-mg",
362
+ "osm-mk",
363
+ "osm-mm",
364
+ "osm-mn",
365
+ "osm-mo",
366
+ "osm-mt",
367
+ "osm-mv",
368
+ "osm-mz",
369
+ "osm-na",
370
+ "osm-np",
371
+ "osm-om",
372
+ "osm-pa",
373
+ "osm-pf",
374
+ "osm-py",
375
+ "osm-qa",
376
+ "osm-rw",
377
+ "osm-sa",
378
+ "osm-sm",
379
+ "osm-sn",
380
+ "osm-sr",
381
+ "osm-sv",
382
+ "osm-th",
383
+ "osm-tj",
384
+ "osm-tm",
385
+ "osm-tn",
386
+ "osm-tt",
387
+ "osm-tz",
388
+ "osm-ug",
389
+ "osm-ve",
390
+ "osm-vi",
391
+ "osm-zm",
392
+ "osm-zw",
393
+ // One id per dependency a national register covers beside its state, written by
394
+ // `corpus run <adapter> --country <cc> --source-name`, so the dependency's rows hold their own weight.
395
+ "ryhti-ax",
396
+ "matrikkelen-sj",
286
397
  // One id per French overseas jurisdiction for the BAN department files,
287
398
  // written by `corpus run ban --source-name`.
288
399
  // Metropolitan rows keep `ban`, so its weight does not decide the rate at which a territory is drawn.
@@ -26,30 +26,14 @@ import { stableSourceID } from "#adapters/source-id"
26
26
  import { readCSVRecords, readOATuples, recipeSourceID, type CorpusRecipe } from "#recipes/scaffold"
27
27
  import { defaultRecipeSource } from "#recipes/sources"
28
28
  import type { CanonicalRow } from "#types"
29
+ import {
30
+ NAD_SKELETON_DIRECTORY,
31
+ US_SKELETON_EVAL_SOURCE,
32
+ US_SKELETON_TRAIN_SOURCES,
33
+ type USSkeletonSource,
34
+ } from "#us/recipes/skeleton-sources"
29
35
  import { alignRow } from "#utils"
30
36
 
31
- interface USSource {
32
- zip: PathBuilderLike
33
- csv: string
34
- region: string
35
- }
36
-
37
- const TRAIN_SOURCES: readonly USSource[] = [
38
- { zip: dataRootPath("oa-cache", "us__ca__berkeley.zip"), csv: "us/ca/berkeley.csv", region: "CA" },
39
- { zip: dataRootPath("oa-cache", "us__ca__marin.zip"), csv: "us/ca/marin.csv", region: "CA" },
40
- { zip: dataRootPath("oa-cache", "us__dc__statewide.zip"), csv: "us/dc/statewide.csv", region: "DC" },
41
- { zip: dataRootPath("oa-cache", "us__ia__statewide.zip"), csv: "us/ia/statewide.csv", region: "IA" },
42
- { zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", region: "IL" },
43
- { zip: dataRootPath("oa-cache", "us__mt__statewide.zip"), csv: "us/mt/statewide.csv", region: "MT" },
44
- { zip: dataRootPath("oa-cache", "us__sd__statewide.zip"), csv: "us/sd/statewide.csv", region: "SD" },
45
- ]
46
-
47
- const EVAL_SOURCE: USSource = {
48
- zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
49
- csv: "us/vt/statewide.csv",
50
- region: "VT",
51
- }
52
-
53
37
  interface BalanceSource {
54
38
  zip: PathBuilderLike
55
39
  csv: string
@@ -118,12 +102,12 @@ interface BalanceTuple {
118
102
 
119
103
  type Prefix = Pick<NonNullable<ReturnType<typeof matchLeadingDirectional>>, "canonical" | "abbreviation">
120
104
 
121
- async function readTuples(source: USSource): Promise<USTuple[]> {
105
+ async function readTuples(source: USSkeletonSource): Promise<USTuple[]> {
122
106
  return readOATuples(source, {
123
107
  extra: (fields, _row, key) => ({
124
108
  ...fields,
125
109
  region: source.region,
126
- base_source_id: `openaddresses:${source.csv}:${key}`,
110
+ base_source_id: `${source.register}:${source.csv}:${key}`,
127
111
  }),
128
112
  })
129
113
  }
@@ -372,7 +356,7 @@ export const streetAffixRecipe: CorpusRecipe = {
372
356
  const count = opts.count ?? 50_000
373
357
  const source = opts.sourceName ?? defaultRecipeSource("synth-affix")
374
358
  const multilocaleCount = opts.multilocaleCount ?? 0
375
- const sources = opts.golden ? [EVAL_SOURCE] : TRAIN_SOURCES
359
+ const sources = opts.golden ? [US_SKELETON_EVAL_SOURCE] : US_SKELETON_TRAIN_SOURCES
376
360
 
377
361
  const pool: USTuple[] = []
378
362
 
@@ -387,7 +371,10 @@ export const streetAffixRecipe: CorpusRecipe = {
387
371
  }
388
372
 
389
373
  if (!pool.length) {
390
- throw new Error(`No US tuples found — are the cached OA zips present in ${dataRootPath("oa-cache")}?`)
374
+ throw new Error(
375
+ `No US tuples found — are the cached OA zips present in ${dataRootPath("oa-cache")} and the NAD ` +
376
+ `skeletons in ${dataRootPath(NAD_SKELETON_DIRECTORY)}?`
377
+ )
391
378
  }
392
379
 
393
380
  let emitted = 0
@@ -448,7 +435,8 @@ export const streetAffixRecipe: CorpusRecipe = {
448
435
  source,
449
436
  source_id: stableSourceID(source, components),
450
437
  corpus_version: "0.4.0",
451
- license: "OpenAddresses US (non-VT) skeletons, street split via USPS Pub-28 C1/C2 (codex)",
438
+ license:
439
+ "OpenAddresses and National Address Database US (non-VT) skeletons, street split via USPS Pub-28 C1/C2 (codex)",
452
440
  }
453
441
 
454
442
  const aligned = alignRow(canonical)
@@ -564,7 +552,7 @@ export const suffixBoundaryRecipe: CorpusRecipe = {
564
552
  const random = makeMulberry32(opts.seed)
565
553
  const count = opts.count ?? 30_000
566
554
  const source = opts.sourceName ?? defaultRecipeSource("synth-suffix-boundary")
567
- const sources = opts.golden ? [EVAL_SOURCE] : TRAIN_SOURCES
555
+ const sources = opts.golden ? [US_SKELETON_EVAL_SOURCE] : US_SKELETON_TRAIN_SOURCES
568
556
 
569
557
  const venuePool = await readVenuePool(VENUE_POOL_CSV)
570
558
 
@@ -630,7 +618,7 @@ export const suffixBoundaryRecipe: CorpusRecipe = {
630
618
  source_id: recipeSourceID(source, { ...components, class: rowClass, n: String(emitted) }),
631
619
  corpus_version: "0.19.0",
632
620
  license:
633
- "OpenAddresses US (non-VT) skeletons; terminal suffix labels via USPS Pub-28 codex; venue shell from HRSA site names (US public domain)",
621
+ "OpenAddresses and National Address Database US (non-VT) skeletons; terminal suffix labels via USPS Pub-28 codex; venue shell from HRSA site names (US public domain)",
634
622
  }
635
623
 
636
624
  const aligned = alignRow(canonical)
@@ -6,6 +6,7 @@
6
6
  */
7
7
 
8
8
  import { dataRootPath } from "@mailwoman/core/data-root"
9
+ import { stringifyJSON } from "@mailwoman/core/json"
9
10
  import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
10
11
  import type { PathBuilderLike } from "path-ts"
11
12
 
@@ -22,24 +23,37 @@ import type { LocaleBaseTuple } from "#surfaces/locale"
22
23
  * so the CSV handling (quoted fields, crlf, city-noise cleaning, the DE per-part region fallback)
23
24
  * has exactly one implementation.
24
25
  *
25
- * The DE adapter reads two members from `europe.zip` instead of `oa-cache/de__*.zip`.
26
- * The cached per-state zips used by the `locale` recipe are absent on this host.
26
+ * The DE parts read the extracted `de/berlin.csv` and `de/sn/statewide.csv`.
27
+ * They are the `europe.zip` members the recipe read before that archive left the data root,
28
+ * with the md5s `9903d438…` and `623b099b…` that the note "a file's mtime is not
29
+ * its data's vintage" in `corpus/agents.md` records.
27
30
  *
28
- * The archive members match OA's current run byte for byte.
29
- * The note "a file's mtime is not its data's vintage" in `corpus/agents.md` records this comparison.
31
+ * A missing part throws in {@link readLocaleTuples}, so the DE leg cannot emit zero rows silently.
30
32
  */
31
33
  const CONTEXT_PARTS: Readonly<Record<string, readonly LocalePart[]>> = {
32
34
  DE: [
33
- { zip: dataRootPath("openaddresses", "europe.zip"), csv: "de/berlin.csv", region: "Berlin" },
34
- { zip: dataRootPath("openaddresses", "europe.zip"), csv: "de/sn/statewide.csv", region: "Sachsen" },
35
+ { path: dataRootPath("openaddresses", "extracted", "de", "berlin.csv"), region: "Berlin" },
36
+ { path: dataRootPath("openaddresses", "extracted", "de", "sn", "statewide.csv"), region: "Sachsen" },
35
37
  ],
36
38
  ES: [{ path: dataRootPath("openaddresses", "extracted", "es", "countrywide.csv") }],
37
39
  }
38
40
 
39
41
  /**
40
- * Load the address skeletons every leg renders onto: GB / US / FR from the house-venue v3 tuples
41
- * (the same 176,519 real rows the `invented-house-venue` recipe output is built from, so the two
42
- * recipes' address halves are drawn from one pool), DE and ES streamed out of OpenAddresses.
42
+ * The `source` a Great Britain house-venue tuple must carry.
43
+ *
44
+ * The v3 tuples took their GB rows from HM Land Registry Price Paid Data, whose address
45
+ * fields Royal Mail and Ordnance Survey license for non-commercial use only (#2635).
46
+ * Their GB rows carry no `source`.
47
+ */
48
+ export const GB_CONTEXT_TUPLE_SOURCE = "gb-os-open"
49
+
50
+ /**
51
+ * Load the address skeletons every leg renders onto: GB / US / FR from the house-venue tuples
52
+ * (`corpora/house_venue_tuples.py`, whose GB rows come from the `gb-os-open` adapter's OS OpenData rows),
53
+ * DE and ES streamed out of OpenAddresses.
54
+ *
55
+ * @throws When a GB tuple does not carry {@linkcode GB_CONTEXT_TUPLE_SOURCE},
56
+ * so a tuples file built from Price Paid Data is refused rather than rendered.
43
57
  */
44
58
  export async function loadContextTuples(
45
59
  tuplesPath: PathBuilderLike,
@@ -50,6 +64,16 @@ export async function loadContextTuples(
50
64
  for await (const tuple of readRecipeTuples(tuplesPath)) {
51
65
  const country = String(tuple.country ?? "")
52
66
 
67
+ // country-branch: GB is the one country whose earlier tuples came from a refused source (#2635).
68
+ // The tuples file records provenance for GB rows only, so the check reads GB rows only.
69
+ if (country === "GB" && tuple.source !== GB_CONTEXT_TUPLE_SOURCE) {
70
+ throw new Error(
71
+ `${tuplesPath.toString()} holds a GB tuple from source ${stringifyJSON(tuple.source ?? null)}; ` +
72
+ `GB address skeletons must come from ${GB_CONTEXT_TUPLE_SOURCE}. Rebuild the tuples with ` +
73
+ "`python -m mailwoman_train.corpora.house_venue_tuples`."
74
+ )
75
+ }
76
+
53
77
  if (!country || !tuple.locality || !tuple.street) continue
54
78
 
55
79
  const mapped: LocaleBaseTuple = {
@@ -12,6 +12,7 @@ import { upperFirst } from "@mailwoman/normalize/case"
12
12
  import type { PathBuilderLike } from "path-ts"
13
13
 
14
14
  import { escapeSQLString, openDuckDB } from "#parquet/duckdb"
15
+ import { SourceRegister } from "#registers"
15
16
  import { classifyIdentifier, readSubVenueJSONL } from "#subvenue/harvest"
16
17
  import { SUBVENUE_PROMOTIONS, type SubVenuePromotion } from "#subvenue/promotions"
17
18
  import type { SubVenueLexiconTable } from "#subvenue/table"
@@ -378,11 +379,34 @@ export function sampleIdentifier(model: IdentifierModel, designatorID: string, r
378
379
  }
379
380
 
380
381
  /**
381
- * The address context and name pools for one recipe leg.
382
+ * The publication a pooled name was read from.
383
+ *
384
+ * Each row records the sources of the names it carries, because the two publications are
385
+ * licensed differently: OpenStreetMap under ODbL 1.0, Overture Places under CDLA-Permissive-2.0.
382
386
  */
383
- export interface LegPools {
384
- context: LocaleBaseTuple[]
387
+ export const NameSource = {
388
+ OpenStreetMap: SourceRegister.OpenStreetMap,
389
+ OverturePlaces: "overture-places",
390
+ } as const
385
391
 
392
+ export type NameSource = (typeof NameSource)[keyof typeof NameSource]
393
+
394
+ /**
395
+ * A pooled name with the publication it was read from.
396
+ *
397
+ * A name both sources hold appears once per source, so its draw still records the pool it came from.
398
+ */
399
+ export interface SourcedName {
400
+ name: string
401
+ source: NameSource
402
+ }
403
+
404
+ /**
405
+ * The name pools that one source contributes to a recipe leg.
406
+ *
407
+ * Only OSM extracts fill `attested` and `unpromotedShapes`, because POI data has no tier or localized names.
408
+ */
409
+ export interface NamePools {
386
410
  /**
387
411
  * Real venue-tier names that fit the venue slot, such as stations, airports, campuses and hospitals.
388
412
  */
@@ -410,11 +434,19 @@ export interface LegPools {
410
434
  }
411
435
 
412
436
  /**
413
- * The name pools that one source contributes to a recipe leg.
437
+ * The address context and name pools for one recipe leg.
414
438
  *
415
- * Only OSM extracts fill `attested` and `unpromotedShapes`, because POI data has no tier or localized names.
439
+ * `attested` and `unpromotedShapes` hold OpenStreetMap names only, as {@linkcode NamePools} states.
440
+ * The other three pools merge both sources and keep each name's source.
416
441
  */
417
- export type NamePools = Pick<LegPools, "venues" | "attested" | "rejectedVenues" | "longerNames" | "unpromotedShapes">
442
+ export interface LegPools {
443
+ context: LocaleBaseTuple[]
444
+ venues: SourcedName[]
445
+ attested: string[]
446
+ rejectedVenues: SourcedName[]
447
+ longerNames: SourcedName[]
448
+ unpromotedShapes: string[]
449
+ }
418
450
 
419
451
  /**
420
452
  * Empty name pools for a source that a leg lacks.
@@ -571,14 +603,23 @@ export async function readOvertureNamePools(
571
603
  }
572
604
 
573
605
  /**
574
- * Concatenates two sources' name pools.
606
+ * Concatenates an OpenStreetMap extract's pools and the Overture Places pools,
607
+ * OpenStreetMap first, and tags each merged name with its source.
608
+ *
609
+ * The order and length of every pool match a plain concatenation, so a seeded run
610
+ * draws the same names as one that recorded no source.
575
611
  */
576
- export function mergeNamePools(a: NamePools, b: NamePools): NamePools {
612
+ export function mergeNamePools(osm: NamePools, overture: NamePools): Omit<LegPools, "context"> {
613
+ const tag = (a: readonly string[], b: readonly string[]): SourcedName[] => [
614
+ ...a.map((name) => ({ name, source: NameSource.OpenStreetMap })),
615
+ ...b.map((name) => ({ name, source: NameSource.OverturePlaces })),
616
+ ]
617
+
577
618
  return {
578
- venues: [...a.venues, ...b.venues],
579
- attested: [...a.attested, ...b.attested],
580
- rejectedVenues: [...a.rejectedVenues, ...b.rejectedVenues],
581
- longerNames: [...a.longerNames, ...b.longerNames],
582
- unpromotedShapes: [...a.unpromotedShapes, ...b.unpromotedShapes],
619
+ venues: tag(osm.venues, overture.venues),
620
+ attested: [...osm.attested, ...overture.attested],
621
+ rejectedVenues: tag(osm.rejectedVenues, overture.rejectedVenues),
622
+ longerNames: tag(osm.longerNames, overture.longerNames),
623
+ unpromotedShapes: [...osm.unpromotedShapes, ...overture.unpromotedShapes],
583
624
  }
584
625
  }