@mailwoman/corpus 10.6.0 → 10.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (223) hide show
  1. package/data/PROVENANCE.md +24 -0
  2. package/data/address-source-register.json +18948 -8757
  3. package/data/corpus-source-inputs.json +4756 -0
  4. package/data/license-decisions.json +7260 -0
  5. package/data/licenses/whosonfirst-licenses.md +3 -2
  6. package/data/reviewed-ve-postcode-tuples.json +5 -72
  7. package/data/source-additions.json +2450 -0
  8. package/data/source-resolutions.json +87 -0
  9. package/data/training-manifests/v0.7.10-gb-county.effective.json +2335 -0
  10. package/data/training-manifests/v0.7.10-gb-county.json +1132 -0
  11. package/data/training-manifests/v0.7.2-address-systems.effective.json +3 -2
  12. package/data/training-manifests/v0.7.7-no-ppd.json +550 -0
  13. package/data/training-manifests/v0.7.8-rights-clean.json +538 -0
  14. package/data/training-manifests/v0.7.9-country-gap.json +1132 -0
  15. package/data/us-situs-sources.json +128 -0
  16. package/lib/adapters/cordis/adapter.ts +1 -1
  17. package/lib/adapters/libpostal-dictionary.ts +23 -3
  18. package/lib/adapters/osm/adapter.ts +42 -11
  19. package/lib/adapters.ts +14 -0
  20. package/lib/az/adapters/opendata/adapter.ts +389 -0
  21. package/lib/bs/adapters/gbpa/adapter.ts +248 -0
  22. package/lib/build/types.ts +16 -0
  23. package/lib/build.ts +21 -0
  24. package/lib/ee/adapters/ads/adapter.ts +18 -0
  25. package/lib/fo/adapters/adressur/adapter.ts +242 -0
  26. package/lib/gb/adapters/os-open/adapter.ts +12 -0
  27. package/lib/gb/county.ts +59 -0
  28. package/lib/gb/os-open/open/names.ts +41 -0
  29. package/lib/gu/adapters/bsp/adapter.ts +434 -0
  30. package/lib/im/adapters/land-registry/adapter.ts +340 -0
  31. package/lib/international/recipes/locale.ts +164 -19
  32. package/lib/li/adapters/llv/adapter.ts +246 -0
  33. package/lib/recipes/bare/postcode.ts +10 -34
  34. package/lib/recipes/country-balanced.ts +95 -78
  35. package/lib/recipes/intersection.ts +22 -30
  36. package/lib/recipes/po/box/cedex/recipe.ts +8 -5
  37. package/lib/recipes/po/box/cedex/sources.ts +2 -23
  38. package/lib/recipes/scaffold.ts +28 -3
  39. package/lib/recipes/sources.ts +112 -1
  40. package/lib/recipes/street/affix.ts +17 -29
  41. package/lib/recipes/sub/venue/context.ts +33 -9
  42. package/lib/recipes/sub/venue/sources.ts +54 -13
  43. package/lib/recipes/sub/venue.ts +68 -23
  44. package/lib/recipes/unit.ts +20 -7
  45. package/lib/registers.ts +33 -0
  46. package/lib/source-register/corpus-inputs.ts +528 -0
  47. package/lib/source-register/effective-manifest.ts +37 -5
  48. package/lib/source-register/types.ts +20 -0
  49. package/lib/south-asia/recipes/register.ts +1 -1
  50. package/lib/tools/overlay/manifest.ts +109 -1
  51. package/lib/tools/source-register/build.ts +47 -1
  52. package/lib/tools/variety-sample.ts +633 -0
  53. package/lib/us/adapters/usgov/nad/adapter.ts +39 -10
  54. package/lib/us/recipes/skeleton-sources.ts +156 -0
  55. package/lib/us/tools/nad-skeletons.ts +395 -0
  56. package/lib/uy/adapters/ide/adapter.ts +275 -0
  57. package/lib/ve/recipes/reviewed-postcode-tail.ts +29 -6
  58. package/out/adapters/cordis/adapter.d.ts +1 -1
  59. package/out/adapters/cordis/adapter.js +1 -1
  60. package/out/adapters/libpostal-dictionary.d.ts +9 -0
  61. package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
  62. package/out/adapters/libpostal-dictionary.js +19 -3
  63. package/out/adapters/libpostal-dictionary.js.map +1 -1
  64. package/out/adapters/osm/adapter.d.ts +12 -1
  65. package/out/adapters/osm/adapter.d.ts.map +1 -1
  66. package/out/adapters/osm/adapter.js +30 -13
  67. package/out/adapters/osm/adapter.js.map +1 -1
  68. package/out/adapters.d.ts.map +1 -1
  69. package/out/adapters.js +14 -0
  70. package/out/adapters.js.map +1 -1
  71. package/out/az/adapters/opendata/adapter.d.ts +88 -0
  72. package/out/az/adapters/opendata/adapter.d.ts.map +1 -0
  73. package/out/az/adapters/opendata/adapter.js +282 -0
  74. package/out/az/adapters/opendata/adapter.js.map +1 -0
  75. package/out/bs/adapters/gbpa/adapter.d.ts +64 -0
  76. package/out/bs/adapters/gbpa/adapter.d.ts.map +1 -0
  77. package/out/bs/adapters/gbpa/adapter.js +187 -0
  78. package/out/bs/adapters/gbpa/adapter.js.map +1 -0
  79. package/out/build/types.d.ts +15 -0
  80. package/out/build/types.d.ts.map +1 -1
  81. package/out/build/types.js.map +1 -1
  82. package/out/build.d.ts.map +1 -1
  83. package/out/build.js +17 -0
  84. package/out/build.js.map +1 -1
  85. package/out/ee/adapters/ads/adapter.d.ts +6 -0
  86. package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
  87. package/out/ee/adapters/ads/adapter.js +16 -0
  88. package/out/ee/adapters/ads/adapter.js.map +1 -1
  89. package/out/fo/adapters/adressur/adapter.d.ts +63 -0
  90. package/out/fo/adapters/adressur/adapter.d.ts.map +1 -0
  91. package/out/fo/adapters/adressur/adapter.js +177 -0
  92. package/out/fo/adapters/adressur/adapter.js.map +1 -0
  93. package/out/gb/adapters/os-open/adapter.d.ts +8 -0
  94. package/out/gb/adapters/os-open/adapter.d.ts.map +1 -1
  95. package/out/gb/adapters/os-open/adapter.js +11 -0
  96. package/out/gb/adapters/os-open/adapter.js.map +1 -1
  97. package/out/gb/county.d.ts +30 -0
  98. package/out/gb/county.d.ts.map +1 -0
  99. package/out/gb/county.js +51 -0
  100. package/out/gb/county.js.map +1 -0
  101. package/out/gb/os-open/open/names.d.ts +17 -0
  102. package/out/gb/os-open/open/names.d.ts.map +1 -1
  103. package/out/gb/os-open/open/names.js +19 -0
  104. package/out/gb/os-open/open/names.js.map +1 -1
  105. package/out/gu/adapters/bsp/adapter.d.ts +83 -0
  106. package/out/gu/adapters/bsp/adapter.d.ts.map +1 -0
  107. package/out/gu/adapters/bsp/adapter.js +346 -0
  108. package/out/gu/adapters/bsp/adapter.js.map +1 -0
  109. package/out/im/adapters/land-registry/adapter.d.ts +68 -0
  110. package/out/im/adapters/land-registry/adapter.d.ts.map +1 -0
  111. package/out/im/adapters/land-registry/adapter.js +261 -0
  112. package/out/im/adapters/land-registry/adapter.js.map +1 -0
  113. package/out/international/recipes/locale.d.ts +24 -4
  114. package/out/international/recipes/locale.d.ts.map +1 -1
  115. package/out/international/recipes/locale.js +107 -15
  116. package/out/international/recipes/locale.js.map +1 -1
  117. package/out/li/adapters/llv/adapter.d.ts +69 -0
  118. package/out/li/adapters/llv/adapter.d.ts.map +1 -0
  119. package/out/li/adapters/llv/adapter.js +181 -0
  120. package/out/li/adapters/llv/adapter.js.map +1 -0
  121. package/out/recipes/bare/postcode.d.ts.map +1 -1
  122. package/out/recipes/bare/postcode.js +9 -30
  123. package/out/recipes/bare/postcode.js.map +1 -1
  124. package/out/recipes/country-balanced.d.ts.map +1 -1
  125. package/out/recipes/country-balanced.js +82 -73
  126. package/out/recipes/country-balanced.js.map +1 -1
  127. package/out/recipes/intersection.d.ts +3 -2
  128. package/out/recipes/intersection.d.ts.map +1 -1
  129. package/out/recipes/intersection.js +20 -26
  130. package/out/recipes/intersection.js.map +1 -1
  131. package/out/recipes/po/box/cedex/recipe.d.ts.map +1 -1
  132. package/out/recipes/po/box/cedex/recipe.js +5 -4
  133. package/out/recipes/po/box/cedex/recipe.js.map +1 -1
  134. package/out/recipes/po/box/cedex/sources.d.ts +2 -45
  135. package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
  136. package/out/recipes/po/box/cedex/sources.js +0 -20
  137. package/out/recipes/po/box/cedex/sources.js.map +1 -1
  138. package/out/recipes/scaffold.d.ts +15 -2
  139. package/out/recipes/scaffold.d.ts.map +1 -1
  140. package/out/recipes/scaffold.js +21 -1
  141. package/out/recipes/scaffold.js.map +1 -1
  142. package/out/recipes/sources.d.ts.map +1 -1
  143. package/out/recipes/sources.js +112 -1
  144. package/out/recipes/sources.js.map +1 -1
  145. package/out/recipes/street/affix.d.ts.map +1 -1
  146. package/out/recipes/street/affix.js +8 -20
  147. package/out/recipes/street/affix.js.map +1 -1
  148. package/out/recipes/sub/venue/context.d.ts +14 -3
  149. package/out/recipes/sub/venue/context.d.ts.map +1 -1
  150. package/out/recipes/sub/venue/context.js +29 -9
  151. package/out/recipes/sub/venue/context.js.map +1 -1
  152. package/out/recipes/sub/venue/sources.d.ts +41 -8
  153. package/out/recipes/sub/venue/sources.d.ts.map +1 -1
  154. package/out/recipes/sub/venue/sources.js +26 -7
  155. package/out/recipes/sub/venue/sources.js.map +1 -1
  156. package/out/recipes/sub/venue.d.ts.map +1 -1
  157. package/out/recipes/sub/venue.js +35 -14
  158. package/out/recipes/sub/venue.js.map +1 -1
  159. package/out/recipes/unit.d.ts +4 -0
  160. package/out/recipes/unit.d.ts.map +1 -1
  161. package/out/recipes/unit.js +7 -6
  162. package/out/recipes/unit.js.map +1 -1
  163. package/out/registers.d.ts +33 -0
  164. package/out/registers.d.ts.map +1 -1
  165. package/out/registers.js +33 -0
  166. package/out/registers.js.map +1 -1
  167. package/out/source-register/corpus-inputs.d.ts +240 -0
  168. package/out/source-register/corpus-inputs.d.ts.map +1 -0
  169. package/out/source-register/corpus-inputs.js +275 -0
  170. package/out/source-register/corpus-inputs.js.map +1 -0
  171. package/out/source-register/effective-manifest.d.ts +2 -0
  172. package/out/source-register/effective-manifest.d.ts.map +1 -1
  173. package/out/source-register/effective-manifest.js +33 -4
  174. package/out/source-register/effective-manifest.js.map +1 -1
  175. package/out/source-register/types.d.ts +21 -1
  176. package/out/source-register/types.d.ts.map +1 -1
  177. package/out/source-register/types.js +20 -0
  178. package/out/source-register/types.js.map +1 -1
  179. package/out/south-asia/recipes/register.js +1 -1
  180. package/out/south-asia/recipes/register.js.map +1 -1
  181. package/out/tools/overlay/manifest.d.ts +7 -0
  182. package/out/tools/overlay/manifest.d.ts.map +1 -1
  183. package/out/tools/overlay/manifest.js +72 -1
  184. package/out/tools/overlay/manifest.js.map +1 -1
  185. package/out/tools/source-register/build.d.ts +9 -0
  186. package/out/tools/source-register/build.d.ts.map +1 -1
  187. package/out/tools/source-register/build.js +25 -3
  188. package/out/tools/source-register/build.js.map +1 -1
  189. package/out/tools/variety-sample.d.ts +196 -0
  190. package/out/tools/variety-sample.d.ts.map +1 -0
  191. package/out/tools/variety-sample.js +417 -0
  192. package/out/tools/variety-sample.js.map +1 -0
  193. package/out/us/adapters/usgov/nad/adapter.d.ts +80 -0
  194. package/out/us/adapters/usgov/nad/adapter.d.ts.map +1 -1
  195. package/out/us/adapters/usgov/nad/adapter.js +27 -8
  196. package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
  197. package/out/us/recipes/skeleton-sources.d.ts +92 -0
  198. package/out/us/recipes/skeleton-sources.d.ts.map +1 -0
  199. package/out/us/recipes/skeleton-sources.js +121 -0
  200. package/out/us/recipes/skeleton-sources.js.map +1 -0
  201. package/out/us/tools/nad-skeletons.d.ts +120 -0
  202. package/out/us/tools/nad-skeletons.d.ts.map +1 -0
  203. package/out/us/tools/nad-skeletons.js +262 -0
  204. package/out/us/tools/nad-skeletons.js.map +1 -0
  205. package/out/uy/adapters/ide/adapter.d.ts +64 -0
  206. package/out/uy/adapters/ide/adapter.d.ts.map +1 -0
  207. package/out/uy/adapters/ide/adapter.js +206 -0
  208. package/out/uy/adapters/ide/adapter.js.map +1 -0
  209. package/out/ve/recipes/reviewed-postcode-tail.d.ts +6 -2
  210. package/out/ve/recipes/reviewed-postcode-tail.d.ts.map +1 -1
  211. package/out/ve/recipes/reviewed-postcode-tail.js +16 -5
  212. package/out/ve/recipes/reviewed-postcode-tail.js.map +1 -1
  213. package/package.json +14 -14
  214. package/lib/recipes/unit/sources.ts +0 -44
  215. package/lib/tools/fetch/ppd.ts +0 -121
  216. package/out/recipes/unit/sources.d.ts +0 -28
  217. package/out/recipes/unit/sources.d.ts.map +0 -1
  218. package/out/recipes/unit/sources.js +0 -30
  219. package/out/recipes/unit/sources.js.map +0 -1
  220. package/out/tools/fetch/ppd.d.ts +0 -15
  221. package/out/tools/fetch/ppd.d.ts.map +0 -1
  222. package/out/tools/fetch/ppd.js +0 -86
  223. package/out/tools/fetch/ppd.js.map +0 -1
@@ -0,0 +1,246 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `llv-li`: the official index of building addresses of Liechtenstein, `Verzeichnis der
7
+ * Gebäudeadressen`, which the Liechtensteinische Landesverwaltung publishes on
8
+ * `service.geo.llv.li/download/` as one ZIP (`getfile.php?theme=officialAdress`).
9
+ *
10
+ * The ZIP holds `dataset.csv`, a shapefile of the same records and the license PDF. The CSV is
11
+ * Windows-1252 (`Äussere Wiesen` reads `\xc4ussere Wiesen`), CRLF-terminated and `;`-separated, with no
12
+ * quoting. The adapter reads `dataset.csv` from the ZIP, or from a path to the extracted CSV.
13
+ *
14
+ * The adapter reads `STN_NAME` as the street, `ADR_NUMBER` as the house number (it carries its own
15
+ * lower-case letter suffix, `12c`), `ZIP_ZIP4` as the postcode and `ZIP_NAME` as the postal locality.
16
+ * `COM_FOSNR` is the municipality's number and no column holds its name, so no region is emitted. The LI
17
+ * layout prints no region in any case.
18
+ *
19
+ * `STATUS` reads `real` or `planned`. A planned address is one assigned to a building not yet built, so
20
+ * the adapter refuses it. `OFFICIAL` reads `TRUE` or `FALSE`; an address the register marks as not
21
+ * official is refused too, because its text is not the designation the municipality assigned.
22
+ *
23
+ * The publisher grants reproduction, redistribution, enrichment, editing, combination and commercial use
24
+ * under its `ABI/OGD-Version 2 vom 5.2.2021` terms, on condition that the source
25
+ * `Liechtensteinische Landesverwaltung` is stated and the terms are enclosed or linked. Every row
26
+ * records that license.
27
+ */
28
+
29
+ import { formatAddressRow } from "@mailwoman/codex/address/format"
30
+ import { matchCountryPostcode } from "@mailwoman/codex/postcode/country-formats"
31
+ import { readLocalBuffer } from "@mailwoman/core/fs/readers"
32
+ import { decodeBytes } from "@mailwoman/core/fs/streams"
33
+ import { readZipEntry } from "@mailwoman/core/fs/zip"
34
+ import { resolvePath } from "path-ts"
35
+ import { CSVSpliterator } from "spliterator"
36
+
37
+ import { UnsupportedCountryError } from "#adapters/errors"
38
+ import { stableSourceIDFromParts } from "#adapters/source-id"
39
+ import { SourceRegister } from "#registers"
40
+ import {
41
+ AddressRole,
42
+ type AdapterOptions,
43
+ type CanonicalRow,
44
+ type CorpusAdapter,
45
+ countDropped,
46
+ SurfaceOrigin,
47
+ } from "#types"
48
+
49
+ /**
50
+ * Registry id for this adapter, stamped into every row it emits.
51
+ */
52
+ export const LLV_LI_ADAPTER_ID = "llv-li"
53
+
54
+ /**
55
+ * The Landesverwaltung's own terms, `ABI/OGD-Version 2 vom 5.2.2021`.
56
+ * They have no SPDX identifier.
57
+ */
58
+ export const LLV_LI_LICENSE = "LicenseRef-LI-ABI-OGD-2"
59
+
60
+ /**
61
+ * Every jurisdiction this adapter emits, checked against a caller's `--country`.
62
+ */
63
+ export const LLV_LI_COUNTRIES: readonly string[] = ["LI"]
64
+
65
+ /**
66
+ * The ZIP member that holds the address table.
67
+ */
68
+ export const LLV_LI_MEMBER = "dataset.csv"
69
+
70
+ /**
71
+ * The columns the adapter reads from `dataset.csv`.
72
+ */
73
+ export interface LLVLiechtensteinRecord {
74
+ ADR_EGAID: string
75
+ STN_NAME: string
76
+ ADR_NUMBER: string
77
+ ZIP_ZIP4: string
78
+ ZIP_NAME: string
79
+ STATUS: string
80
+ OFFICIAL: string
81
+ }
82
+
83
+ function present(value: string | undefined): string | null {
84
+ const trimmed = value?.trim() ?? ""
85
+
86
+ return trimmed || null
87
+ }
88
+
89
+ /**
90
+ * Maps one record onto the components the corpus asserts, or returns null when the row is refused.
91
+ *
92
+ * Each refusal increments its reason in `dropped`.
93
+ */
94
+ export function componentsForLLVRecord(
95
+ record: LLVLiechtensteinRecord,
96
+ dropped?: Pick<AdapterOptions, "dropped">
97
+ ): CanonicalRow["components"] | null {
98
+ const tally = dropped ?? {}
99
+
100
+ if (record.STATUS?.trim() !== "real") {
101
+ countDropped(tally, record.STATUS?.trim() === "planned" ? "row:status-planned" : "row:status-unknown")
102
+
103
+ return null
104
+ }
105
+
106
+ if (record.OFFICIAL?.trim().toUpperCase() !== "TRUE") {
107
+ countDropped(tally, "row:not-official")
108
+
109
+ return null
110
+ }
111
+
112
+ const street = present(record.STN_NAME)
113
+
114
+ if (!street) {
115
+ countDropped(tally, "row:no-street")
116
+
117
+ return null
118
+ }
119
+
120
+ const number = present(record.ADR_NUMBER)
121
+
122
+ if (!number) {
123
+ countDropped(tally, "row:no-house-number")
124
+
125
+ return null
126
+ }
127
+
128
+ const components: CanonicalRow["components"] = {
129
+ street,
130
+ house_number: number,
131
+ }
132
+
133
+ const postcode = present(record.ZIP_ZIP4)
134
+ const matched = postcode ? matchCountryPostcode(postcode, "LI") : null
135
+
136
+ if (matched) {
137
+ components.postcode = matched
138
+ } else if (postcode) {
139
+ countDropped(tally, "component:postcode:not-country-format")
140
+ }
141
+
142
+ const locality = present(record.ZIP_NAME)
143
+
144
+ if (locality) {
145
+ components.locality = locality
146
+ }
147
+
148
+ if (!components.locality && !components.postcode) {
149
+ countDropped(tally, "row:no-locality-or-postcode")
150
+
151
+ return null
152
+ }
153
+
154
+ return components
155
+ }
156
+
157
+ /**
158
+ * The bytes of `dataset.csv`, read from the publisher's ZIP or from the extracted file.
159
+ *
160
+ * The table is about 2 MB, so it is held whole.
161
+ */
162
+ async function datasetBytes(inputPath: string): Promise<Buffer> {
163
+ if (!inputPath.toLowerCase().endsWith(".zip")) return readLocalBuffer(inputPath)
164
+
165
+ const chunks: Uint8Array[] = []
166
+
167
+ for await (const chunk of readZipEntry(inputPath, LLV_LI_MEMBER)) {
168
+ chunks.push(chunk)
169
+ }
170
+
171
+ return Buffer.concat(chunks)
172
+ }
173
+
174
+ export function createLLVLiechtensteinAdapter(): CorpusAdapter {
175
+ return {
176
+ id: LLV_LI_ADAPTER_ID,
177
+ defaultLicense: LLV_LI_LICENSE,
178
+ addressRole: AddressRole.Premise,
179
+ register: SourceRegister.LiechtensteinGebaeudeadressen,
180
+ surface: SurfaceOrigin.Rendered,
181
+ description: "Verzeichnis der Gebäudeadressen (Liechtensteinische Landesverwaltung): national building addresses.",
182
+
183
+ async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
184
+ if (opts.country && !LLV_LI_COUNTRIES.includes(opts.country)) {
185
+ throw new UnsupportedCountryError(LLV_LI_ADAPTER_ID, LLV_LI_COUNTRIES, opts.country)
186
+ }
187
+
188
+ const text = decodeBytes(await datasetBytes(resolvePath(opts.inputPath).toString()), "windows-1252")
189
+
190
+ const records = CSVSpliterator.from<LLVLiechtensteinRecord>(Buffer.from(text, "utf8"), {
191
+ normalizeKeys: false,
192
+ columnDelimiter: ";",
193
+ })
194
+
195
+ let emitted = 0
196
+
197
+ for (const record of records) {
198
+ if (opts.signal?.aborted) return
199
+
200
+ if (opts.limit !== undefined && emitted >= opts.limit) return
201
+
202
+ const components = componentsForLLVRecord(record, opts)
203
+
204
+ if (!components) continue
205
+
206
+ const rendered = formatAddressRow(components, "LI", {
207
+ singleLine: true,
208
+ })
209
+
210
+ if (!rendered) {
211
+ countDropped(opts, "row:render-failed")
212
+
213
+ continue
214
+ }
215
+
216
+ for (const tag of rendered.unplaced) {
217
+ countDropped(opts, `component:${tag}:not-rendered`)
218
+ }
219
+
220
+ const { raw, components: aligned } = rendered
221
+
222
+ yield {
223
+ raw,
224
+ components: aligned,
225
+ country: "LI",
226
+ locale: "de-LI",
227
+ source: LLV_LI_ADAPTER_ID,
228
+ // `ADR_EGAID` is the federal address identifier, unique per row of the register.
229
+ source_id: stableSourceIDFromParts(LLV_LI_ADAPTER_ID, {
230
+ ...aligned,
231
+ egaid: record.ADR_EGAID ?? "",
232
+ }),
233
+ corpus_version: "",
234
+ license: LLV_LI_LICENSE,
235
+ }
236
+
237
+ emitted++
238
+ }
239
+ },
240
+ }
241
+ }
242
+
243
+ /**
244
+ * The configured adapter instance registered with the corpus builder.
245
+ */
246
+ export const llvLiechtensteinAdapter = createLLVLiechtensteinAdapter()
@@ -44,8 +44,7 @@ const BARE_POSTCODE_PROVENANCE = {
44
44
  }
45
45
 
46
46
  /**
47
- * One country's postcodes: the CSV under the extracted OpenAddresses tree and the country it covers,
48
- * where Sweden publishes sixteen municipality files against CZ/SK/NL's one countrywide file each.
47
+ * One country's postcodes: the CSV under the extracted OpenAddresses tree and the country it covers.
49
48
  */
50
49
  interface PostcodeSource {
51
50
  csv: PathBuilderLike
@@ -55,40 +54,18 @@ interface PostcodeSource {
55
54
  const EXTRACTED = dataRootPath("openaddresses", "extracted")
56
55
 
57
56
  /**
58
- * Read from the archive rather than guessed, because OpenAddresses keeps the Swedish
59
- * spelling on two of these (`savsjö`, `Österåker`) while folding the rest to ascii.
60
- */
61
- const SWEDISH_MUNICIPALITIES = [
62
- "alingsas",
63
- "gislaved",
64
- "goteborg",
65
- "helsingborg",
66
- "hoganas",
67
- "kalmar",
68
- "kristinehamn",
69
- "malmo",
70
- "nacka",
71
- "savsjö",
72
- "stockholm",
73
- "uppsala",
74
- "vasteras",
75
- "vaxholm",
76
- "vaxjo",
77
- "Österåker",
78
- ]
79
-
80
- /**
57
+ * Each file's publisher has a register row whose elected decision permits transform
58
+ * and train, and `corpus-source-inputs.json` lists that row under `codex-bare-postcode`.
59
+ *
81
60
  * Greece is deliberately absent: its only member declares a `postcode` column with no values over
82
- * 10,877 rows, though `gr_postcode` shares `NNN NN` with CZ/SK/SE, so no row here claims to be Greek.
61
+ * 10,877 rows, though `gr_postcode` shares `NNN NN` with CZ/SK, so no row here claims to be Greek.
62
+ * Sweden's sixteen `se/municipality_of_*` files are absent because their register row,
63
+ * `se-property-building-2`, records no decision permitting training.
83
64
  */
84
65
  const SOURCES: PostcodeSource[] = [
85
66
  { csv: EXTRACTED("cz", "countrywide.csv"), country: "CZ" },
86
67
  { csv: EXTRACTED("sk", "countrywide.csv"), country: "SK" },
87
68
  { csv: EXTRACTED("nl", "countrywide.csv"), country: "NL" },
88
- ...SWEDISH_MUNICIPALITIES.map((name) => ({
89
- csv: EXTRACTED("se", `municipality_of_${name}.csv`),
90
- country: "SE",
91
- })),
92
69
  ]
93
70
 
94
71
  /**
@@ -185,8 +162,7 @@ export function detectedAsPostcode(surface: string): boolean {
185
162
  */
186
163
  export const barePostcodeRecipe: CorpusRecipe = {
187
164
  name: "bare-postcode",
188
- description:
189
- "A postcode alone, in its country's written form — the shape no other recipe output carries (CZ/SK/SE/NL)",
165
+ description: "A postcode alone, in its country's written form — the shape no other recipe output carries (CZ/SK/NL)",
190
166
  mode: "generate",
191
167
  options: [
192
168
  {
@@ -201,8 +177,8 @@ export const barePostcodeRecipe: CorpusRecipe = {
201
177
  let unrecognized = 0
202
178
 
203
179
  // Preflight the complete input set before writing a row.
204
- // A missing municipality would otherwise produce a plausible artifact with
205
- // less Swedish coverage than the recipe declares.
180
+ // A missing file would otherwise produce a plausible artifact with less
181
+ // coverage than the recipe declares.
206
182
  const missing = await findMissingPostcodeSources(SOURCES.map(({ csv }) => csv))
207
183
 
208
184
  if (missing.length) {
@@ -21,6 +21,7 @@ import { readOATuples, requireRegister, type CorpusRecipe } from "#recipes/scaff
21
21
  import { defaultRecipeSource } from "#recipes/sources"
22
22
  import { SurfaceOrigin } from "#types"
23
23
  import type { CanonicalRow } from "#types"
24
+ import { US_SKELETON_EVAL_SOURCE, US_SKELETON_TRAIN_SOURCES } from "#us/recipes/skeleton-sources"
24
25
  import { alignRow } from "#utils"
25
26
 
26
27
  // Country names are drawn without regard to the address's own country.
@@ -39,7 +40,7 @@ const COUNTRY_FORM_POOL = (() => {
39
40
  const COUNTRY_ABSENT_PROB = 0.3
40
41
 
41
42
  /**
42
- * One OpenAddresses archive and how to render its rows.
43
+ * One OA-layout archive and how to render its rows.
43
44
  */
44
45
  interface CountrySource {
45
46
  zip: PathBuilderLike
@@ -47,76 +48,75 @@ interface CountrySource {
47
48
  iso2: string
48
49
  region: string
49
50
  order: string
51
+ /**
52
+ * The register the archive's rows come from, when it differs from the invocation's `--register`.
53
+ */
54
+ register?: string
50
55
  }
51
56
 
52
57
  /**
53
- * The training sources.
58
+ * The training sources, in two groups that {@linkcode readSourcePool} gives equal halves of the read budget.
59
+ *
60
+ * The first group is the shared US skeletons.
61
+ * They take the half that the four OpenAddresses US state files held before `us/il/cook` was withdrawn.
62
+ *
63
+ * The number of US files therefore leaves the pool's country mix unchanged.
54
64
  */
55
- const SOURCES: readonly CountrySource[] = [
56
- {
57
- zip: dataRootPath("oa-cache", "us__ia__statewide.zip"),
58
- csv: "us/ia/statewide.csv",
59
- iso2: "US",
60
- region: "IA",
61
- order: "us",
62
- },
63
- { zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", iso2: "US", region: "IL", order: "us" },
64
- {
65
- zip: dataRootPath("oa-cache", "us__mt__statewide.zip"),
66
- csv: "us/mt/statewide.csv",
65
+ const SOURCE_GROUPS: readonly (readonly CountrySource[])[] = [
66
+ US_SKELETON_TRAIN_SOURCES.map(({ zip, csv, region, register }) => ({
67
+ zip,
68
+ csv,
67
69
  iso2: "US",
68
- region: "MT",
70
+ region,
69
71
  order: "us",
70
- },
71
- {
72
- zip: dataRootPath("oa-cache", "us__sd__statewide.zip"),
73
- csv: "us/sd/statewide.csv",
74
- iso2: "US",
75
- region: "SD",
76
- order: "us",
77
- },
78
- {
79
- zip: dataRootPath("oa-cache", "de__sn__statewide.zip"),
80
- csv: "de/sn/statewide.csv",
81
- iso2: "DE",
82
- region: "",
83
- order: "eu",
84
- },
85
- {
86
- zip: dataRootPath("oa-cache", "fr__countrywide.zip"),
87
- csv: "fr/countrywide.csv",
88
- iso2: "FR",
89
- region: "",
90
- order: "fr",
91
- },
92
- {
93
- zip: dataRootPath("oa-cache", "it__countrywide.zip"),
94
- csv: "it/countrywide.csv",
95
- iso2: "IT",
96
- region: "",
97
- order: "eu",
98
- },
99
- {
100
- zip: dataRootPath("oa-cache", "nl__countrywide.zip"),
101
- csv: "nl/countrywide.csv",
102
- iso2: "NL",
103
- region: "",
104
- order: "eu",
105
- },
72
+ register,
73
+ })),
74
+ [
75
+ {
76
+ zip: dataRootPath("oa-cache", "de__sn__statewide.zip"),
77
+ csv: "de/sn/statewide.csv",
78
+ iso2: "DE",
79
+ region: "",
80
+ order: "eu",
81
+ },
82
+ {
83
+ zip: dataRootPath("oa-cache", "fr__countrywide.zip"),
84
+ csv: "fr/countrywide.csv",
85
+ iso2: "FR",
86
+ region: "",
87
+ order: "fr",
88
+ },
89
+ {
90
+ zip: dataRootPath("oa-cache", "it__countrywide.zip"),
91
+ csv: "it/countrywide.csv",
92
+ iso2: "IT",
93
+ region: "",
94
+ order: "eu",
95
+ },
96
+ {
97
+ zip: dataRootPath("oa-cache", "nl__countrywide.zip"),
98
+ csv: "nl/countrywide.csv",
99
+ iso2: "NL",
100
+ region: "",
101
+ order: "eu",
102
+ },
103
+ ],
106
104
  ]
107
105
 
108
106
  /**
109
- * The held-out sources that `--golden` reads.
107
+ * The held-out sources that `--golden` reads, one per group.
110
108
  */
111
- const EVAL_SOURCES: readonly CountrySource[] = [
112
- {
113
- zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
114
- csv: "us/vt/statewide.csv",
115
- iso2: "US",
116
- region: "VT",
117
- order: "us",
118
- },
119
- { zip: dataRootPath("oa-cache", "de__berlin.zip"), csv: "de/berlin.csv", iso2: "DE", region: "", order: "eu" },
109
+ const EVAL_SOURCE_GROUPS: readonly (readonly CountrySource[])[] = [
110
+ [
111
+ {
112
+ zip: US_SKELETON_EVAL_SOURCE.zip,
113
+ csv: US_SKELETON_EVAL_SOURCE.csv,
114
+ iso2: "US",
115
+ region: US_SKELETON_EVAL_SOURCE.region,
116
+ order: "us",
117
+ },
118
+ ],
119
+ [{ zip: dataRootPath("oa-cache", "de__berlin.zip"), csv: "de/berlin.csv", iso2: "DE", region: "", order: "eu" }],
120
120
  ]
121
121
 
122
122
  /**
@@ -130,6 +130,7 @@ interface CountryTuple {
130
130
  postcode: string
131
131
  iso2: string
132
132
  order: string
133
+ register?: string
133
134
  }
134
135
 
135
136
  /**
@@ -143,10 +144,37 @@ async function readTuples(source: CountrySource, limit: number): Promise<Country
143
144
  region: row.region || source.region,
144
145
  iso2: source.iso2,
145
146
  order: source.order,
147
+ ...(source.register ? { register: source.register } : {}),
146
148
  }),
147
149
  })
148
150
  }
149
151
 
152
+ /**
153
+ * Reads the tuple pool for `count` rows: a threefold read of the target,
154
+ * which leaves room for skipped rows.
155
+ *
156
+ * Each group takes an equal share of that read, split evenly between its sources.
157
+ */
158
+ async function readSourcePool(groups: readonly (readonly CountrySource[])[], count: number): Promise<CountryTuple[]> {
159
+ const pool: CountryTuple[] = []
160
+
161
+ for (const group of groups) {
162
+ const perSource = Math.ceil((count * 3) / groups.length / group.length)
163
+
164
+ for (const s of group) {
165
+ const t = await readTuples(s, perSource)
166
+
167
+ console.error(` ${s.csv} (${s.iso2}): ${t.length} tuples`)
168
+
169
+ for (const x of t) {
170
+ pool.push(x)
171
+ }
172
+ }
173
+ }
174
+
175
+ return pool
176
+ }
177
+
150
178
  // The share of country draws taken from the curated surface forms instead of canonical names.
151
179
  const SURFACE_FORM_SHARE = 0.6
152
180
 
@@ -391,21 +419,7 @@ export const countryBalancedRecipe: CorpusRecipe = {
391
419
  const count = opts.count
392
420
  const random = makeMulberry32(opts.seed)
393
421
  const source = opts.sourceName ?? defaultRecipeSource("synth-country")
394
- const sources = opts.golden ? EVAL_SOURCES : SOURCES
395
- // A threefold read of the target leaves room for skipped rows.
396
- const perSource = Math.ceil((count * 3) / sources.length)
397
-
398
- const pool: CountryTuple[] = []
399
-
400
- for (const s of sources) {
401
- const t = await readTuples(s, perSource)
402
-
403
- console.error(` ${s.csv} (${s.iso2}): ${t.length} tuples`)
404
-
405
- for (const x of t) {
406
- pool.push(x)
407
- }
408
- }
422
+ const pool = await readSourcePool(opts.golden ? EVAL_SOURCE_GROUPS : SOURCE_GROUPS, count)
409
423
 
410
424
  if (!pool.length) {
411
425
  throw new Error(`No tuples — are the cached OA zips present in ${dataRootPath("oa-cache")}?`)
@@ -420,6 +434,7 @@ export const countryBalancedRecipe: CorpusRecipe = {
420
434
  const roll = random()
421
435
  let rendered: { fmt: string; raw: string; components: Partial<Record<ComponentTag, string>> }
422
436
  let rowISO2: string
437
+ let rowRegister: string | undefined
423
438
 
424
439
  if (roll < HOMOGRAPH_FRAC) {
425
440
  const h = renderHomograph(random)
@@ -434,6 +449,7 @@ export const countryBalancedRecipe: CorpusRecipe = {
434
449
  const country = pickCountry(random)
435
450
  rendered = renderCountry(random, t, country)
436
451
  rowISO2 = t.iso2
452
+ rowRegister = t.register
437
453
 
438
454
  if (country && !rendered.raw.includes(country)) {
439
455
  skipped++
@@ -461,7 +477,8 @@ export const countryBalancedRecipe: CorpusRecipe = {
461
477
  source,
462
478
  source_id: stableSourceID(source, components),
463
479
  corpus_version: "0.4.0",
464
- license: "OpenAddresses multi-locale skeletons + injected ISO-3166 country surface forms (codex)",
480
+ license:
481
+ "OpenAddresses multi-locale and National Address Database US skeletons + injected ISO-3166 country surface forms (codex)",
465
482
  }
466
483
 
467
484
  const aligned = alignRow(canonical)
@@ -477,7 +494,7 @@ export const countryBalancedRecipe: CorpusRecipe = {
477
494
  ...aligned.row,
478
495
  recipe: "country",
479
496
  base_source_id: null,
480
- register: requireRegister(opts, "country-balanced"),
497
+ register: rowRegister ?? requireRegister(opts, "country-balanced"),
481
498
  surface: SurfaceOrigin.Composed,
482
499
  })
483
500
  )
@@ -20,7 +20,7 @@
20
20
  * Junction-format variety includes padded and tight `&` and `/`, `and`, `at`, `@`, and the
21
21
  * leading phrases `corner of` and `intersection of`, crossed with tails (bare / `, ST` / `, ST ZIP` /
22
22
  * `, City, ST [ZIP]`) and case variants. ZIPs are the crossing's own tiger edge zipl (real). The
23
- * locality tail comes from the OA Cook-county ZIP→city majority map.
23
+ * locality tail comes from the NAD ZIP→city majority tables for Cook and Morris counties.
24
24
  *
25
25
  * Audit: every emitted row is label-checked on the RAW surface via the char-offset span triple. Any
26
26
  * violation fails the build (throws). A JSON audit report lands next to the output.
@@ -29,7 +29,8 @@
29
29
  *
30
30
  * - <edges-dir>/tl_2023_{17031,34027,50023}_edges.shp (unzipped tiger 2023 edges. default
31
31
  * `$MAILWOMAN_DATA_ROOT/census/tiger2023-edges`, where `mailwoman situs interpolation` puts them)
32
- * - `$MAILWOMAN_DATA_ROOT/oa-cache/us__il__cook.zip` (ZIP→city tails)
32
+ * - `$MAILWOMAN_DATA_ROOT/nad-skeletons/us__{il__cook,nj__morris}__zip-city.csv` (ZIP→city tails, written by
33
+ * `packages/mailwoman/tools/dev-tools/corpus/nad-skeletons.run.ts`)
33
34
  */
34
35
 
35
36
  import type { DuckDBConnection } from "@duckdb/node-api"
@@ -46,10 +47,11 @@ import { JSONSpliterator } from "spliterator"
46
47
 
47
48
  import { stableSourceID } from "#adapters/source-id"
48
49
  import { openDuckDB } from "#parquet/duckdb"
49
- import { readZippedCSVRecords, type CorpusRecipe } from "#recipes/scaffold"
50
+ import { readCSVRecords, type CorpusRecipe } from "#recipes/scaffold"
50
51
  import { defaultRecipeSource } from "#recipes/sources"
51
52
  import { weightedPick } from "#synthesizers/utils"
52
53
  import type { CanonicalRow, LabeledRow } from "#types"
54
+ import { NAD_ZIP_CITY_COUNTIES, nadZipCityPath } from "#us/recipes/skeleton-sources"
53
55
  import { alignRow } from "#utils"
54
56
 
55
57
  interface County {
@@ -75,8 +77,6 @@ const EVAL_GOLD_PATH = repoRootPath("data", "eval", "external", "intersection-re
75
77
  */
76
78
  const DEFAULT_EDGES_DIR = dataRootPath("census", "tiger2023-edges")
77
79
 
78
- const OA_COOK = { zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv" }
79
-
80
80
  /**
81
81
  * One real crossing extracted from a county's tiger edges shapefile.
82
82
  */
@@ -257,37 +257,28 @@ async function extractCrossings(
257
257
  }
258
258
 
259
259
  /**
260
- * ZIP → majority city from the cached OA Cook-county CSV (real ZIP/city pairings).
260
+ * ZIP → majority city in the training counties, from the NAD ZIP-to-city tables (real ZIP/city pairings).
261
+ *
262
+ * A missing table throws: an empty map would silently downgrade every city tail to the region tail.
261
263
  */
262
264
  async function buildZipCityMap(): Promise<Map<string, string>> {
263
- const counts = new Map<string, Map<string, number>>()
264
-
265
- for await (const row of readZippedCSVRecords(OA_COOK.zip, OA_COOK.csv)) {
266
- const city = row.city ?? ""
267
- const zip = row.postcode ?? ""
265
+ const map = new Map<string, string>()
268
266
 
269
- if (!city || !/^\d{5}$/.test(zip) || BAD_NAME.test(city)) continue
270
- let byCity = counts.get(zip)
267
+ for (const county of NAD_ZIP_CITY_COUNTIES) {
268
+ const path = nadZipCityPath(county)
271
269
 
272
- if (!byCity) {
273
- counts.set(zip, (byCity = new Map()))
270
+ if (!(await pathExists(path))) {
271
+ throw new Error(
272
+ `${path} is missing; write it with packages/mailwoman/tools/dev-tools/corpus/nad-skeletons.run.ts`
273
+ )
274
274
  }
275
275
 
276
- byCity.set(city, (byCity.get(city) ?? 0) + 1)
277
- }
278
-
279
- const map = new Map<string, string>()
280
-
281
- for (const [zip, byCity] of counts) {
282
- let best: { city: string; n: number } | null = null
283
-
284
- for (const [city, n] of byCity)
285
- if (!best || n > best.n) {
286
- best = { city, n }
287
- }
276
+ for await (const row of readCSVRecords(path)) {
277
+ const city = row.city ?? ""
278
+ const zip = row.zip ?? ""
288
279
 
289
- if (best) {
290
- map.set(zip, best.city)
280
+ if (!city || !/^\d{5}$/.test(zip) || BAD_NAME.test(city)) continue
281
+ map.set(zip, city)
291
282
  }
292
283
  }
293
284
 
@@ -571,7 +562,8 @@ export const intersectionRecipe: CorpusRecipe = {
571
562
  source,
572
563
  source_id: stableSourceID(source, components),
573
564
  corpus_version: "0.4.0",
574
- license: "TIGER/Line 2023 EDGES (US Census, public domain) real street pairs; OA Cook IL zip-to-city tails",
565
+ license:
566
+ "TIGER/Line 2023 EDGES (US Census, public domain) real street pairs; National Address Database (public domain) Cook IL and Morris NJ zip-to-city tails",
575
567
  }
576
568
 
577
569
  // Verbatim-only alignment: raw is built from the component values, so a fuzzy fallback