@mailwoman/corpus 10.5.0 → 10.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. package/data/PROVENANCE.md +24 -0
  2. package/data/address-source-register.json +18948 -8757
  3. package/data/corpus-source-inputs.json +4756 -0
  4. package/data/license-decisions.json +7260 -0
  5. package/data/licenses/whosonfirst-licenses.md +3 -2
  6. package/data/reviewed-ve-postcode-tuples.json +5 -72
  7. package/data/source-additions.json +2450 -0
  8. package/data/source-resolutions.json +87 -0
  9. package/data/training-manifests/v0.7.10-gb-county.effective.json +2335 -0
  10. package/data/training-manifests/v0.7.10-gb-county.json +1132 -0
  11. package/data/training-manifests/v0.7.2-address-systems.effective.json +3 -2
  12. package/data/training-manifests/v0.7.7-no-ppd.json +550 -0
  13. package/data/training-manifests/v0.7.8-rights-clean.json +538 -0
  14. package/data/training-manifests/v0.7.9-country-gap.json +1132 -0
  15. package/data/us-situs-sources.json +128 -0
  16. package/lib/adapters/cordis/adapter.ts +1 -1
  17. package/lib/adapters/libpostal-dictionary.ts +23 -3
  18. package/lib/adapters/osm/adapter.ts +42 -11
  19. package/lib/adapters.ts +14 -0
  20. package/lib/az/adapters/opendata/adapter.ts +389 -0
  21. package/lib/bs/adapters/gbpa/adapter.ts +248 -0
  22. package/lib/build/types.ts +16 -0
  23. package/lib/build.ts +21 -0
  24. package/lib/ee/adapters/ads/adapter.ts +18 -0
  25. package/lib/fo/adapters/adressur/adapter.ts +242 -0
  26. package/lib/gb/adapters/os-open/adapter.ts +14 -2
  27. package/lib/gb/county.ts +59 -0
  28. package/lib/gb/os-open/{open-names.ts → open/names.ts} +41 -0
  29. package/lib/gu/adapters/bsp/adapter.ts +434 -0
  30. package/lib/im/adapters/land-registry/adapter.ts +340 -0
  31. package/lib/international/recipes/locale.ts +164 -19
  32. package/lib/li/adapters/llv/adapter.ts +246 -0
  33. package/lib/recipes/bare/postcode.ts +10 -34
  34. package/lib/recipes/country-balanced.ts +95 -78
  35. package/lib/recipes/intersection.ts +22 -30
  36. package/lib/recipes/po/box/cedex/recipe.ts +8 -5
  37. package/lib/recipes/po/box/cedex/sources.ts +2 -23
  38. package/lib/recipes/scaffold.ts +28 -3
  39. package/lib/recipes/sources.ts +112 -1
  40. package/lib/recipes/street/affix.ts +17 -29
  41. package/lib/recipes/sub/venue/context.ts +33 -9
  42. package/lib/recipes/sub/venue/sources.ts +54 -13
  43. package/lib/recipes/sub/venue.ts +68 -23
  44. package/lib/recipes/unit.ts +20 -7
  45. package/lib/registers.ts +33 -0
  46. package/lib/source-register/corpus-inputs.ts +528 -0
  47. package/lib/source-register/effective-manifest.ts +37 -5
  48. package/lib/source-register/types.ts +20 -0
  49. package/lib/south-asia/recipes/register.ts +1 -1
  50. package/lib/tools/overlay/manifest.ts +109 -1
  51. package/lib/tools/source-register/build.ts +47 -1
  52. package/lib/tools/variety-sample.ts +633 -0
  53. package/lib/us/adapters/usgov/nad/adapter.ts +39 -10
  54. package/lib/us/recipes/skeleton-sources.ts +156 -0
  55. package/lib/us/tools/nad-skeletons.ts +395 -0
  56. package/lib/uy/adapters/ide/adapter.ts +275 -0
  57. package/lib/ve/recipes/reviewed-postcode-tail.ts +29 -6
  58. package/out/adapters/cordis/adapter.d.ts +1 -1
  59. package/out/adapters/cordis/adapter.js +1 -1
  60. package/out/adapters/libpostal-dictionary.d.ts +9 -0
  61. package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
  62. package/out/adapters/libpostal-dictionary.js +19 -3
  63. package/out/adapters/libpostal-dictionary.js.map +1 -1
  64. package/out/adapters/osm/adapter.d.ts +12 -1
  65. package/out/adapters/osm/adapter.d.ts.map +1 -1
  66. package/out/adapters/osm/adapter.js +30 -13
  67. package/out/adapters/osm/adapter.js.map +1 -1
  68. package/out/adapters.d.ts.map +1 -1
  69. package/out/adapters.js +14 -0
  70. package/out/adapters.js.map +1 -1
  71. package/out/az/adapters/opendata/adapter.d.ts +88 -0
  72. package/out/az/adapters/opendata/adapter.d.ts.map +1 -0
  73. package/out/az/adapters/opendata/adapter.js +282 -0
  74. package/out/az/adapters/opendata/adapter.js.map +1 -0
  75. package/out/bs/adapters/gbpa/adapter.d.ts +64 -0
  76. package/out/bs/adapters/gbpa/adapter.d.ts.map +1 -0
  77. package/out/bs/adapters/gbpa/adapter.js +187 -0
  78. package/out/bs/adapters/gbpa/adapter.js.map +1 -0
  79. package/out/build/types.d.ts +15 -0
  80. package/out/build/types.d.ts.map +1 -1
  81. package/out/build/types.js.map +1 -1
  82. package/out/build.d.ts.map +1 -1
  83. package/out/build.js +17 -0
  84. package/out/build.js.map +1 -1
  85. package/out/ee/adapters/ads/adapter.d.ts +6 -0
  86. package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
  87. package/out/ee/adapters/ads/adapter.js +16 -0
  88. package/out/ee/adapters/ads/adapter.js.map +1 -1
  89. package/out/fo/adapters/adressur/adapter.d.ts +63 -0
  90. package/out/fo/adapters/adressur/adapter.d.ts.map +1 -0
  91. package/out/fo/adapters/adressur/adapter.js +177 -0
  92. package/out/fo/adapters/adressur/adapter.js.map +1 -0
  93. package/out/gb/adapters/os-open/adapter.d.ts +10 -2
  94. package/out/gb/adapters/os-open/adapter.d.ts.map +1 -1
  95. package/out/gb/adapters/os-open/adapter.js +13 -2
  96. package/out/gb/adapters/os-open/adapter.js.map +1 -1
  97. package/out/gb/county.d.ts +30 -0
  98. package/out/gb/county.d.ts.map +1 -0
  99. package/out/gb/county.js +51 -0
  100. package/out/gb/county.js.map +1 -0
  101. package/out/gb/os-open/{open-names.d.ts → open/names.d.ts} +18 -1
  102. package/out/gb/os-open/open/names.d.ts.map +1 -0
  103. package/out/gb/os-open/{open-names.js → open/names.js} +20 -1
  104. package/out/gb/os-open/open/names.js.map +1 -0
  105. package/out/gb/os-open/{open-roads.d.ts → open/roads.d.ts} +1 -1
  106. package/out/gb/os-open/open/roads.d.ts.map +1 -0
  107. package/out/gb/os-open/{open-roads.js → open/roads.js} +1 -1
  108. package/out/gb/os-open/open/roads.js.map +1 -0
  109. package/out/gu/adapters/bsp/adapter.d.ts +83 -0
  110. package/out/gu/adapters/bsp/adapter.d.ts.map +1 -0
  111. package/out/gu/adapters/bsp/adapter.js +346 -0
  112. package/out/gu/adapters/bsp/adapter.js.map +1 -0
  113. package/out/im/adapters/land-registry/adapter.d.ts +68 -0
  114. package/out/im/adapters/land-registry/adapter.d.ts.map +1 -0
  115. package/out/im/adapters/land-registry/adapter.js +261 -0
  116. package/out/im/adapters/land-registry/adapter.js.map +1 -0
  117. package/out/international/recipes/locale.d.ts +24 -4
  118. package/out/international/recipes/locale.d.ts.map +1 -1
  119. package/out/international/recipes/locale.js +107 -15
  120. package/out/international/recipes/locale.js.map +1 -1
  121. package/out/li/adapters/llv/adapter.d.ts +69 -0
  122. package/out/li/adapters/llv/adapter.d.ts.map +1 -0
  123. package/out/li/adapters/llv/adapter.js +181 -0
  124. package/out/li/adapters/llv/adapter.js.map +1 -0
  125. package/out/recipes/bare/postcode.d.ts.map +1 -1
  126. package/out/recipes/bare/postcode.js +9 -30
  127. package/out/recipes/bare/postcode.js.map +1 -1
  128. package/out/recipes/country-balanced.d.ts.map +1 -1
  129. package/out/recipes/country-balanced.js +82 -73
  130. package/out/recipes/country-balanced.js.map +1 -1
  131. package/out/recipes/intersection.d.ts +3 -2
  132. package/out/recipes/intersection.d.ts.map +1 -1
  133. package/out/recipes/intersection.js +20 -26
  134. package/out/recipes/intersection.js.map +1 -1
  135. package/out/recipes/po/box/cedex/recipe.d.ts.map +1 -1
  136. package/out/recipes/po/box/cedex/recipe.js +5 -4
  137. package/out/recipes/po/box/cedex/recipe.js.map +1 -1
  138. package/out/recipes/po/box/cedex/sources.d.ts +2 -45
  139. package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
  140. package/out/recipes/po/box/cedex/sources.js +0 -20
  141. package/out/recipes/po/box/cedex/sources.js.map +1 -1
  142. package/out/recipes/scaffold.d.ts +15 -2
  143. package/out/recipes/scaffold.d.ts.map +1 -1
  144. package/out/recipes/scaffold.js +21 -1
  145. package/out/recipes/scaffold.js.map +1 -1
  146. package/out/recipes/sources.d.ts.map +1 -1
  147. package/out/recipes/sources.js +112 -1
  148. package/out/recipes/sources.js.map +1 -1
  149. package/out/recipes/street/affix.d.ts.map +1 -1
  150. package/out/recipes/street/affix.js +8 -20
  151. package/out/recipes/street/affix.js.map +1 -1
  152. package/out/recipes/sub/venue/context.d.ts +14 -3
  153. package/out/recipes/sub/venue/context.d.ts.map +1 -1
  154. package/out/recipes/sub/venue/context.js +29 -9
  155. package/out/recipes/sub/venue/context.js.map +1 -1
  156. package/out/recipes/sub/venue/sources.d.ts +41 -8
  157. package/out/recipes/sub/venue/sources.d.ts.map +1 -1
  158. package/out/recipes/sub/venue/sources.js +26 -7
  159. package/out/recipes/sub/venue/sources.js.map +1 -1
  160. package/out/recipes/sub/venue.d.ts.map +1 -1
  161. package/out/recipes/sub/venue.js +35 -14
  162. package/out/recipes/sub/venue.js.map +1 -1
  163. package/out/recipes/unit.d.ts +4 -0
  164. package/out/recipes/unit.d.ts.map +1 -1
  165. package/out/recipes/unit.js +7 -6
  166. package/out/recipes/unit.js.map +1 -1
  167. package/out/registers.d.ts +33 -0
  168. package/out/registers.d.ts.map +1 -1
  169. package/out/registers.js +33 -0
  170. package/out/registers.js.map +1 -1
  171. package/out/source-register/corpus-inputs.d.ts +240 -0
  172. package/out/source-register/corpus-inputs.d.ts.map +1 -0
  173. package/out/source-register/corpus-inputs.js +275 -0
  174. package/out/source-register/corpus-inputs.js.map +1 -0
  175. package/out/source-register/effective-manifest.d.ts +2 -0
  176. package/out/source-register/effective-manifest.d.ts.map +1 -1
  177. package/out/source-register/effective-manifest.js +33 -4
  178. package/out/source-register/effective-manifest.js.map +1 -1
  179. package/out/source-register/types.d.ts +21 -1
  180. package/out/source-register/types.d.ts.map +1 -1
  181. package/out/source-register/types.js +20 -0
  182. package/out/source-register/types.js.map +1 -1
  183. package/out/south-asia/recipes/register.js +1 -1
  184. package/out/south-asia/recipes/register.js.map +1 -1
  185. package/out/tools/overlay/manifest.d.ts +7 -0
  186. package/out/tools/overlay/manifest.d.ts.map +1 -1
  187. package/out/tools/overlay/manifest.js +72 -1
  188. package/out/tools/overlay/manifest.js.map +1 -1
  189. package/out/tools/source-register/build.d.ts +9 -0
  190. package/out/tools/source-register/build.d.ts.map +1 -1
  191. package/out/tools/source-register/build.js +25 -3
  192. package/out/tools/source-register/build.js.map +1 -1
  193. package/out/tools/variety-sample.d.ts +196 -0
  194. package/out/tools/variety-sample.d.ts.map +1 -0
  195. package/out/tools/variety-sample.js +417 -0
  196. package/out/tools/variety-sample.js.map +1 -0
  197. package/out/us/adapters/usgov/nad/adapter.d.ts +80 -0
  198. package/out/us/adapters/usgov/nad/adapter.d.ts.map +1 -1
  199. package/out/us/adapters/usgov/nad/adapter.js +27 -8
  200. package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
  201. package/out/us/recipes/skeleton-sources.d.ts +92 -0
  202. package/out/us/recipes/skeleton-sources.d.ts.map +1 -0
  203. package/out/us/recipes/skeleton-sources.js +121 -0
  204. package/out/us/recipes/skeleton-sources.js.map +1 -0
  205. package/out/us/tools/nad-skeletons.d.ts +120 -0
  206. package/out/us/tools/nad-skeletons.d.ts.map +1 -0
  207. package/out/us/tools/nad-skeletons.js +262 -0
  208. package/out/us/tools/nad-skeletons.js.map +1 -0
  209. package/out/uy/adapters/ide/adapter.d.ts +64 -0
  210. package/out/uy/adapters/ide/adapter.d.ts.map +1 -0
  211. package/out/uy/adapters/ide/adapter.js +206 -0
  212. package/out/uy/adapters/ide/adapter.js.map +1 -0
  213. package/out/ve/recipes/reviewed-postcode-tail.d.ts +6 -2
  214. package/out/ve/recipes/reviewed-postcode-tail.d.ts.map +1 -1
  215. package/out/ve/recipes/reviewed-postcode-tail.js +16 -5
  216. package/out/ve/recipes/reviewed-postcode-tail.js.map +1 -1
  217. package/package.json +14 -14
  218. package/lib/recipes/unit/sources.ts +0 -44
  219. package/lib/tools/fetch/ppd.ts +0 -121
  220. package/out/gb/os-open/open-names.d.ts.map +0 -1
  221. package/out/gb/os-open/open-names.js.map +0 -1
  222. package/out/gb/os-open/open-roads.d.ts.map +0 -1
  223. package/out/gb/os-open/open-roads.js.map +0 -1
  224. package/out/recipes/unit/sources.d.ts +0 -28
  225. package/out/recipes/unit/sources.d.ts.map +0 -1
  226. package/out/recipes/unit/sources.js +0 -30
  227. package/out/recipes/unit/sources.js.map +0 -1
  228. package/out/tools/fetch/ppd.d.ts +0 -15
  229. package/out/tools/fetch/ppd.d.ts.map +0 -1
  230. package/out/tools/fetch/ppd.js +0 -86
  231. package/out/tools/fetch/ppd.js.map +0 -1
  232. /package/lib/gb/os-open/{open-roads.ts → open/roads.ts} +0 -0
@@ -0,0 +1,340 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `landregistry-im`: the property addresses of the Isle of Man Land Registry's land transactions, which the
7
+ * Isle of Man Government publishes on gov.im as one CSV of every registered sale since 2000.
8
+ *
9
+ * Each row is a sale, so one property appears once per sale. The adapter emits each distinct address
10
+ * once, comparing the emitted components case-folded and keeping the first. It reads only the address
11
+ * columns and never the price or date columns.
12
+ *
13
+ * `SubUnit_Name` is the flat or apartment (`Flat 2`, `Apartment 5`), `House_Number` the door number,
14
+ * `House_Name` the house's given name (`Ballacain Farm`) or the building's (`Spectrum Apartments`),
15
+ * `Street_Name` the street, `Locality` the estate or village inside the town, and `Town` the post town.
16
+ * The IM layout prints no region, so `Parish_` is not read.
17
+ *
18
+ * The register also records sales of land that has no postal address. Such a row describes the land
19
+ * (`Field North Of Ballawhannel Farm`, `Parcel Of Land`, `Land Adjacent To Mostyn Ville`) in place of
20
+ * naming a property, and the adapter refuses it, as it refuses a garage, a parking space or a substation
21
+ * sold on its own.
22
+ *
23
+ * The Isle of Man Government publishes the dataset under the Isle of Man Open Government Licence, which
24
+ * requires the attribution statement below. Every row records that license.
25
+ */
26
+
27
+ import { formatAddressRow } from "@mailwoman/codex/address/format"
28
+ import { matchCountryPostcode } from "@mailwoman/codex/postcode/country-formats"
29
+ import { readLocalBuffer } from "@mailwoman/core/fs/readers"
30
+ import { resolvePath } from "path-ts"
31
+ import { CSVSpliterator } from "spliterator"
32
+
33
+ import { UnsupportedCountryError } from "#adapters/errors"
34
+ import { stableSourceIDFromParts } from "#adapters/source-id"
35
+ import { SourceRegister } from "#registers"
36
+ import {
37
+ AddressRole,
38
+ type AdapterOptions,
39
+ type CanonicalRow,
40
+ type CorpusAdapter,
41
+ countDropped,
42
+ SurfaceOrigin,
43
+ } from "#types"
44
+
45
+ /**
46
+ * Registry id for this adapter, stamped into every row it emits.
47
+ */
48
+ export const LANDREGISTRY_IM_ADAPTER_ID = "landregistry-im"
49
+
50
+ /**
51
+ * The Isle of Man Open Government Licence, whose version the dataset page does not state.
52
+ * It has no SPDX identifier.
53
+ */
54
+ export const LANDREGISTRY_IM_LICENSE = "LicenseRef-IM-OGL"
55
+
56
+ /**
57
+ * The attribution statement the license requires.
58
+ */
59
+ export const LANDREGISTRY_IM_ATTRIBUTION =
60
+ "Contains public sector information licensed under the Isle of Man Open Government Licence"
61
+
62
+ /**
63
+ * Every jurisdiction this adapter emits, checked against a caller's `--country`.
64
+ */
65
+ export const LANDREGISTRY_IM_COUNTRIES: readonly string[] = ["IM"]
66
+
67
+ /**
68
+ * The address columns the adapter reads from the land transactions CSV.
69
+ * The price and date columns are not read.
70
+ */
71
+ export interface IsleOfManLandTransactionRecord {
72
+ SubUnit_Name: string
73
+ House_Number: string
74
+ House_Name: string
75
+ Street_Name: string
76
+ Locality: string
77
+ Town: string
78
+ Postcode: string
79
+ }
80
+
81
+ function present(value: string | undefined): string | null {
82
+ const trimmed = value?.replaceAll(/\s+/gu, " ").trim() ?? ""
83
+
84
+ return trimmed || null
85
+ }
86
+
87
+ /**
88
+ * A value that describes a piece of land rather than naming a property: it opens with the kind
89
+ * of land (`Land`, `Plot Of Land`, `Field Number 111277`, `Part Of The Estate`, `25.2 Acres Of Land`)
90
+ * or places it against another property (`Abutting`, `Adjacent To`, `Rear Of`, `Known As`).
91
+ */
92
+ const LAND_DESCRIPTION =
93
+ /^(?:(?:a|the)\s+)?(?:lands?|plots?|parcels?|fields?|part|strip|portion|[\d.]+\s+acres?)\b|\b(?:abutting|adjacent|adjoining|adjoins|rear\s+off?|part\s+of|known\s+as|formerly|situate|being\s+part|close\s+to|near|nr)\b/iu
94
+
95
+ /**
96
+ * In a flat or house name, a land word anywhere marks a description
97
+ * (`Riverside Cottage And Land`, `Two Fields At`, `Agricultural Land`),
98
+ * where in a street or locality it may be part of the name.
99
+ */
100
+ const LAND_WORD = /\b(?:lands?|fields?|plots?|acres?)\b/iu
101
+
102
+ /**
103
+ * A value that describes a structure or a share sold apart from any dwelling,
104
+ * which has no postal address of its own.
105
+ */
106
+ const NOT_A_PREMISE =
107
+ /^(?:(?:a|the)\s+)?(?:garages?|lock[- ]?ups?|stores?|car\s+park(?:ing)?|forecourt|roadway|access\s+roads?|lane)\b|\b(?:site|substation|reservoir|parking|driveway|pumping\s+station|private\s+(?:roadway|road|drive|lane|access)|(?:common|shared|landscaped)\s+areas?|share\s+(?:in|of))\b/iu
108
+
109
+ /**
110
+ * A house name that joins two properties (`Gorse Cottage And Willow Cottage`, `"Hawarden" And "Far End"`)
111
+ * records a sale of several premises at once.
112
+ */
113
+ const SEVERAL_PROPERTIES = /\band\b|&/iu
114
+
115
+ /**
116
+ * A house name that opens with a door number (`15 Godred Court`) holds a number
117
+ * and a secondary street rather than a property name.
118
+ */
119
+ const LEADING_NUMBER = /^\d/u
120
+
121
+ /**
122
+ * The flat designators `SubUnit_Name` holds that a single dwelling is written with.
123
+ */
124
+ const UNIT_DESIGNATOR =
125
+ /^(?:(?:flat|apartment|apt\.?|unit|penthouse|suite|studio|maisonette)\s*[A-Z]?\d+[A-Z]?|(?:garden|basement|ground\s+floor|first\s+floor|top\s+floor|upper|lower)\s+(?:flat|apartment))$/iu
126
+
127
+ /**
128
+ * A door number, with an optional letter, or a range of two.
129
+ */
130
+ const HOUSE_NUMBER = /^\d+[A-Z]?(?:\s*-\s*\d+[A-Z]?)?$/iu
131
+
132
+ /**
133
+ * A second-line value that is a thoroughfare (`Main Road`, `Off Clifton Drive`) rather than a place.
134
+ */
135
+ const THOROUGHFARE =
136
+ /^off\b|\b(?:road|street|lane|avenue|drive|promenade|terrace|crescent|quay|way|close|parade|square)$/iu
137
+
138
+ /**
139
+ * Maps one record onto the components the corpus asserts, or returns null when the row is refused.
140
+ *
141
+ * Each refusal and each dropped component increments its reason in `dropped`.
142
+ */
143
+ export function componentsForIsleOfManLandTransactionRecord(
144
+ record: IsleOfManLandTransactionRecord,
145
+ dropped?: Pick<AdapterOptions, "dropped">
146
+ ): CanonicalRow["components"] | null {
147
+ const tally = dropped ?? {}
148
+ const subunit = present(record.SubUnit_Name)
149
+ const name = present(record.House_Name)
150
+ const street = present(record.Street_Name)
151
+ const dependent = present(record.Locality)
152
+
153
+ if (
154
+ [subunit, name, street, dependent].some((value) => value && LAND_DESCRIPTION.test(value)) ||
155
+ [subunit, name].some((value) => value && LAND_WORD.test(value))
156
+ ) {
157
+ countDropped(tally, "row:land-parcel-description")
158
+
159
+ return null
160
+ }
161
+
162
+ if ([subunit, name].some((value) => value && NOT_A_PREMISE.test(value))) {
163
+ countDropped(tally, "row:not-a-premise")
164
+
165
+ return null
166
+ }
167
+
168
+ if (name && SEVERAL_PROPERTIES.test(name)) {
169
+ countDropped(tally, "row:several-properties")
170
+
171
+ return null
172
+ }
173
+
174
+ if (!street) {
175
+ countDropped(tally, "row:no-street")
176
+
177
+ return null
178
+ }
179
+
180
+ const components: CanonicalRow["components"] = { street }
181
+ const number = present(record.House_Number)
182
+
183
+ if (number && HOUSE_NUMBER.test(number)) {
184
+ components.house_number = number.replaceAll(/\s*-\s*/gu, "-")
185
+ } else if (number) {
186
+ // `9/11`, `3 & 5` and `Unit 4` name several doors or a unit rather than one door number.
187
+ countDropped(tally, "component:house_number:not-one-number")
188
+ }
189
+
190
+ if (subunit && UNIT_DESIGNATOR.test(subunit)) {
191
+ components.unit = subunit
192
+ } else if (subunit) {
193
+ // `Apartment 4 and Garage 4` and a bare `7` do not name one dwelling the layout can print.
194
+ countDropped(tally, "component:unit:not-a-unit-designator")
195
+ }
196
+
197
+ if (name && UNIT_DESIGNATOR.test(name)) {
198
+ // A flat recorded in the house-name column (`Apartment 9`) is the unit when the unit column is empty.
199
+ if (components.unit) {
200
+ countDropped(tally, "component:venue:unit-designator")
201
+ } else {
202
+ components.unit = name
203
+ }
204
+ } else if (name && LEADING_NUMBER.test(name)) {
205
+ countDropped(tally, "component:venue:holds-a-number")
206
+ } else if (name && /\p{L}{3}/u.test(name)) {
207
+ components.venue = name
208
+ } else if (name) {
209
+ countDropped(tally, "component:venue:code")
210
+ }
211
+
212
+ if (!components.house_number && !components.venue) {
213
+ countDropped(tally, "row:no-house-number-or-name")
214
+
215
+ return null
216
+ }
217
+
218
+ const town = present(record.Town)
219
+
220
+ if (town) {
221
+ components.locality = town
222
+ }
223
+
224
+ if (dependent && THOROUGHFARE.test(dependent)) {
225
+ countDropped(tally, "component:dependent_locality:thoroughfare")
226
+ } else if (dependent && dependent.toLowerCase() === town?.toLowerCase()) {
227
+ countDropped(tally, "component:dependent_locality:repeats-locality")
228
+ } else if (dependent) {
229
+ components.dependent_locality = dependent
230
+ }
231
+
232
+ const postcode = present(record.Postcode)?.toUpperCase()
233
+ const matched = postcode ? matchCountryPostcode(postcode, "IM") : null
234
+
235
+ if (matched) {
236
+ components.postcode = matched
237
+ } else if (postcode) {
238
+ countDropped(tally, "component:postcode:not-country-format")
239
+ }
240
+
241
+ if (!components.locality && !components.postcode) {
242
+ countDropped(tally, "row:no-locality-or-postcode")
243
+
244
+ return null
245
+ }
246
+
247
+ return components
248
+ }
249
+
250
+ /**
251
+ * The case-folded key two sales of one property share.
252
+ */
253
+ function addressKey(components: CanonicalRow["components"]): string {
254
+ return Object.keys(components)
255
+ .toSorted()
256
+ .map((tag) => `${tag}=${components[tag as keyof typeof components]?.toLowerCase() ?? ""}`)
257
+ .join("\u001F")
258
+ }
259
+
260
+ export function createIsleOfManLandRegistryAdapter(): CorpusAdapter {
261
+ return {
262
+ id: LANDREGISTRY_IM_ADAPTER_ID,
263
+ defaultLicense: LANDREGISTRY_IM_LICENSE,
264
+ addressRole: AddressRole.Premise,
265
+ register: SourceRegister.IsleOfManLandTransactions,
266
+ surface: SurfaceOrigin.Rendered,
267
+ description: "Isle of Man Land Registry land transactions (gov.im): the addresses of registered property sales.",
268
+
269
+ async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
270
+ if (opts.country && !LANDREGISTRY_IM_COUNTRIES.includes(opts.country)) {
271
+ throw new UnsupportedCountryError(LANDREGISTRY_IM_ADAPTER_ID, LANDREGISTRY_IM_COUNTRIES, opts.country)
272
+ }
273
+
274
+ // The file is 4 MB, so it is decoded whole.
275
+ // The BOM would otherwise prefix the first header.
276
+ const text = (await readLocalBuffer(resolvePath(opts.inputPath))).toString("utf8").replace(/^/u, "")
277
+
278
+ const records = CSVSpliterator.from<IsleOfManLandTransactionRecord>(Buffer.from(text, "utf8"), {
279
+ normalizeKeys: false,
280
+ })
281
+
282
+ const seen = new Set<string>()
283
+ let emitted = 0
284
+
285
+ for (const record of records) {
286
+ if (opts.signal?.aborted) return
287
+
288
+ if (opts.limit !== undefined && emitted >= opts.limit) return
289
+
290
+ const components = componentsForIsleOfManLandTransactionRecord(record, opts)
291
+
292
+ if (!components) continue
293
+
294
+ const key = addressKey(components)
295
+
296
+ if (seen.has(key)) {
297
+ countDropped(opts, "row:repeat-sale")
298
+
299
+ continue
300
+ }
301
+
302
+ seen.add(key)
303
+
304
+ const rendered = formatAddressRow(components, "IM", { singleLine: true })
305
+
306
+ if (!rendered) {
307
+ countDropped(opts, "row:render-failed")
308
+
309
+ continue
310
+ }
311
+
312
+ for (const tag of rendered.unplaced) {
313
+ countDropped(opts, `component:${tag}:not-rendered`)
314
+ }
315
+
316
+ const { raw, components: aligned } = rendered
317
+
318
+ yield {
319
+ raw,
320
+ components: aligned,
321
+ country: "IM",
322
+ locale: "en-IM",
323
+ source: LANDREGISTRY_IM_ADAPTER_ID,
324
+ // The file carries no record id or coordinates, and the dedupe above leaves
325
+ // one row per distinct address, so the components identify the row.
326
+ source_id: stableSourceIDFromParts(LANDREGISTRY_IM_ADAPTER_ID, aligned),
327
+ corpus_version: "",
328
+ license: LANDREGISTRY_IM_LICENSE,
329
+ }
330
+
331
+ emitted++
332
+ }
333
+ },
334
+ }
335
+ }
336
+
337
+ /**
338
+ * The configured adapter instance registered with the corpus builder.
339
+ */
340
+ export const isleOfManLandRegistryAdapter = createIsleOfManLandRegistryAdapter()
@@ -9,6 +9,7 @@
9
9
 
10
10
  import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country"
11
11
  import { dataRootPath } from "@mailwoman/core/data-root"
12
+ import { pathExists } from "@mailwoman/core/fs/readers/stat"
12
13
  import { openReadStream } from "@mailwoman/core/fs/streams"
13
14
  import { readZipEntry } from "@mailwoman/core/fs/zip"
14
15
  import { stringifyJSON } from "@mailwoman/core/json"
@@ -16,10 +17,12 @@ import { isPresent } from "@mailwoman/core/objects"
16
17
  import { sample } from "@mailwoman/core/random"
17
18
  import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
18
19
  import type { PathBuilderLike } from "path-ts"
19
- import { CSVSpliterator } from "spliterator"
20
+ import { CSVSpliterator, JSONSpliterator } from "spliterator"
20
21
 
21
22
  import { stableSourceID } from "#adapters/source-id"
23
+ import { GB_OS_OPEN_ADAPTER_ID, gbOSOpenCanonicalRowsPath } from "#gb/adapters/os-open/adapter"
22
24
  import { openConstituentCountryReader } from "#gb/constituent-country"
25
+ import { readCountyOfPostcode } from "#gb/county"
23
26
  import type { CorpusRecipe } from "#recipes/scaffold"
24
27
  import { defaultRecipeSource } from "#recipes/sources"
25
28
  import { SourceRegister } from "#registers"
@@ -28,12 +31,26 @@ import { SurfaceOrigin } from "#types"
28
31
  import { alignRow } from "#utils"
29
32
 
30
33
  /**
31
- * One source file: either a CSV member of a ZIP (`zip` and `csv`) or an extracted CSV (`path`).
34
+ * One source file: a CSV member of a ZIP (`zip` and `csv`), an extracted CSV (`path`),
35
+ * or an adapter's canonical rows (`canonicalRows`).
32
36
  */
33
37
  export interface LocalePart {
34
38
  zip?: PathBuilderLike
35
39
  csv?: string
36
40
  path?: PathBuilderLike
41
+ /**
42
+ * An adapter's `canonical.jsonl`, read in place of a CSV.
43
+ *
44
+ * Each row's `components` supplies the house number, street, locality and postcode.
45
+ * Every row must carry {@linkcode LocalePart.canonicalSource} as its `source`.
46
+ */
47
+ canonicalRows?: PathBuilderLike
48
+ /**
49
+ * The `source` every row of {@linkcode LocalePart.canonicalRows} must carry.
50
+ *
51
+ * A row from any other source throws, so a file another adapter wrote is never read as this one.
52
+ */
53
+ canonicalSource?: string
37
54
  region?: string
38
55
  /**
39
56
  * Maps the district column to locality and the city column to dependent locality.
@@ -51,9 +68,32 @@ export interface LocalePart {
51
68
  /**
52
69
  * Opens a reader that answers the row's region from its postcode, replacing the region column.
53
70
  *
54
- * A GB part answers the constituent country this way, and its county column is not read.
71
+ * A GB part answers the county or the constituent country this way.
55
72
  */
56
- regionOfPostcode?: () => (postcode: string) => string | null
73
+ regionOfPostcode?: () => Promise<(postcode: string) => string | null>
74
+ }
75
+
76
+ /**
77
+ * The share of GB postcodes whose rows label the county `region`, out of five.
78
+ *
79
+ * The other postcodes' rows label the constituent country, which Who's On First
80
+ * records as a `macroregion` under the United Kingdom.
81
+ */
82
+ const GB_COUNTY_REGION_FIFTHS = 4
83
+
84
+ /**
85
+ * Whether a GB postcode's rows label the county rather than the constituent country.
86
+ *
87
+ * The choice is a function of the postcode alone, so a rebuild makes the same choice for every row.
88
+ */
89
+ function gbPostcodeTakesCounty(postcode: string): boolean {
90
+ let sum = 0
91
+
92
+ for (const char of postcode) {
93
+ sum += char.codePointAt(0)!
94
+ }
95
+
96
+ return sum % 5 < GB_COUNTY_REGION_FIFTHS
57
97
  }
58
98
 
59
99
  /**
@@ -70,6 +110,12 @@ export interface LocaleCountrySource {
70
110
  * The corpus version written to each row.
71
111
  */
72
112
  corpusVersion: string
113
+ /**
114
+ * The dataset and terms the row's `license` names, before the rendering order.
115
+ *
116
+ * @defaultValue `OpenAddresses <CC> tuples`
117
+ */
118
+ license?: string
73
119
  /**
74
120
  * Replacement parts used when `--district-as-locality` is passed as true.
75
121
  */
@@ -161,19 +207,25 @@ const COUNTRY_SOURCES: Record<string, LocaleCountrySource> = {
161
207
  parts: [{ path: dataRootPath("openaddresses", "extracted", "nz", "countrywide.csv"), districtAsLocality: true }],
162
208
  },
163
209
  GB: {
164
- // Price Paid Data stores the postal town in `district` and an optional dependent locality in `city`.
165
- source: defaultRecipeSource("synth-gb"),
166
- register: SourceRegister.LandRegistryPricePaid,
210
+ // The `gb-os-open` adapter's rows: Code-Point Open postcodes joined to OS Open Roads and OS Open Names.
211
+ // `rendered-gb` read HM Land Registry Price Paid Data, whose address fields
212
+ // Royal Mail and Ordnance Survey license for non-commercial use only (#2635),
213
+ // so GB rows carry a source name of their own.
214
+ source: defaultRecipeSource("synth-gb-os-open"),
215
+ register: SourceRegister.OrdnanceSurveyOpenData,
167
216
  corpusVersion: "0.9.9",
168
- // The `REGION` column holds a county, so the constituent country is read from NSUL by postcode instead.
217
+ license: "OS OpenData GB tuples (OGL-UK-3.0)",
218
+ // The adapter's rows carry no region.
219
+ // The county comes from OS Open Names and the constituent country from NSUL, both by postcode.
169
220
  parts: [
170
221
  {
171
- path: dataRootPath("ppd", "2026-07-22", "gb-tuples.csv"),
172
- districtAsLocality: true,
173
- regionOfPostcode: () => {
222
+ canonicalRows: gbOSOpenCanonicalRowsPath(),
223
+ canonicalSource: GB_OS_OPEN_ADAPTER_ID,
224
+ regionOfPostcode: async () => {
174
225
  const countryOf = openConstituentCountryReader()
226
+ const countyOf = await readCountyOfPostcode()
175
227
 
176
- return (postcode) => countryOf(postcode).name
228
+ return (postcode) => (gbPostcodeTakesCounty(postcode) && countyOf.get(postcode)) || countryOf(postcode).name
177
229
  },
178
230
  },
179
231
  ],
@@ -224,13 +276,102 @@ interface ColumnIndex {
224
276
  tipoVial: number
225
277
  }
226
278
 
279
+ /**
280
+ * One row of an adapter's `canonical.jsonl`, as far as the locale recipe reads it.
281
+ */
282
+ interface CanonicalTupleRow {
283
+ source?: unknown
284
+ components?: Partial<Record<"house_number" | "street" | "locality" | "postcode", unknown>>
285
+ }
286
+
287
+ /**
288
+ * Reservoir-samples an adapter's canonical rows with Algorithm R, using `rng` for reproducibility.
289
+ *
290
+ * A row without a street or locality is skipped.
291
+ * A missing file throws rather than reading as an empty part.
292
+ *
293
+ * @throws When a row's `source` differs from `part.canonicalSource`.
294
+ */
295
+ async function readCanonicalTuples(
296
+ part: LocalePart,
297
+ rows: PathBuilderLike,
298
+ rng: () => number
299
+ ): Promise<LocaleBaseTuple[]> {
300
+ if (!(await pathExists(rows))) {
301
+ throw new Error(
302
+ `${rows.toString()} is absent. Write it with \`mailwoman corpus run ${part.canonicalSource ?? "<adapter>"}\`.`
303
+ )
304
+ }
305
+
306
+ const reservoir: LocaleBaseTuple[] = []
307
+ const regionOf = await part.regionOfPostcode?.()
308
+ let seen = 0
309
+ let regionUnknown = 0
310
+
311
+ for await (const row of JSONSpliterator.fromAsync<CanonicalTupleRow>(rows)) {
312
+ if (part.canonicalSource && row.source !== part.canonicalSource) {
313
+ throw new Error(
314
+ `${rows.toString()} holds a row from source ${stringifyJSON(row.source)}; this part reads ${part.canonicalSource} rows only.`
315
+ )
316
+ }
317
+
318
+ const components = row.components ?? {}
319
+ const text = (value: unknown): string => (typeof value === "string" ? value.trim() : "")
320
+ const street = text(components.street)
321
+ const locality = text(components.locality)
322
+
323
+ if (!street || !locality) continue
324
+
325
+ const postcode = text(components.postcode)
326
+ let region = part.region ?? ""
327
+
328
+ if (regionOf) {
329
+ region = (postcode && regionOf(postcode)) || ""
330
+
331
+ if (!region) {
332
+ regionUnknown++
333
+ }
334
+ }
335
+
336
+ const tuple: LocaleBaseTuple = {
337
+ house_number: text(components.house_number),
338
+ street,
339
+ locality,
340
+ region,
341
+ postcode,
342
+ }
343
+
344
+ seen++
345
+
346
+ if (reservoir.length < RESERVOIR_CAP) {
347
+ reservoir.push(tuple)
348
+ } else {
349
+ const j = Math.floor(rng() * seen)
350
+
351
+ if (j < RESERVOIR_CAP) {
352
+ reservoir[j] = tuple
353
+ }
354
+ }
355
+ }
356
+
357
+ console.error(` ${rows.toString()}: ${reservoir.length} sampled of ${seen} rows`)
358
+
359
+ if (regionOf) {
360
+ console.error(` ${rows.toString()}: ${regionUnknown} rows have no region for their postcode`)
361
+ }
362
+
363
+ return reservoir
364
+ }
365
+
227
366
  /**
228
367
  * Streams tuples from one part and reservoir-samples them with Algorithm R,
229
368
  * using `rng` for reproducibility.
230
369
  *
231
- * A read failure logs a warning and returns an empty list.
370
+ * @throws When the part cannot be read, naming the file.
232
371
  */
233
372
  export async function readTuples(part: LocalePart, rng: () => number): Promise<LocaleBaseTuple[]> {
373
+ if (part.canonicalRows) return readCanonicalTuples(part, part.canonicalRows, rng)
374
+
234
375
  // The stream stays as bytes because CSVSpliterator decodes UTF-8 itself.
235
376
  const input: NodeJS.ReadableStream | AsyncIterable<Uint8Array> = part.path
236
377
  ? openReadStream(part.path)
@@ -242,7 +383,7 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
242
383
  let header: string[] | null = null
243
384
  let seen = 0
244
385
  let dropped = 0
245
- const regionOf = part.regionOfPostcode?.()
386
+ const regionOf = await part.regionOfPostcode?.()
246
387
  let regionUnknown = 0
247
388
 
248
389
  try {
@@ -358,9 +499,12 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
358
499
  }
359
500
  }
360
501
  } catch (error) {
361
- console.error(` WARN: read failed for ${part.path ?? part.zip}: ${(error as Error).message}`)
362
-
363
- return []
502
+ // An unreadable part throws rather than reading as an empty one.
503
+ // The sub-venue recipe's DE leg once emitted zero rows behind a warning
504
+ // when `europe.zip` left the data root.
505
+ throw new Error(`${(part.path ?? part.zip)?.toString()} could not be read: ${(error as Error).message}`, {
506
+ cause: error,
507
+ })
364
508
  }
365
509
 
366
510
  console.error(` ${part.path ?? part.csv}: ${reservoir.length} sampled of ${seen} rows (${dropped} city-noise drops)`)
@@ -421,7 +565,8 @@ export function resolveLocaleParts(countrySource: LocaleCountrySource, override:
421
565
  */
422
566
  export const localeRecipe: CorpusRecipe = {
423
567
  name: "locale",
424
- description: "Per-locale coverage rows (DE/FR/NL/IT/ES/NZ/GB) from real OA tuples, both orders → renderLocaleRow",
568
+ description:
569
+ "Per-locale coverage rows (DE/FR/NL/IT/ES/NZ/GB) from real OA tuples (OS OpenData for GB), both orders → renderLocaleRow",
425
570
  mode: "generate",
426
571
  options: [
427
572
  { flag: "--country <cc>", description: "Target country (DE|FR|NL|IT|ES|NZ|GB). Default DE" },
@@ -562,7 +707,7 @@ export const localeRecipe: CorpusRecipe = {
562
707
  source,
563
708
  source_id: sourceID,
564
709
  corpus_version: countrySource.corpusVersion,
565
- license: `OpenAddresses ${country} tuples, rendered ${order}-order — see ingest SOURCES`,
710
+ license: `${countrySource.license ?? `OpenAddresses ${country} tuples`}, rendered ${order}-order — see ingest SOURCES`,
566
711
  }
567
712
 
568
713
  const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])