@mailwoman/corpus 9.0.0 → 9.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (524) hide show
  1. package/README.md +3 -3
  2. package/data/PROVENANCE.md +227 -0
  3. package/data/reviewed-ve-postcode-tuples.json +74 -0
  4. package/data/sub-venue-lexicon.json +18269 -0
  5. package/out/src/adapters/ban/adapter.d.ts +1 -1
  6. package/out/src/adapters/ban/adapter.d.ts.map +1 -1
  7. package/out/src/adapters/ban/adapter.js +5 -4
  8. package/out/src/adapters/ban/adapter.js.map +1 -1
  9. package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -1
  10. package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
  11. package/out/src/adapters/fcc-bdc/adapter.js +3 -3
  12. package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
  13. package/out/src/adapters/geonames/adapter.d.ts +1 -1
  14. package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
  15. package/out/src/adapters/geonames/adapter.js +2 -2
  16. package/out/src/adapters/geonames/adapter.js.map +1 -1
  17. package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
  18. package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
  19. package/out/src/adapters/geonames-postal/adapter.js +3 -3
  20. package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
  21. package/out/src/adapters/gnaf/adapter.d.ts +1 -1
  22. package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
  23. package/out/src/adapters/gnaf/adapter.js +2 -2
  24. package/out/src/adapters/gnaf/adapter.js.map +1 -1
  25. package/out/src/adapters/gnaf/assemble.js +1 -1
  26. package/out/src/adapters/gnaf/assemble.js.map +1 -1
  27. package/out/src/adapters/index.d.ts +1 -1
  28. package/out/src/adapters/index.d.ts.map +1 -1
  29. package/out/src/adapters/index.js +1 -1
  30. package/out/src/adapters/index.js.map +1 -1
  31. package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
  32. package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
  33. package/out/src/adapters/openaddresses/adapter.js +3 -3
  34. package/out/src/adapters/openaddresses/adapter.js.map +1 -1
  35. package/out/src/adapters/overture/adapter.d.ts +1 -1
  36. package/out/src/adapters/overture/adapter.d.ts.map +1 -1
  37. package/out/src/adapters/overture/adapter.js +2 -2
  38. package/out/src/adapters/overture/adapter.js.map +1 -1
  39. package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
  40. package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
  41. package/out/src/adapters/state-hi-schools/adapter.js +7 -6
  42. package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
  43. package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
  44. package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
  45. package/out/src/adapters/state-ia-contractors/adapter.js +12 -7
  46. package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
  47. package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
  48. package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
  49. package/out/src/adapters/state-ny-notaries/adapter.js +12 -18
  50. package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
  51. package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
  52. package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
  53. package/out/src/adapters/state-tx-notaries/adapter.js +12 -7
  54. package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
  55. package/out/src/adapters/synth-po-box/adapter.d.ts +3 -3
  56. package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
  57. package/out/src/adapters/synth-po-box/adapter.js +3 -3
  58. package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
  59. package/out/src/adapters/tiger/adapter.d.ts +1 -1
  60. package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
  61. package/out/src/adapters/tiger/adapter.js +2 -2
  62. package/out/src/adapters/tiger/adapter.js.map +1 -1
  63. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +1 -1
  64. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
  65. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +7 -6
  66. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
  67. package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
  68. package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
  69. package/out/src/adapters/usgov-imls-pls/adapter.js +11 -7
  70. package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
  71. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
  72. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
  73. package/out/src/adapters/usgov-irs-bmf/adapter.js +6 -5
  74. package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
  75. package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
  76. package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
  77. package/out/src/adapters/usgov-nad/adapter.js +8 -8
  78. package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
  79. package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
  80. package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
  81. package/out/src/adapters/usgov-nppes/adapter.js +10 -10
  82. package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
  83. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
  84. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
  85. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +7 -6
  86. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
  87. package/out/src/{adapter.d.ts → adapters/utils/index.d.ts} +8 -2
  88. package/out/src/adapters/utils/index.d.ts.map +1 -0
  89. package/out/src/{adapter.js → adapters/utils/index.js} +11 -3
  90. package/out/src/adapters/utils/index.js.map +1 -0
  91. package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
  92. package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
  93. package/out/src/adapters/wof-admin-jp/adapter.js +5 -5
  94. package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
  95. package/out/src/adapters/wof-admin-json/adapter.d.ts +4 -15
  96. package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
  97. package/out/src/adapters/wof-admin-json/adapter.js +11 -35
  98. package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
  99. package/out/src/adapters/wof-json-rows.d.ts +32 -0
  100. package/out/src/adapters/wof-json-rows.d.ts.map +1 -0
  101. package/out/src/adapters/wof-json-rows.js +45 -0
  102. package/out/src/adapters/wof-json-rows.js.map +1 -0
  103. package/out/src/adapters/wof-postalcode-json/adapter.d.ts +4 -9
  104. package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
  105. package/out/src/adapters/wof-postalcode-json/adapter.js +12 -37
  106. package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
  107. package/out/src/build.d.ts +4 -4
  108. package/out/src/build.d.ts.map +1 -1
  109. package/out/src/build.js +7 -7
  110. package/out/src/build.js.map +1 -1
  111. package/out/src/index.d.ts +3 -17
  112. package/out/src/index.d.ts.map +1 -1
  113. package/out/src/index.js +3 -17
  114. package/out/src/index.js.map +1 -1
  115. package/out/src/name-prone-us-suffixes.d.ts +12 -0
  116. package/out/src/name-prone-us-suffixes.d.ts.map +1 -0
  117. package/out/src/name-prone-us-suffixes.js +12 -0
  118. package/out/src/name-prone-us-suffixes.js.map +1 -0
  119. package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
  120. package/out/src/parquet-wrapper/reader.js +5 -0
  121. package/out/src/parquet-wrapper/reader.js.map +1 -1
  122. package/out/src/runner.d.ts +2 -2
  123. package/out/src/runner.d.ts.map +1 -1
  124. package/out/src/runner.js +1 -1
  125. package/out/src/runner.js.map +1 -1
  126. package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
  127. package/out/src/shard-recipes/anchor-absorption.js +4 -5
  128. package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
  129. package/out/src/shard-recipes/bare-country.d.ts +25 -0
  130. package/out/src/shard-recipes/bare-country.d.ts.map +1 -0
  131. package/out/src/shard-recipes/bare-country.js +87 -0
  132. package/out/src/shard-recipes/bare-country.js.map +1 -0
  133. package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
  134. package/out/src/shard-recipes/boundary-stress.js +2 -2
  135. package/out/src/shard-recipes/boundary-stress.js.map +1 -1
  136. package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
  137. package/out/src/shard-recipes/country-balanced.js +14 -21
  138. package/out/src/shard-recipes/country-balanced.js.map +1 -1
  139. package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
  140. package/out/src/shard-recipes/cz-pcfirst-preposition.js +28 -10
  141. package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
  142. package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
  143. package/out/src/shard-recipes/fr-admin-split.js +4 -3
  144. package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
  145. package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
  146. package/out/src/shard-recipes/fr-bare-street.js +72 -11
  147. package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
  148. package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
  149. package/out/src/shard-recipes/fr-lieudit.js +4 -3
  150. package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
  151. package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
  152. package/out/src/shard-recipes/fr-order.js +11 -20
  153. package/out/src/shard-recipes/fr-order.js.map +1 -1
  154. package/out/src/shard-recipes/german.d.ts.map +1 -1
  155. package/out/src/shard-recipes/german.js +8 -14
  156. package/out/src/shard-recipes/german.js.map +1 -1
  157. package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
  158. package/out/src/shard-recipes/house-venue.js +1 -1
  159. package/out/src/shard-recipes/house-venue.js.map +1 -1
  160. package/out/src/shard-recipes/index.d.ts.map +1 -1
  161. package/out/src/shard-recipes/index.js +8 -1
  162. package/out/src/shard-recipes/index.js.map +1 -1
  163. package/out/src/shard-recipes/intersection.d.ts.map +1 -1
  164. package/out/src/shard-recipes/intersection.js +7 -13
  165. package/out/src/shard-recipes/intersection.js.map +1 -1
  166. package/out/src/shard-recipes/locale.d.ts +5 -5
  167. package/out/src/shard-recipes/locale.d.ts.map +1 -1
  168. package/out/src/shard-recipes/locale.js +13 -20
  169. package/out/src/shard-recipes/locale.js.map +1 -1
  170. package/out/src/shard-recipes/no-street.d.ts.map +1 -1
  171. package/out/src/shard-recipes/no-street.js +1 -1
  172. package/out/src/shard-recipes/no-street.js.map +1 -1
  173. package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
  174. package/out/src/shard-recipes/po-box-cedex.js +60 -43
  175. package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
  176. package/out/src/shard-recipes/po-box.d.ts.map +1 -1
  177. package/out/src/shard-recipes/po-box.js +1 -1
  178. package/out/src/shard-recipes/po-box.js.map +1 -1
  179. package/out/src/shard-recipes/reviewed-postcode-tail.d.ts +47 -0
  180. package/out/src/shard-recipes/reviewed-postcode-tail.d.ts.map +1 -0
  181. package/out/src/shard-recipes/reviewed-postcode-tail.js +125 -0
  182. package/out/src/shard-recipes/reviewed-postcode-tail.js.map +1 -0
  183. package/out/src/shard-recipes/scaffold.d.ts +51 -23
  184. package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
  185. package/out/src/shard-recipes/scaffold.js +59 -23
  186. package/out/src/shard-recipes/scaffold.js.map +1 -1
  187. package/out/src/shard-recipes/street-affix.d.ts +54 -0
  188. package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
  189. package/out/src/shard-recipes/street-affix.js +203 -35
  190. package/out/src/shard-recipes/street-affix.js.map +1 -1
  191. package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
  192. package/out/src/shard-recipes/street-bare.js +3 -3
  193. package/out/src/shard-recipes/street-bare.js.map +1 -1
  194. package/out/src/shard-recipes/street.d.ts.map +1 -1
  195. package/out/src/shard-recipes/street.js +2 -2
  196. package/out/src/shard-recipes/street.js.map +1 -1
  197. package/out/src/shard-recipes/sub-venue-sources.d.ts +3 -6
  198. package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -1
  199. package/out/src/shard-recipes/sub-venue-sources.js +4 -9
  200. package/out/src/shard-recipes/sub-venue-sources.js.map +1 -1
  201. package/out/src/shard-recipes/sub-venue.d.ts +2 -2
  202. package/out/src/shard-recipes/sub-venue.d.ts.map +1 -1
  203. package/out/src/shard-recipes/sub-venue.js +7 -6
  204. package/out/src/shard-recipes/sub-venue.js.map +1 -1
  205. package/out/src/shard-recipes/trailing-region.d.ts +74 -0
  206. package/out/src/shard-recipes/trailing-region.d.ts.map +1 -0
  207. package/out/src/shard-recipes/trailing-region.js +157 -0
  208. package/out/src/shard-recipes/trailing-region.js.map +1 -0
  209. package/out/src/shard-recipes/unit.d.ts.map +1 -1
  210. package/out/src/shard-recipes/unit.js +6 -12
  211. package/out/src/shard-recipes/unit.js.map +1 -1
  212. package/out/src/{synthesize-anchor-absorption.d.ts → synthesizers/anchor-absorption.d.ts} +1 -1
  213. package/out/src/synthesizers/anchor-absorption.d.ts.map +1 -0
  214. package/out/src/{synthesize-anchor-absorption.js → synthesizers/anchor-absorption.js} +1 -1
  215. package/out/src/synthesizers/anchor-absorption.js.map +1 -0
  216. package/out/src/{synthesize-boundary-stress.d.ts → synthesizers/boundary-stress.d.ts} +3 -3
  217. package/out/src/synthesizers/boundary-stress.d.ts.map +1 -0
  218. package/out/src/{synthesize-boundary-stress.js → synthesizers/boundary-stress.js} +2 -2
  219. package/out/src/synthesizers/boundary-stress.js.map +1 -0
  220. package/out/src/{synthesize-german.d.ts → synthesizers/german.d.ts} +2 -2
  221. package/out/src/synthesizers/german.d.ts.map +1 -0
  222. package/out/src/{synthesize-german.js → synthesizers/german.js} +2 -2
  223. package/out/src/synthesizers/german.js.map +1 -0
  224. package/out/src/{synthesize-house-venue.d.ts → synthesizers/house-venue.d.ts} +4 -4
  225. package/out/src/synthesizers/house-venue.d.ts.map +1 -0
  226. package/out/src/{synthesize-house-venue.js → synthesizers/house-venue.js} +20 -6
  227. package/out/src/synthesizers/house-venue.js.map +1 -0
  228. package/out/src/{synthesize-intersection.d.ts → synthesizers/intersection.d.ts} +2 -2
  229. package/out/src/synthesizers/intersection.d.ts.map +1 -0
  230. package/out/src/{synthesize-intersection.js → synthesizers/intersection.js} +1 -1
  231. package/out/src/synthesizers/intersection.js.map +1 -0
  232. package/out/src/{synthesize-no-street.d.ts → synthesizers/no-street.d.ts} +3 -3
  233. package/out/src/synthesizers/no-street.d.ts.map +1 -0
  234. package/out/src/{synthesize-no-street.js → synthesizers/no-street.js} +2 -2
  235. package/out/src/synthesizers/no-street.js.map +1 -0
  236. package/out/src/{synthesize-po-box.d.ts → synthesizers/po-box.d.ts} +2 -2
  237. package/out/src/synthesizers/po-box.d.ts.map +1 -0
  238. package/out/src/{synthesize-po-box.js → synthesizers/po-box.js} +1 -1
  239. package/out/src/synthesizers/po-box.js.map +1 -0
  240. package/out/src/{synthesize-street.d.ts → synthesizers/street.d.ts} +2 -2
  241. package/out/src/synthesizers/street.d.ts.map +1 -0
  242. package/out/src/{synthesize-street.js → synthesizers/street.js} +4 -3
  243. package/out/src/synthesizers/street.js.map +1 -0
  244. package/out/src/{synthesize.d.ts → synthesizers/utils.d.ts} +3 -3
  245. package/out/src/synthesizers/utils.d.ts.map +1 -0
  246. package/out/src/{synthesize.js → synthesizers/utils.js} +5 -4
  247. package/out/src/synthesizers/utils.js.map +1 -0
  248. package/out/src/tools/align-shard.d.ts.map +1 -1
  249. package/out/src/tools/align-shard.js +2 -2
  250. package/out/src/tools/align-shard.js.map +1 -1
  251. package/out/src/tools/fetch/ban.d.ts.map +1 -1
  252. package/out/src/tools/fetch/ban.js +2 -18
  253. package/out/src/tools/fetch/ban.js.map +1 -1
  254. package/out/src/tools/fetch/download.d.ts +12 -3
  255. package/out/src/tools/fetch/download.d.ts.map +1 -1
  256. package/out/src/tools/fetch/download.js +20 -4
  257. package/out/src/tools/fetch/download.js.map +1 -1
  258. package/out/src/tools/fetch/geonames-dump.d.ts +87 -0
  259. package/out/src/tools/fetch/geonames-dump.d.ts.map +1 -0
  260. package/out/src/tools/fetch/geonames-dump.js +178 -0
  261. package/out/src/tools/fetch/geonames-dump.js.map +1 -0
  262. package/out/src/tools/fetch/geonames-postal.d.ts +76 -0
  263. package/out/src/tools/fetch/geonames-postal.d.ts.map +1 -0
  264. package/out/src/tools/fetch/geonames-postal.js +125 -0
  265. package/out/src/tools/fetch/geonames-postal.js.map +1 -0
  266. package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
  267. package/out/src/tools/fetch/imls-pls.js +5 -17
  268. package/out/src/tools/fetch/imls-pls.js.map +1 -1
  269. package/out/src/tools/fetch/index.d.ts +11 -0
  270. package/out/src/tools/fetch/index.d.ts.map +1 -1
  271. package/out/src/tools/fetch/index.js +11 -0
  272. package/out/src/tools/fetch/index.js.map +1 -1
  273. package/out/src/tools/fetch/nad.d.ts.map +1 -1
  274. package/out/src/tools/fetch/nad.js +20 -11
  275. package/out/src/tools/fetch/nad.js.map +1 -1
  276. package/out/src/tools/fetch/nppes.d.ts.map +1 -1
  277. package/out/src/tools/fetch/nppes.js +17 -22
  278. package/out/src/tools/fetch/nppes.js.map +1 -1
  279. package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
  280. package/out/src/tools/fetch/openaddresses.js +18 -17
  281. package/out/src/tools/fetch/openaddresses.js.map +1 -1
  282. package/out/src/tools/fetch/ourairports.d.ts.map +1 -1
  283. package/out/src/tools/fetch/ourairports.js +7 -2
  284. package/out/src/tools/fetch/ourairports.js.map +1 -1
  285. package/out/src/tools/fetch/ppd.d.ts +0 -4
  286. package/out/src/tools/fetch/ppd.d.ts.map +1 -1
  287. package/out/src/tools/fetch/ppd.js +1 -1
  288. package/out/src/tools/fetch/ppd.js.map +1 -1
  289. package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
  290. package/out/src/tools/fetch/state-hi-schools.js +3 -19
  291. package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
  292. package/out/src/tools/fetch/state-sources.d.ts +4 -0
  293. package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
  294. package/out/src/tools/fetch/state-sources.js +1 -5
  295. package/out/src/tools/fetch/state-sources.js.map +1 -1
  296. package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
  297. package/out/src/tools/fetch/tiger-full.js +15 -21
  298. package/out/src/tools/fetch/tiger-full.js.map +1 -1
  299. package/out/src/tools/golden-expand.d.ts.map +1 -1
  300. package/out/src/tools/golden-expand.js +34 -25
  301. package/out/src/tools/golden-expand.js.map +1 -1
  302. package/out/src/tools/golden-relabel-street.d.ts.map +1 -1
  303. package/out/src/tools/golden-relabel-street.js +4 -77
  304. package/out/src/tools/golden-relabel-street.js.map +1 -1
  305. package/out/src/tools/index.d.ts +2 -2
  306. package/out/src/tools/index.d.ts.map +1 -1
  307. package/out/src/tools/index.js +2 -2
  308. package/out/src/tools/index.js.map +1 -1
  309. package/out/src/tools/ingest-csv.d.ts.map +1 -1
  310. package/out/src/tools/ingest-csv.js +0 -11
  311. package/out/src/tools/ingest-csv.js.map +1 -1
  312. package/out/src/tools/overlay-manifest.d.ts +4 -0
  313. package/out/src/tools/overlay-manifest.d.ts.map +1 -1
  314. package/out/src/tools/overlay-manifest.js +16 -6
  315. package/out/src/tools/overlay-manifest.js.map +1 -1
  316. package/out/src/tools/postcode-triples.d.ts +175 -0
  317. package/out/src/tools/postcode-triples.d.ts.map +1 -0
  318. package/out/src/tools/postcode-triples.js +304 -0
  319. package/out/src/tools/postcode-triples.js.map +1 -0
  320. package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
  321. package/out/src/tools/shard-kryptonite.js +1 -3
  322. package/out/src/tools/shard-kryptonite.js.map +1 -1
  323. package/out/src/tools/shard-translit.d.ts.map +1 -1
  324. package/out/src/tools/shard-translit.js +4 -27
  325. package/out/src/tools/shard-translit.js.map +1 -1
  326. package/out/src/tools/sub-venue/harvest.d.ts +100 -0
  327. package/out/src/tools/sub-venue/harvest.d.ts.map +1 -0
  328. package/out/src/tools/sub-venue/harvest.js +168 -0
  329. package/out/src/tools/sub-venue/harvest.js.map +1 -0
  330. package/out/src/tools/sub-venue/head-nouns.d.ts +50 -0
  331. package/out/src/tools/sub-venue/head-nouns.d.ts.map +1 -0
  332. package/out/src/tools/sub-venue/head-nouns.js +210 -0
  333. package/out/src/tools/sub-venue/head-nouns.js.map +1 -0
  334. package/out/src/tools/sub-venue/surfaces.d.ts +45 -0
  335. package/out/src/tools/sub-venue/surfaces.d.ts.map +1 -0
  336. package/out/src/tools/sub-venue/surfaces.js +77 -0
  337. package/out/src/tools/sub-venue/surfaces.js.map +1 -0
  338. package/out/src/tools/sub-venue/table.d.ts +229 -0
  339. package/out/src/tools/sub-venue/table.d.ts.map +1 -0
  340. package/out/src/tools/sub-venue/table.js +108 -0
  341. package/out/src/tools/sub-venue/table.js.map +1 -0
  342. package/out/src/tools/sub-venue/wikidata.d.ts +26 -0
  343. package/out/src/tools/sub-venue/wikidata.d.ts.map +1 -0
  344. package/out/src/tools/sub-venue/wikidata.js +62 -0
  345. package/out/src/tools/sub-venue/wikidata.js.map +1 -0
  346. package/out/src/tools/sub-venue-lexicon.d.ts +27 -378
  347. package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -1
  348. package/out/src/tools/sub-venue-lexicon.js +30 -565
  349. package/out/src/tools/sub-venue-lexicon.js.map +1 -1
  350. package/out/src/{align.d.ts → utils/align.d.ts} +1 -1
  351. package/out/src/utils/align.d.ts.map +1 -0
  352. package/out/src/utils/align.js.map +1 -0
  353. package/out/src/utils/golden.d.ts.map +1 -0
  354. package/out/src/{golden.js → utils/golden.js} +2 -2
  355. package/out/src/utils/golden.js.map +1 -0
  356. package/out/src/utils/index.d.ts +14 -0
  357. package/out/src/utils/index.d.ts.map +1 -0
  358. package/out/src/utils/index.js +14 -0
  359. package/out/src/utils/index.js.map +1 -0
  360. package/out/src/utils/license.d.ts.map +1 -0
  361. package/out/src/{license.js → utils/license.js} +1 -1
  362. package/out/src/utils/license.js.map +1 -0
  363. package/out/src/{parquet.d.ts → utils/parquet.d.ts} +24 -5
  364. package/out/src/utils/parquet.d.ts.map +1 -0
  365. package/out/src/{parquet.js → utils/parquet.js} +6 -3
  366. package/out/src/utils/parquet.js.map +1 -0
  367. package/out/src/{split.d.ts → utils/split.d.ts} +1 -1
  368. package/out/src/utils/split.d.ts.map +1 -0
  369. package/out/src/{split.js → utils/split.js} +2 -2
  370. package/out/src/utils/split.js.map +1 -0
  371. package/out/src/utils/tokenize.d.ts.map +1 -0
  372. package/out/src/utils/tokenize.js.map +1 -0
  373. package/out/src/utils/wof-json.d.ts.map +1 -0
  374. package/out/src/utils/wof-json.js.map +1 -0
  375. package/package.json +287 -21
  376. package/src/adapters/ban/adapter.ts +7 -5
  377. package/src/adapters/fcc-bdc/adapter.ts +5 -4
  378. package/src/adapters/geonames/adapter.ts +3 -3
  379. package/src/adapters/geonames-postal/adapter.ts +4 -4
  380. package/src/adapters/gnaf/adapter.ts +3 -3
  381. package/src/adapters/gnaf/assemble.ts +1 -1
  382. package/src/adapters/index.ts +3 -2
  383. package/src/adapters/openaddresses/adapter.ts +4 -4
  384. package/src/adapters/overture/adapter.ts +3 -3
  385. package/src/adapters/state-hi-schools/adapter.ts +8 -7
  386. package/src/adapters/state-ia-contractors/adapter.ts +13 -8
  387. package/src/adapters/state-ny-notaries/adapter.ts +13 -32
  388. package/src/adapters/state-tx-notaries/adapter.ts +13 -8
  389. package/src/adapters/synth-po-box/adapter.ts +4 -4
  390. package/src/adapters/tiger/adapter.ts +5 -3
  391. package/src/adapters/usgov-hrsa-fqhc/adapter.ts +8 -7
  392. package/src/adapters/usgov-imls-pls/adapter.ts +12 -8
  393. package/src/adapters/usgov-irs-bmf/adapter.ts +7 -6
  394. package/src/adapters/usgov-nad/adapter.ts +9 -9
  395. package/src/adapters/usgov-nppes/adapter.ts +13 -12
  396. package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +8 -7
  397. package/src/{adapter.ts → adapters/utils/index.ts} +15 -3
  398. package/src/adapters/wof-admin-jp/adapter.ts +9 -12
  399. package/src/adapters/wof-admin-json/adapter.ts +15 -54
  400. package/src/adapters/wof-json-rows.ts +79 -0
  401. package/src/adapters/wof-postalcode-json/adapter.ts +16 -50
  402. package/src/build.ts +8 -8
  403. package/src/index.ts +3 -17
  404. package/src/name-prone-us-suffixes.ts +12 -0
  405. package/src/parquet-wrapper/reader.ts +9 -0
  406. package/src/runner.ts +2 -2
  407. package/src/shard-recipes/anchor-absorption.ts +5 -5
  408. package/src/shard-recipes/bare-country.ts +95 -0
  409. package/src/shard-recipes/boundary-stress.ts +3 -2
  410. package/src/shard-recipes/country-balanced.ts +16 -27
  411. package/src/shard-recipes/cz-pcfirst-preposition.ts +29 -10
  412. package/src/shard-recipes/fr-admin-split.ts +6 -4
  413. package/src/shard-recipes/fr-bare-street.ts +85 -10
  414. package/src/shard-recipes/fr-lieudit.ts +6 -4
  415. package/src/shard-recipes/fr-order.ts +13 -26
  416. package/src/shard-recipes/german.ts +9 -18
  417. package/src/shard-recipes/house-venue.ts +2 -1
  418. package/src/shard-recipes/index.ts +8 -1
  419. package/src/shard-recipes/intersection.ts +9 -17
  420. package/src/shard-recipes/locale.ts +16 -24
  421. package/src/shard-recipes/no-street.ts +2 -1
  422. package/src/shard-recipes/po-box-cedex.ts +76 -58
  423. package/src/shard-recipes/po-box.ts +2 -1
  424. package/src/shard-recipes/reviewed-postcode-tail.ts +189 -0
  425. package/src/shard-recipes/scaffold.ts +99 -24
  426. package/src/shard-recipes/street-affix.ts +284 -45
  427. package/src/shard-recipes/street-bare.ts +4 -3
  428. package/src/shard-recipes/street.ts +3 -2
  429. package/src/shard-recipes/sub-venue-sources.ts +13 -23
  430. package/src/shard-recipes/sub-venue.ts +11 -9
  431. package/src/shard-recipes/trailing-region.ts +183 -0
  432. package/src/shard-recipes/unit.ts +8 -17
  433. package/src/{synthesize-boundary-stress.ts → synthesizers/boundary-stress.ts} +2 -2
  434. package/src/{synthesize-german.ts → synthesizers/german.ts} +3 -2
  435. package/src/{synthesize-house-venue.ts → synthesizers/house-venue.ts} +20 -6
  436. package/src/{synthesize-intersection.ts → synthesizers/intersection.ts} +1 -1
  437. package/src/{synthesize-no-street.ts → synthesizers/no-street.ts} +2 -2
  438. package/src/{synthesize-po-box.ts → synthesizers/po-box.ts} +1 -1
  439. package/src/{synthesize-street.ts → synthesizers/street.ts} +5 -3
  440. package/src/{synthesize.ts → synthesizers/utils.ts} +6 -4
  441. package/src/tools/align-shard.ts +3 -2
  442. package/src/tools/fetch/ban.ts +2 -22
  443. package/src/tools/fetch/download.ts +22 -4
  444. package/src/tools/fetch/geonames-dump.ts +267 -0
  445. package/src/tools/fetch/geonames-postal.ts +180 -0
  446. package/src/tools/fetch/imls-pls.ts +6 -20
  447. package/src/tools/fetch/index.ts +11 -0
  448. package/src/tools/fetch/nad.ts +21 -10
  449. package/src/tools/fetch/nppes.ts +17 -23
  450. package/src/tools/fetch/openaddresses.ts +18 -22
  451. package/src/tools/fetch/ourairports.ts +7 -2
  452. package/src/tools/fetch/ppd.ts +1 -1
  453. package/src/tools/fetch/state-hi-schools.ts +3 -23
  454. package/src/tools/fetch/state-sources.ts +1 -1
  455. package/src/tools/fetch/tiger-full.ts +17 -24
  456. package/src/tools/golden-expand.ts +44 -33
  457. package/src/tools/golden-relabel-street.ts +5 -68
  458. package/src/tools/index.ts +2 -2
  459. package/src/tools/ingest-csv.ts +0 -13
  460. package/src/tools/overlay-manifest.ts +20 -8
  461. package/src/tools/postcode-triples.ts +375 -0
  462. package/src/tools/shard-kryptonite.ts +4 -5
  463. package/src/tools/shard-translit.ts +16 -35
  464. package/src/tools/sub-venue/harvest.ts +235 -0
  465. package/src/tools/sub-venue/head-nouns.ts +242 -0
  466. package/src/tools/sub-venue/surfaces.ts +95 -0
  467. package/src/tools/sub-venue/table.ts +281 -0
  468. package/src/tools/sub-venue/wikidata.ts +85 -0
  469. package/src/tools/sub-venue-lexicon.ts +67 -886
  470. package/src/{align.ts → utils/align.ts} +2 -1
  471. package/src/{golden.ts → utils/golden.ts} +2 -3
  472. package/src/utils/index.ts +14 -0
  473. package/src/{license.ts → utils/license.ts} +1 -1
  474. package/src/{parquet.ts → utils/parquet.ts} +22 -9
  475. package/src/{split.ts → utils/split.ts} +3 -3
  476. package/out/src/adapter.d.ts.map +0 -1
  477. package/out/src/adapter.js.map +0 -1
  478. package/out/src/align.d.ts.map +0 -1
  479. package/out/src/align.js.map +0 -1
  480. package/out/src/format.d.ts +0 -14
  481. package/out/src/format.d.ts.map +0 -1
  482. package/out/src/format.js +0 -14
  483. package/out/src/format.js.map +0 -1
  484. package/out/src/golden.d.ts.map +0 -1
  485. package/out/src/golden.js.map +0 -1
  486. package/out/src/license.d.ts.map +0 -1
  487. package/out/src/license.js.map +0 -1
  488. package/out/src/parquet.d.ts.map +0 -1
  489. package/out/src/parquet.js.map +0 -1
  490. package/out/src/split.d.ts.map +0 -1
  491. package/out/src/split.js.map +0 -1
  492. package/out/src/synthesize-anchor-absorption.d.ts.map +0 -1
  493. package/out/src/synthesize-anchor-absorption.js.map +0 -1
  494. package/out/src/synthesize-boundary-stress.d.ts.map +0 -1
  495. package/out/src/synthesize-boundary-stress.js.map +0 -1
  496. package/out/src/synthesize-german.d.ts.map +0 -1
  497. package/out/src/synthesize-german.js.map +0 -1
  498. package/out/src/synthesize-house-venue.d.ts.map +0 -1
  499. package/out/src/synthesize-house-venue.js.map +0 -1
  500. package/out/src/synthesize-intersection.d.ts.map +0 -1
  501. package/out/src/synthesize-intersection.js.map +0 -1
  502. package/out/src/synthesize-no-street.d.ts.map +0 -1
  503. package/out/src/synthesize-no-street.js.map +0 -1
  504. package/out/src/synthesize-po-box.d.ts.map +0 -1
  505. package/out/src/synthesize-po-box.js.map +0 -1
  506. package/out/src/synthesize-street.d.ts.map +0 -1
  507. package/out/src/synthesize-street.js.map +0 -1
  508. package/out/src/synthesize.d.ts.map +0 -1
  509. package/out/src/synthesize.js.map +0 -1
  510. package/out/src/tokenize.d.ts.map +0 -1
  511. package/out/src/tokenize.js.map +0 -1
  512. package/out/src/wof-json.d.ts.map +0 -1
  513. package/out/src/wof-json.js.map +0 -1
  514. package/src/format.ts +0 -14
  515. /package/out/src/{align.js → utils/align.js} +0 -0
  516. /package/out/src/{golden.d.ts → utils/golden.d.ts} +0 -0
  517. /package/out/src/{license.d.ts → utils/license.d.ts} +0 -0
  518. /package/out/src/{tokenize.d.ts → utils/tokenize.d.ts} +0 -0
  519. /package/out/src/{tokenize.js → utils/tokenize.js} +0 -0
  520. /package/out/src/{wof-json.d.ts → utils/wof-json.d.ts} +0 -0
  521. /package/out/src/{wof-json.js → utils/wof-json.js} +0 -0
  522. /package/src/{synthesize-anchor-absorption.ts → synthesizers/anchor-absorption.ts} +0 -0
  523. /package/src/{tokenize.ts → utils/tokenize.ts} +0 -0
  524. /package/src/{wof-json.ts → utils/wof-json.ts} +0 -0
@@ -0,0 +1,79 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ * @file Deterministic canonical-row emission shared by WOF GeoJSON adapters.
6
+ */
7
+
8
+ import type { ComponentTag } from "@mailwoman/core/types"
9
+ import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
10
+
11
+ import type { AdapterOptions, CanonicalRow } from "#types"
12
+ import type { WOFRecord } from "#utils"
13
+
14
+ export interface WOFVariantSpec {
15
+ suffix: string
16
+ components: Partial<Record<ComponentTag, string>>
17
+ }
18
+
19
+ interface EmitWOFJSONRowsOptions {
20
+ records: ReadonlyMap<number, WOFRecord>
21
+ ancestry: ReadonlyMap<number, WOFRecord[]>
22
+ adapterOptions: AdapterOptions
23
+ adapterID: string
24
+ localeByCountry: Readonly<Record<string, string>>
25
+ shouldEmit?: (record: WOFRecord) => boolean
26
+ nameSlotsFor: (record: WOFRecord) => Array<{ key: string; value: string }>
27
+ variantsFor: (record: WOFRecord, ancestry: WOFRecord[], selfName: string) => WOFVariantSpec[]
28
+ }
29
+
30
+ /**
31
+ * Emit aligned rows in WOF-id order, enforcing the adapter limit across all name and hierarchy variants.
32
+ */
33
+ export function* emitWOFJSONRows(options: EmitWOFJSONRowsOptions): Generator<CanonicalRow> {
34
+ const {
35
+ records,
36
+ ancestry,
37
+ adapterOptions,
38
+ adapterID,
39
+ localeByCountry,
40
+ shouldEmit = () => true,
41
+ nameSlotsFor,
42
+ variantsFor,
43
+ } = options
44
+
45
+ let emitted = 0
46
+
47
+ for (const id of [...records.keys()].toSorted((a, b) => a - b)) {
48
+ if (adapterOptions.signal?.aborted) return
49
+ const record = records.get(id)!
50
+
51
+ if (!shouldEmit(record)) continue
52
+
53
+ for (const slot of nameSlotsFor(record)) {
54
+ for (const variant of variantsFor(record, ancestry.get(id) ?? [], slot.value)) {
55
+ if (adapterOptions.limit !== undefined && emitted >= adapterOptions.limit) return
56
+
57
+ const raw = formatAddress(variant.components, record.country, { separator: ", " })
58
+
59
+ if (!raw) continue
60
+ const components = reconcileComponents(variant.components, raw)
61
+
62
+ if (!Object.keys(components).length) continue
63
+
64
+ yield {
65
+ raw,
66
+ components,
67
+ country: record.country,
68
+ locale: localeByCountry[record.country],
69
+ source: adapterID,
70
+ source_id: `${adapterID}-${record.id}-${slot.key}-${variant.suffix}`,
71
+ corpus_version: "",
72
+ license: "CC0-1.0",
73
+ }
74
+
75
+ emitted++
76
+ }
77
+ }
78
+ }
79
+ }
@@ -40,9 +40,10 @@
40
40
  import type { WhosOnFirstPlacetype } from "@mailwoman/core/resources/whosonfirst"
41
41
  import type { ComponentTag } from "@mailwoman/core/types"
42
42
 
43
- import { formatAddress, reconcileComponents } from "../../format.ts"
44
- import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
45
- import { buildAncestryIndex, normalizeNameKey, walkFeatures, type WOFRecord } from "../../wof-json.ts"
43
+ import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
44
+ import { buildAncestryIndex, normalizeNameKey, walkFeatures, type WOFRecord } from "#utils"
45
+
46
+ import { emitWOFJSONRows, type WOFVariantSpec } from "../wof-json-rows.ts"
46
47
 
47
48
  const COUNTRY_DISPLAY_NAME: Record<string, string> = {
48
49
  US: "United States of America",
@@ -71,16 +72,11 @@ function placetypeToTag(placetype: WhosOnFirstPlacetype | string): ComponentTag
71
72
  }
72
73
  }
73
74
 
74
- interface VariantSpec {
75
- suffix: string
76
- components: Partial<Record<ComponentTag, string>>
77
- }
78
-
79
75
  /**
80
76
  * Compute hierarchy variants for a postcode record. `selfName` is the postcode surface form (canonical `wof:name` for
81
77
  * the `default` slot, a `name:*` localized variant otherwise).
82
78
  */
83
- export function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): VariantSpec[] {
79
+ export function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): WOFVariantSpec[] {
84
80
  if (placetypeToTag(row.placetype) !== "postcode") return []
85
81
 
86
82
  const locality = ancestry.find((a) => placetypeToTag(a.placetype) === "locality")
@@ -88,7 +84,7 @@ export function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfN
88
84
  const country = ancestry.find((a) => placetypeToTag(a.placetype) === "country")
89
85
  const countryDisplay = COUNTRY_DISPLAY_NAME[row.country] ?? country?.name ?? row.country
90
86
 
91
- const variants: VariantSpec[] = [{ suffix: "self", components: { postcode: selfName } }]
87
+ const variants: WOFVariantSpec[] = [{ suffix: "self", components: { postcode: selfName } }]
92
88
 
93
89
  if (locality) {
94
90
  variants.push({
@@ -167,46 +163,16 @@ export function createWOFPostalcodeAdapter(): CorpusAdapter {
167
163
  const ancestry = buildAncestryIndex(byID)
168
164
 
169
165
  // Pass 2: emit postcode rows only, sorted by id for determinism.
170
- const ids = [...byID.keys()].toSorted((a, b) => a - b)
171
- let emitted = 0
172
-
173
- for (const id of ids) {
174
- if (opts.signal?.aborted) return
175
- const rec = byID.get(id)!
176
-
177
- if (placetypeToTag(rec.placetype) !== "postcode") continue
178
-
179
- const chain = ancestry.get(id) ?? []
180
- const slots = nameSlotsFor(rec)
181
-
182
- for (const slot of slots) {
183
- const variants = postcodeVariantsFor(rec, chain, slot.value)
184
-
185
- for (const variant of variants) {
186
- if (opts.limit !== undefined && emitted >= opts.limit) return
187
-
188
- const raw = formatAddress(variant.components, rec.country, { separator: ", " })
189
-
190
- if (!raw) continue
191
- const aligned = reconcileComponents(variant.components, raw)
192
-
193
- if (!Object.keys(aligned).length) continue
194
-
195
- yield {
196
- raw,
197
- components: aligned,
198
- country: rec.country,
199
- locale: LOCALE_BY_COUNTRY[rec.country],
200
- source: WOF_POSTALCODE_ADAPTER_ID,
201
- source_id: `${WOF_POSTALCODE_ADAPTER_ID}-${rec.id}-${slot.key}-${variant.suffix}`,
202
- corpus_version: "",
203
- license: "CC0-1.0",
204
- }
205
-
206
- emitted++
207
- }
208
- }
209
- }
166
+ yield* emitWOFJSONRows({
167
+ records: byID,
168
+ ancestry,
169
+ adapterOptions: opts,
170
+ adapterID: WOF_POSTALCODE_ADAPTER_ID,
171
+ localeByCountry: LOCALE_BY_COUNTRY,
172
+ shouldEmit: (record) => placetypeToTag(record.placetype) === "postcode",
173
+ nameSlotsFor,
174
+ variantsFor: postcodeVariantsFor,
175
+ })
210
176
  },
211
177
  }
212
178
  }
package/src/build.ts CHANGED
@@ -55,20 +55,20 @@ import { $public } from "@mailwoman/core/env"
55
55
  import { parseJSONStrict } from "@mailwoman/core/objects"
56
56
  import { JSONSpliterator } from "spliterator"
57
57
 
58
- import { defaultAdapterRegistry } from "./adapter.ts"
59
- import { alignRow } from "./align.ts"
60
- import { licenseExcluded } from "./license.ts"
61
- import { writeShards, type ShardManifest } from "./parquet.ts"
62
- import { runAdapter, type AdapterRunManifest } from "./runner.ts"
58
+ import { defaultAdapterRegistry } from "#adapters/utils"
59
+ import { runAdapter, type AdapterRunManifest } from "#runner"
60
+ import { defaultAugmentationsForCountry, synthesizeRow } from "#synthesizers/utils"
61
+ import type { AdapterOptions, CanonicalRow, CorpusAdapter, LabeledRow } from "#types"
62
+ import { alignRow } from "#utils/align"
63
+ import { licenseExcluded } from "#utils/license"
64
+ import { writeShards, type ShardManifest } from "#utils/parquet"
63
65
  import {
64
66
  defaultHoldouts,
65
67
  splitForRow,
66
68
  writeSplitManifestsFromLabeledFiles,
67
69
  type SplitManifest,
68
70
  type SplitName,
69
- } from "./split.ts"
70
- import { defaultAugmentationsForCountry, synthesizeRow } from "./synthesize.ts"
71
- import type { AdapterOptions, CanonicalRow, CorpusAdapter, LabeledRow } from "./types.ts"
71
+ } from "#utils/split"
72
72
 
73
73
  /**
74
74
  * Stage tags surfaced to `onProgress`.
package/src/index.ts CHANGED
@@ -4,25 +4,11 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
6
 
7
- export * from "./adapter.ts"
7
+ export * from "./adapters/utils/index.ts"
8
8
  export * from "./adapters/gnaf/assemble.ts"
9
9
  export * from "./adapters/index.ts"
10
- export * from "./align.ts"
11
10
  export * from "./build.ts"
12
- export * from "./format.ts"
13
- export * from "./golden.ts"
14
- export * from "./parquet.ts"
15
- export * from "./parquet-wrapper/index.ts"
16
11
  export * from "./runner.ts"
12
+ export * from "./parquet-wrapper/index.ts"
17
13
  export * from "./shard-recipes/index.ts"
18
- export * from "./split.ts"
19
- export * from "./synthesize-anchor-absorption.ts"
20
- export * from "./synthesize-german.ts"
21
- export * from "./synthesize-house-venue.ts"
22
- export * from "./synthesize-intersection.ts"
23
- export * from "./synthesize-no-street.ts"
24
- export * from "./synthesize-po-box.ts"
25
- export * from "./synthesize-street.ts"
26
- export * from "./synthesize.ts"
27
- export * from "./tokenize.ts"
28
- export * from "./types.ts"
14
+ export * from "#types"
@@ -0,0 +1,12 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Re-export shim: the name-prone set moved to `@mailwoman/codex/us` (data in
7
+ * `codex/us/street-suffix.json`, 2026-08-10) so the golden relabel flags, the #1569 shard
8
+ * recipe, AND the Python relabel pass (via the `gazetteer affix-relabel` v2 lexicon) all read
9
+ * one record. Import from `@mailwoman/codex/us` in new code.
10
+ */
11
+
12
+ export { NAME_PRONE_US_SUFFIXES } from "@mailwoman/codex/us"
@@ -67,6 +67,15 @@ export class ParquetReader<T extends ParquetRecordLike> extends BaseParquetReade
67
67
  * caller wanting the whole record should use.
68
68
  */
69
69
  public async *project<K extends keyof T>(...columns: K[]): AsyncGenerator<Pick<T, K>, void, unknown> {
70
+ const missing = columns.filter((column) => !Object.hasOwn(this.schema.schema, column))
71
+
72
+ if (missing.length) {
73
+ throw new Error(
74
+ `Parquet projection requested column${missing.length === 1 ? "" : "s"} absent from the file schema: ` +
75
+ missing.map(String).join(", ")
76
+ )
77
+ }
78
+
70
79
  const cursor = this.getCursor(columns.map((column) => [column as string]))
71
80
 
72
81
  for (;;) {
package/src/runner.ts CHANGED
@@ -31,8 +31,8 @@ import { createWriteStream, type WriteStream } from "node:fs"
31
31
  import { mkdir, writeFile } from "node:fs/promises"
32
32
  import { dirname, join } from "node:path"
33
33
 
34
- import { canonicalDedupKey, streamingSha256, type AdapterRegistry, type StreamingHasher } from "./adapter.ts"
35
- import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "./types.ts"
34
+ import { canonicalDedupKey, streamingSha256, type AdapterRegistry, type StreamingHasher } from "#adapters/utils"
35
+ import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
36
36
 
37
37
  /**
38
38
  * Snapshot of the runner's state, emitted on every progress tick.
@@ -17,10 +17,10 @@ import { readFileSync } from "node:fs"
17
17
  import { parseJSONStrict } from "@mailwoman/core/objects"
18
18
  import { dataRootPath } from "@mailwoman/core/utils"
19
19
 
20
- import { stableSourceID } from "../adapter.ts"
21
- import { alignRow } from "../align.ts"
22
- import { synthesizeAnchorAbsorptionRow } from "../synthesize-anchor-absorption.ts"
23
- import { makeLcg, type ShardRecipe } from "./scaffold.ts"
20
+ import { synthesizeAnchorAbsorptionRow } from "#synthesizers/anchor-absorption"
21
+ import { alignRow } from "#utils"
22
+
23
+ import { makeLcg, shardSourceID, type ShardRecipe } from "./scaffold.ts"
24
24
 
25
25
  /**
26
26
  * The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through the data-root
@@ -76,7 +76,7 @@ export const anchorAbsorptionRecipe: ShardRecipe = {
76
76
  country,
77
77
  locale: synth.locale,
78
78
  source,
79
- source_id: stableSourceID(source, `${i}` as unknown as Parameters<typeof stableSourceID>[1]),
79
+ source_id: shardSourceID(source, { v: String(i) }),
80
80
  }
81
81
 
82
82
  const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
@@ -0,0 +1,95 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `bare-country` — the country name as the WHOLE query. Addresses almost never consist of one bare
7
+ * country name, so the class is out of the model's training distribution and the tag is a coin
8
+ * flip: measured on the 2026-08-13 panel, France/Germany/United States/New Zealand parsed
9
+ * `country` while Japan/China/Nigeria/Australia/Deutschland parsed `locality`. The retrieval-side
10
+ * bare-country race (#1651) covers the answer either way; this shard closes the PARSE half so the
11
+ * tag itself is right.
12
+ *
13
+ * Source is the codex country table — every ISO canonical English name plus the curated surface
14
+ * forms (endonyms and long-form names). The 2–3 letter CODES are deliberately excluded: a bare
15
+ * `JP` or `GER` is an abbreviation register with its own ambiguity surface (`IN`, `DE`, `TO` are
16
+ * English words), and teaching it here would be an unmeasured claim. Case augmentation is the
17
+ * loader's job, as everywhere.
18
+ */
19
+
20
+ import { COUNTRY_SURFACE_FORMS, CountryNames, matchCountry } from "@mailwoman/codex/country"
21
+
22
+ import { alignAndWrite, makeMulberry32, type ShardRecipe, shardSourceID } from "./scaffold.ts"
23
+
24
+ /**
25
+ * Surfaces shorter than this are the code register (`JP`, `GER`), not a name — excluded (see the module doc).
26
+ */
27
+ const MIN_NAME_LENGTH = 4
28
+
29
+ /**
30
+ * Every (surface, iso2) pair the shard emits: the ISO canonical names plus each country's curated surface forms,
31
+ * deduplicated on the surface string (a form shared across countries — none known — would keep its first bearer).
32
+ */
33
+ function* bareCountrySurfaces(): Generator<{ surface: string; iso2: string }> {
34
+ const seen = new Set<string>()
35
+
36
+ for (const [iso2, forms] of Object.entries(COUNTRY_SURFACE_FORMS)) {
37
+ for (const surface of forms) {
38
+ if (surface.length < MIN_NAME_LENGTH || seen.has(surface)) continue
39
+ seen.add(surface)
40
+ yield { surface, iso2 }
41
+ }
42
+ }
43
+
44
+ for (const name of CountryNames) {
45
+ if (name.length < MIN_NAME_LENGTH || seen.has(name)) continue
46
+ const matched = matchCountry(name)
47
+
48
+ if (!matched) continue
49
+ seen.add(name)
50
+ yield { surface: name, iso2: matched.iso2 }
51
+ }
52
+ }
53
+
54
+ /**
55
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
56
+ * and `description` below for the surface form it generates.
57
+ */
58
+ export const bareCountryRecipe: ShardRecipe = {
59
+ name: "bare-country",
60
+ description: "The country name as the whole query (#1651 parse half): ISO names + curated endonyms, no codes",
61
+ mode: "generate",
62
+ async run(opts, write) {
63
+ // Seeded for parity with the other recipes; the surfaces themselves drive the content.
64
+ makeMulberry32(opts.seed)
65
+ let read = 0
66
+ let emitted = 0
67
+ let skipped = 0
68
+
69
+ for (const { surface, iso2 } of bareCountrySurfaces()) {
70
+ read++
71
+
72
+ const components: Record<string, string> = { country: surface }
73
+ const source_id = shardSourceID("synth-bare-country", { country: surface, cc: iso2, v: String(read) })
74
+
75
+ const canonical = {
76
+ raw: surface,
77
+ components,
78
+ country: iso2,
79
+ locale: "und",
80
+ source: "synth-bare-country",
81
+ source_id,
82
+ corpus_version: "0.11.0",
83
+ license: "Synthetic — bare-country; surfaces from the codex country table (ISO 3166 + curated forms)",
84
+ }
85
+
86
+ if (alignAndWrite(write, canonical, "bare-country")) {
87
+ emitted++
88
+ } else {
89
+ skipped++
90
+ }
91
+ }
92
+
93
+ return { read, emitted, skipped }
94
+ },
95
+ }
@@ -12,8 +12,9 @@
12
12
  * `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
13
13
  */
14
14
 
15
- import { alignRow } from "../align.ts"
16
- import { type BoundaryStressTemplate, synthesizeBoundaryStressRow } from "../synthesize-boundary-stress.ts"
15
+ import { type BoundaryStressTemplate, synthesizeBoundaryStressRow } from "#synthesizers/boundary-stress"
16
+ import { alignRow } from "#utils"
17
+
17
18
  import { makeMulberry32, type ShardRecipe, shardSourceID } from "./scaffold.ts"
18
19
 
19
20
  /**
@@ -24,16 +24,16 @@
24
24
  * legacy script used.
25
25
  */
26
26
 
27
- import { spawnSync } from "node:child_process"
28
-
29
27
  import { COUNTRY_SURFACE_FORMS, CountryNames } from "@mailwoman/codex/country"
28
+ import { isPresent } from "@mailwoman/core/objects"
30
29
  import type { ComponentTag } from "@mailwoman/core/types"
31
30
  import { dataRootPath } from "@mailwoman/core/utils"
32
31
 
33
- import { stableSourceID } from "../adapter.ts"
34
- import { alignRow } from "../align.ts"
35
- import type { CanonicalRow } from "../types.ts"
36
- import { makeMulberry32, readCSVRecords, type ShardRecipe } from "./scaffold.ts"
32
+ import { stableSourceID } from "#adapters/utils"
33
+ import type { CanonicalRow } from "#types"
34
+ import { alignRow } from "#utils"
35
+
36
+ import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
37
37
 
38
38
  // v2: the country TOKEN is decoupled from the skeleton's locale and drawn from a BROAD pool — every
39
39
  // ISO canonical name + every curated surface form (endonyms/abbrevs). Surface forms are over-weighted
@@ -154,26 +154,15 @@ interface CountryTuple {
154
154
  order: string
155
155
  }
156
156
 
157
- function readTuples(source: CountrySource, limit: number): CountryTuple[] {
158
- // countrywide extracts (FR/IT/NL) are GB-scale — cap the bytes with `head` (read ~8 lines per wanted
159
- // tuple to survive dedup/skips) so the whole extract never has to be held in memory.
160
- const maxLines = Math.max(limit * 8, 20_000) + 1
161
-
162
- const r = spawnSync("bash", ["-c", `unzip -p "${source.zip}" "${source.csv}" | head -n ${maxLines}`], {
163
- maxBuffer: 1024 * 1024 * 1024,
164
- encoding: "buffer",
165
- })
166
-
167
- if (r.status !== 0) {
168
- console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
169
-
170
- return []
171
- }
172
-
157
+ /**
158
+ * The countrywide extracts (FR/IT/NL) are GB-scale, so this reads only as far as `limit` distinct tuples: the `break`
159
+ * closes the reader, which releases the archive without inflating the rest of the member.
160
+ */
161
+ async function readTuples(source: CountrySource, limit: number): Promise<CountryTuple[]> {
173
162
  const tuples: CountryTuple[] = []
174
163
  const seen = new Set<string>()
175
164
 
176
- for (const row of readCSVRecords(r.stdout)) {
165
+ for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
177
166
  if (tuples.length >= limit) break
178
167
 
179
168
  const street = row.street ?? "",
@@ -233,14 +222,14 @@ function renderCountry(
233
222
  let body: string
234
223
 
235
224
  if (order === "us") {
236
- const regPc = [reg, pc].filter(Boolean).join(" ")
225
+ const regPc = [reg, pc].filter(isPresent).join(" ")
237
226
  body = `${hn} ${street}, ${loc}${regPc ? ", " + regPc : ""}`
238
227
  } else if (order === "fr") {
239
228
  // French is NUMBER-street, postcode-city: "84 Route de la Fontaine, 75008 Paris".
240
- body = `${hn} ${street}, ${[pc, loc].filter(Boolean).join(" ")}`
229
+ body = `${hn} ${street}, ${[pc, loc].filter(isPresent).join(" ")}`
241
230
  } else {
242
231
  // DE/IT/NL: street-number, postcode-city: "Pariser Platz 1, 10117 Berlin".
243
- const pcCity = [pc, loc].filter(Boolean).join(" ")
232
+ const pcCity = [pc, loc].filter(isPresent).join(" ")
244
233
  body = `${street} ${hn}, ${pcCity}`
245
234
  }
246
235
 
@@ -449,7 +438,7 @@ export const countryBalancedRecipe: ShardRecipe = {
449
438
  const pool: CountryTuple[] = []
450
439
 
451
440
  for (const s of sources) {
452
- const t = readTuples(s, perSource)
441
+ const t = await readTuples(s, perSource)
453
442
 
454
443
  console.error(` ${s.csv} (${s.iso2}): ${t.length} tuples`)
455
444
 
@@ -16,6 +16,13 @@
16
16
 
17
17
  import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
18
18
 
19
+ /**
20
+ * The order-cycle slot for the STREET-LESS form (`«city» «pc», Česko`) — the exact surface of the
21
+ * `cz-full-praha-100-00` board row, whose absence from the street-bearing orders was the v4.5.0 no-promote's measured
22
+ * gap.
23
+ */
24
+ const STREETLESS_ORDER = 3
25
+
19
26
  /**
20
27
  * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
21
28
  * and `description` below for the surface form it generates.
@@ -44,27 +51,39 @@ export const czPcFirstPrepositionRecipe: ShardRecipe = {
44
51
  continue
45
52
  }
46
53
 
47
- const order = read % 3
54
+ const order = read % 4
55
+ // The OFFICIAL Czech rendering spaces the PSČ as `NNN NN` ('512 44'); OpenAddresses stores it
56
+ // unspaced ('51244'), and a model trained only on the source form reads the spaced surface as
57
+ // house_number + garbage (the 'Praha 100 00' mangle). Alternate the two renderings so both
58
+ // orthographies are attested — the LABEL is the postcode either way.
59
+ const spaced = read % 2 === 0 && /^\d{5}$/.test(postcode)
60
+ const postcodeSurface = spaced ? `${postcode.slice(0, 3)} ${postcode.slice(3)}` : postcode
48
61
  let raw: string
49
62
 
50
- const components: Record<string, string> = {
51
- street,
52
- house_number: number,
53
- postcode,
54
- locality: city,
55
- }
63
+ const components: Record<string, string> =
64
+ order === STREETLESS_ORDER
65
+ ? { locality: city, postcode: postcodeSurface, country: "Česko" }
66
+ : { street, house_number: number, postcode: postcodeSurface, locality: city }
56
67
 
57
68
  if (order === 0) {
58
- raw = `${postcode} ${city}, ${street} ${number}`
69
+ raw = `${postcodeSurface} ${city}, ${street} ${number}`
59
70
  } else if (order === 1) {
60
- raw = `${street} ${number}, ${postcode} ${city}`
71
+ raw = `${street} ${number}, ${postcodeSurface} ${city}`
72
+ } else if (order === 2) {
73
+ raw = `${city}, ${postcodeSurface}, ${street} ${number}`
61
74
  } else {
62
- raw = `${city}, ${postcode}, ${street} ${number}`
75
+ // order === STREETLESS_ORDER: the street-less form — `«city» «pc», Česko` — the exact surface of the
76
+ // cz-full-praha-100-00 board row. The v4.5.0 no-promote receipt measured the gap: every
77
+ // prior order was street-bearing, so the model never saw a spaced PSČ beside a bare
78
+ // locality and mangled 'Praha 100 00, Czechia' into house_number spans. Street/number
79
+ // stay OUT of the components for this form (they are not in the surface).
80
+ raw = `${city} ${postcodeSurface}, Česko`
63
81
  }
64
82
 
65
83
  const source_id = shardSourceID("synth-cz-pcfirst-preposition", {
66
84
  ...components,
67
85
  o: String(order),
86
+ s: spaced ? "1" : "0",
68
87
  v: String(read),
69
88
  })
70
89
 
@@ -32,11 +32,13 @@
32
32
 
33
33
  import { departementForCodePostal } from "@mailwoman/codex/fr"
34
34
  import type { ComponentTag } from "@mailwoman/core/types"
35
+ import { tempRootPath } from "@mailwoman/core/utils"
35
36
  import { CSVSpliterator, Delimiters } from "spliterator"
36
37
 
37
- import { stableSourceID } from "../adapter.ts"
38
- import { alignRow } from "../align.ts"
39
- import type { CanonicalRow } from "../types.ts"
38
+ import { stableSourceID } from "#adapters/utils"
39
+ import type { CanonicalRow } from "#types"
40
+ import { alignRow } from "#utils"
41
+
40
42
  import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
41
43
 
42
44
  /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
@@ -45,7 +47,7 @@ import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
45
47
  would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
46
48
  extracted as named constants above. */
47
49
 
48
- const DEFAULT_COMMUNES = "/tmp/reg/fr-communes.tsv"
50
+ const DEFAULT_COMMUNES = tempRootPath("reg", "fr-communes.tsv")
49
51
  const LICENSE = "BAN (Base Adresse Nationale) commune+postcode tuples, rendered admin-split — see ingest SOURCE"
50
52
 
51
53
  /**