@mailwoman/corpus 8.6.0 → 9.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (497) hide show
  1. package/README.md +2 -2
  2. package/data/PROVENANCE.md +214 -0
  3. package/data/sub-venue-lexicon.json +18269 -0
  4. package/out/src/adapters/ban/adapter.d.ts +5 -4
  5. package/out/src/adapters/ban/adapter.d.ts.map +1 -1
  6. package/out/src/adapters/ban/adapter.js +62 -69
  7. package/out/src/adapters/ban/adapter.js.map +1 -1
  8. package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
  9. package/out/src/adapters/ban/street-decompose.js +18 -25
  10. package/out/src/adapters/ban/street-decompose.js.map +1 -1
  11. package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -7
  12. package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
  13. package/out/src/adapters/fcc-bdc/adapter.js +4 -26
  14. package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
  15. package/out/src/adapters/geonames/adapter.d.ts +1 -1
  16. package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
  17. package/out/src/adapters/geonames/adapter.js +72 -78
  18. package/out/src/adapters/geonames/adapter.js.map +1 -1
  19. package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
  20. package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
  21. package/out/src/adapters/geonames-postal/adapter.js +46 -51
  22. package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
  23. package/out/src/adapters/gnaf/adapter.d.ts +1 -1
  24. package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
  25. package/out/src/adapters/gnaf/adapter.js +6 -9
  26. package/out/src/adapters/gnaf/adapter.js.map +1 -1
  27. package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
  28. package/out/src/adapters/gnaf/assemble.js +7 -12
  29. package/out/src/adapters/gnaf/assemble.js.map +1 -1
  30. package/out/src/adapters/index.d.ts +1 -1
  31. package/out/src/adapters/index.d.ts.map +1 -1
  32. package/out/src/adapters/index.js +1 -1
  33. package/out/src/adapters/index.js.map +1 -1
  34. package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
  35. package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
  36. package/out/src/adapters/openaddresses/adapter.js +5 -10
  37. package/out/src/adapters/openaddresses/adapter.js.map +1 -1
  38. package/out/src/adapters/overture/adapter.d.ts +1 -1
  39. package/out/src/adapters/overture/adapter.d.ts.map +1 -1
  40. package/out/src/adapters/overture/adapter.js +5 -9
  41. package/out/src/adapters/overture/adapter.js.map +1 -1
  42. package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
  43. package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
  44. package/out/src/adapters/state-hi-schools/adapter.js +57 -75
  45. package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
  46. package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
  47. package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
  48. package/out/src/adapters/state-ia-contractors/adapter.js +60 -78
  49. package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
  50. package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
  51. package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
  52. package/out/src/adapters/state-ny-notaries/adapter.js +69 -87
  53. package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
  54. package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
  55. package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
  56. package/out/src/adapters/state-tx-notaries/adapter.js +71 -89
  57. package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
  58. package/out/src/adapters/synth-po-box/adapter.d.ts +3 -3
  59. package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
  60. package/out/src/adapters/synth-po-box/adapter.js +10 -18
  61. package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
  62. package/out/src/adapters/tiger/adapter.d.ts +1 -1
  63. package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
  64. package/out/src/adapters/tiger/adapter.js +2 -2
  65. package/out/src/adapters/tiger/adapter.js.map +1 -1
  66. package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
  67. package/out/src/adapters/tiger/street-decompose.js +18 -27
  68. package/out/src/adapters/tiger/street-decompose.js.map +1 -1
  69. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +4 -4
  70. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
  71. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +64 -82
  72. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
  73. package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
  74. package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
  75. package/out/src/adapters/usgov-imls-pls/adapter.js +62 -84
  76. package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
  77. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
  78. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
  79. package/out/src/adapters/usgov-irs-bmf/adapter.js +59 -66
  80. package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
  81. package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
  82. package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
  83. package/out/src/adapters/usgov-nad/adapter.js +6 -9
  84. package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
  85. package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
  86. package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
  87. package/out/src/adapters/usgov-nppes/adapter.js +60 -78
  88. package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
  89. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
  90. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
  91. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +53 -73
  92. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
  93. package/out/src/{adapter.d.ts → adapters/utils/index.d.ts} +43 -2
  94. package/out/src/adapters/utils/index.d.ts.map +1 -0
  95. package/out/src/{adapter.js → adapters/utils/index.js} +47 -2
  96. package/out/src/adapters/utils/index.js.map +1 -0
  97. package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
  98. package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
  99. package/out/src/adapters/wof-admin-jp/adapter.js +24 -6
  100. package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
  101. package/out/src/adapters/wof-admin-json/adapter.d.ts +2 -2
  102. package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
  103. package/out/src/adapters/wof-admin-json/adapter.js +2 -2
  104. package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
  105. package/out/src/adapters/wof-postalcode-json/adapter.d.ts +2 -2
  106. package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
  107. package/out/src/adapters/wof-postalcode-json/adapter.js +2 -2
  108. package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
  109. package/out/src/build.d.ts +3 -4
  110. package/out/src/build.d.ts.map +1 -1
  111. package/out/src/build.js +6 -8
  112. package/out/src/build.js.map +1 -1
  113. package/out/src/index.d.ts +3 -17
  114. package/out/src/index.d.ts.map +1 -1
  115. package/out/src/index.js +3 -17
  116. package/out/src/index.js.map +1 -1
  117. package/out/src/name-prone-us-suffixes.d.ts +12 -0
  118. package/out/src/name-prone-us-suffixes.d.ts.map +1 -0
  119. package/out/src/name-prone-us-suffixes.js +12 -0
  120. package/out/src/name-prone-us-suffixes.js.map +1 -0
  121. package/out/src/parquet-wrapper/reader.d.ts +8 -0
  122. package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
  123. package/out/src/parquet-wrapper/reader.js +16 -0
  124. package/out/src/parquet-wrapper/reader.js.map +1 -1
  125. package/out/src/parquet-wrapper/schema.d.ts +4 -0
  126. package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
  127. package/out/src/parquet-wrapper/schema.js.map +1 -1
  128. package/out/src/runner.d.ts +2 -2
  129. package/out/src/runner.d.ts.map +1 -1
  130. package/out/src/runner.js +1 -1
  131. package/out/src/runner.js.map +1 -1
  132. package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
  133. package/out/src/shard-recipes/anchor-absorption.js +5 -4
  134. package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
  135. package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
  136. package/out/src/shard-recipes/boundary-stress.js +2 -2
  137. package/out/src/shard-recipes/boundary-stress.js.map +1 -1
  138. package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
  139. package/out/src/shard-recipes/country-balanced.js +75 -75
  140. package/out/src/shard-recipes/country-balanced.js.map +1 -1
  141. package/out/src/shard-recipes/fr-admin-split.js +2 -2
  142. package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
  143. package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
  144. package/out/src/shard-recipes/fr-fragment.js +8 -5
  145. package/out/src/shard-recipes/fr-fragment.js.map +1 -1
  146. package/out/src/shard-recipes/fr-lieudit.js +2 -2
  147. package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
  148. package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
  149. package/out/src/shard-recipes/fr-order.js +18 -72
  150. package/out/src/shard-recipes/fr-order.js.map +1 -1
  151. package/out/src/shard-recipes/german.d.ts.map +1 -1
  152. package/out/src/shard-recipes/german.js +17 -70
  153. package/out/src/shard-recipes/german.js.map +1 -1
  154. package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
  155. package/out/src/shard-recipes/house-venue.js +1 -1
  156. package/out/src/shard-recipes/house-venue.js.map +1 -1
  157. package/out/src/shard-recipes/index.d.ts.map +1 -1
  158. package/out/src/shard-recipes/index.js +4 -1
  159. package/out/src/shard-recipes/index.js.map +1 -1
  160. package/out/src/shard-recipes/intersection.d.ts +2 -2
  161. package/out/src/shard-recipes/intersection.d.ts.map +1 -1
  162. package/out/src/shard-recipes/intersection.js +28 -72
  163. package/out/src/shard-recipes/intersection.js.map +1 -1
  164. package/out/src/shard-recipes/locale.d.ts +5 -5
  165. package/out/src/shard-recipes/locale.d.ts.map +1 -1
  166. package/out/src/shard-recipes/locale.js +25 -35
  167. package/out/src/shard-recipes/locale.js.map +1 -1
  168. package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
  169. package/out/src/shard-recipes/no-fragment.js +8 -5
  170. package/out/src/shard-recipes/no-fragment.js.map +1 -1
  171. package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
  172. package/out/src/shard-recipes/no-street-led.js +8 -5
  173. package/out/src/shard-recipes/no-street-led.js.map +1 -1
  174. package/out/src/shard-recipes/no-street.d.ts.map +1 -1
  175. package/out/src/shard-recipes/no-street.js +1 -1
  176. package/out/src/shard-recipes/no-street.js.map +1 -1
  177. package/out/src/shard-recipes/po-box-cedex.d.ts +3 -3
  178. package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
  179. package/out/src/shard-recipes/po-box-cedex.js +101 -127
  180. package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
  181. package/out/src/shard-recipes/po-box.d.ts.map +1 -1
  182. package/out/src/shard-recipes/po-box.js +1 -1
  183. package/out/src/shard-recipes/po-box.js.map +1 -1
  184. package/out/src/shard-recipes/scaffold.d.ts +58 -9
  185. package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
  186. package/out/src/shard-recipes/scaffold.js +87 -29
  187. package/out/src/shard-recipes/scaffold.js.map +1 -1
  188. package/out/src/shard-recipes/street-affix.d.ts +54 -0
  189. package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
  190. package/out/src/shard-recipes/street-affix.js +253 -111
  191. package/out/src/shard-recipes/street-affix.js.map +1 -1
  192. package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
  193. package/out/src/shard-recipes/street-bare.js +3 -3
  194. package/out/src/shard-recipes/street-bare.js.map +1 -1
  195. package/out/src/shard-recipes/street.d.ts.map +1 -1
  196. package/out/src/shard-recipes/street.js +2 -2
  197. package/out/src/shard-recipes/street.js.map +1 -1
  198. package/out/src/shard-recipes/sub-venue-sources.d.ts +228 -0
  199. package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -0
  200. package/out/src/shard-recipes/sub-venue-sources.js +664 -0
  201. package/out/src/shard-recipes/sub-venue-sources.js.map +1 -0
  202. package/out/src/shard-recipes/sub-venue.d.ts +269 -0
  203. package/out/src/shard-recipes/sub-venue.d.ts.map +1 -0
  204. package/out/src/shard-recipes/sub-venue.js +709 -0
  205. package/out/src/shard-recipes/sub-venue.js.map +1 -0
  206. package/out/src/shard-recipes/unit.d.ts +1 -1
  207. package/out/src/shard-recipes/unit.d.ts.map +1 -1
  208. package/out/src/shard-recipes/unit.js +26 -74
  209. package/out/src/shard-recipes/unit.js.map +1 -1
  210. package/out/src/{synthesize-anchor-absorption.d.ts → synthesizers/anchor-absorption.d.ts} +1 -1
  211. package/out/src/synthesizers/anchor-absorption.d.ts.map +1 -0
  212. package/out/src/{synthesize-anchor-absorption.js → synthesizers/anchor-absorption.js} +1 -1
  213. package/out/src/synthesizers/anchor-absorption.js.map +1 -0
  214. package/out/src/{synthesize-boundary-stress.d.ts → synthesizers/boundary-stress.d.ts} +3 -3
  215. package/out/src/synthesizers/boundary-stress.d.ts.map +1 -0
  216. package/out/src/{synthesize-boundary-stress.js → synthesizers/boundary-stress.js} +2 -2
  217. package/out/src/synthesizers/boundary-stress.js.map +1 -0
  218. package/out/src/{synthesize-german.d.ts → synthesizers/german.d.ts} +2 -2
  219. package/out/src/synthesizers/german.d.ts.map +1 -0
  220. package/out/src/{synthesize-german.js → synthesizers/german.js} +2 -2
  221. package/out/src/synthesizers/german.js.map +1 -0
  222. package/out/src/{synthesize-house-venue.d.ts → synthesizers/house-venue.d.ts} +4 -4
  223. package/out/src/synthesizers/house-venue.d.ts.map +1 -0
  224. package/out/src/{synthesize-house-venue.js → synthesizers/house-venue.js} +4 -4
  225. package/out/src/synthesizers/house-venue.js.map +1 -0
  226. package/out/src/{synthesize-intersection.d.ts → synthesizers/intersection.d.ts} +2 -2
  227. package/out/src/synthesizers/intersection.d.ts.map +1 -0
  228. package/out/src/{synthesize-intersection.js → synthesizers/intersection.js} +1 -1
  229. package/out/src/synthesizers/intersection.js.map +1 -0
  230. package/out/src/{synthesize-no-street.d.ts → synthesizers/no-street.d.ts} +3 -3
  231. package/out/src/synthesizers/no-street.d.ts.map +1 -0
  232. package/out/src/{synthesize-no-street.js → synthesizers/no-street.js} +2 -2
  233. package/out/src/synthesizers/no-street.js.map +1 -0
  234. package/out/src/{synthesize-po-box.d.ts → synthesizers/po-box.d.ts} +2 -2
  235. package/out/src/synthesizers/po-box.d.ts.map +1 -0
  236. package/out/src/{synthesize-po-box.js → synthesizers/po-box.js} +1 -1
  237. package/out/src/synthesizers/po-box.js.map +1 -0
  238. package/out/src/{synthesize-street.d.ts → synthesizers/street.d.ts} +2 -2
  239. package/out/src/synthesizers/street.d.ts.map +1 -0
  240. package/out/src/{synthesize-street.js → synthesizers/street.js} +2 -2
  241. package/out/src/synthesizers/street.js.map +1 -0
  242. package/out/src/{synthesize.d.ts → synthesizers/utils.d.ts} +3 -3
  243. package/out/src/synthesizers/utils.d.ts.map +1 -0
  244. package/out/src/{synthesize.js → synthesizers/utils.js} +4 -15
  245. package/out/src/synthesizers/utils.js.map +1 -0
  246. package/out/src/tools/align-shard.d.ts.map +1 -1
  247. package/out/src/tools/align-shard.js +4 -8
  248. package/out/src/tools/align-shard.js.map +1 -1
  249. package/out/src/tools/audit.d.ts.map +1 -1
  250. package/out/src/tools/audit.js +4 -4
  251. package/out/src/tools/audit.js.map +1 -1
  252. package/out/src/tools/corpus-stats.d.ts +2 -2
  253. package/out/src/tools/corpus-stats.d.ts.map +1 -1
  254. package/out/src/tools/corpus-stats.js +18 -31
  255. package/out/src/tools/corpus-stats.js.map +1 -1
  256. package/out/src/tools/fetch/download.d.ts.map +1 -1
  257. package/out/src/tools/fetch/download.js +5 -6
  258. package/out/src/tools/fetch/download.js.map +1 -1
  259. package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
  260. package/out/src/tools/fetch/imls-pls.js +3 -15
  261. package/out/src/tools/fetch/imls-pls.js.map +1 -1
  262. package/out/src/tools/fetch/index.d.ts +14 -1
  263. package/out/src/tools/fetch/index.d.ts.map +1 -1
  264. package/out/src/tools/fetch/index.js +14 -1
  265. package/out/src/tools/fetch/index.js.map +1 -1
  266. package/out/src/tools/fetch/nad.d.ts +1 -1
  267. package/out/src/tools/fetch/nad.js +1 -1
  268. package/out/src/tools/fetch/nppes.d.ts.map +1 -1
  269. package/out/src/tools/fetch/nppes.js +8 -14
  270. package/out/src/tools/fetch/nppes.js.map +1 -1
  271. package/out/src/tools/fetch/openaddresses.d.ts +1 -1
  272. package/out/src/tools/fetch/openaddresses.js +1 -1
  273. package/out/src/tools/fetch/ourairports.d.ts +45 -0
  274. package/out/src/tools/fetch/ourairports.d.ts.map +1 -0
  275. package/out/src/tools/fetch/ourairports.js +123 -0
  276. package/out/src/tools/fetch/ourairports.js.map +1 -0
  277. package/out/src/tools/fetch/wikidata-subvenue.d.ts +171 -0
  278. package/out/src/tools/fetch/wikidata-subvenue.d.ts.map +1 -0
  279. package/out/src/tools/fetch/wikidata-subvenue.js +275 -0
  280. package/out/src/tools/fetch/wikidata-subvenue.js.map +1 -0
  281. package/out/src/tools/golden-expand.d.ts.map +1 -1
  282. package/out/src/tools/golden-expand.js +18 -16
  283. package/out/src/tools/golden-expand.js.map +1 -1
  284. package/out/src/tools/golden-promote.d.ts.map +1 -1
  285. package/out/src/tools/golden-promote.js +12 -6
  286. package/out/src/tools/golden-promote.js.map +1 -1
  287. package/out/src/tools/golden-relabel-street.d.ts +196 -0
  288. package/out/src/tools/golden-relabel-street.d.ts.map +1 -0
  289. package/out/src/tools/golden-relabel-street.js +440 -0
  290. package/out/src/tools/golden-relabel-street.js.map +1 -0
  291. package/out/src/tools/index.d.ts +4 -0
  292. package/out/src/tools/index.d.ts.map +1 -1
  293. package/out/src/tools/index.js +4 -0
  294. package/out/src/tools/index.js.map +1 -1
  295. package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
  296. package/out/src/tools/jsonl-to-parquet.js +3 -2
  297. package/out/src/tools/jsonl-to-parquet.js.map +1 -1
  298. package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
  299. package/out/src/tools/lint-shard-vocab.js +3 -2
  300. package/out/src/tools/lint-shard-vocab.js.map +1 -1
  301. package/out/src/tools/lint-shard.d.ts +3 -3
  302. package/out/src/tools/lint-shard.d.ts.map +1 -1
  303. package/out/src/tools/lint-shard.js +23 -32
  304. package/out/src/tools/lint-shard.js.map +1 -1
  305. package/out/src/tools/overlay-manifest.d.ts.map +1 -1
  306. package/out/src/tools/overlay-manifest.js +2 -1
  307. package/out/src/tools/overlay-manifest.js.map +1 -1
  308. package/out/src/tools/overture-subvenue.d.ts +112 -0
  309. package/out/src/tools/overture-subvenue.d.ts.map +1 -0
  310. package/out/src/tools/overture-subvenue.js +144 -0
  311. package/out/src/tools/overture-subvenue.js.map +1 -0
  312. package/out/src/tools/shard-kryptonite.d.ts +1 -1
  313. package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
  314. package/out/src/tools/shard-kryptonite.js +6 -6
  315. package/out/src/tools/shard-kryptonite.js.map +1 -1
  316. package/out/src/tools/shard-translit.d.ts +6 -3
  317. package/out/src/tools/shard-translit.d.ts.map +1 -1
  318. package/out/src/tools/shard-translit.js +9 -8
  319. package/out/src/tools/shard-translit.js.map +1 -1
  320. package/out/src/tools/sub-venue-lexicon.d.ts +507 -0
  321. package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -0
  322. package/out/src/tools/sub-venue-lexicon.js +817 -0
  323. package/out/src/tools/sub-venue-lexicon.js.map +1 -0
  324. package/out/src/tools/sub-venue-promotions.d.ts +94 -0
  325. package/out/src/tools/sub-venue-promotions.d.ts.map +1 -0
  326. package/out/src/tools/sub-venue-promotions.js +266 -0
  327. package/out/src/tools/sub-venue-promotions.js.map +1 -0
  328. package/out/src/{align.d.ts → utils/align.d.ts} +1 -1
  329. package/out/src/utils/align.d.ts.map +1 -0
  330. package/out/src/utils/align.js.map +1 -0
  331. package/out/src/{golden.d.ts → utils/golden.d.ts} +5 -0
  332. package/out/src/utils/golden.d.ts.map +1 -0
  333. package/out/src/{golden.js → utils/golden.js} +24 -10
  334. package/out/src/utils/golden.js.map +1 -0
  335. package/out/src/utils/index.d.ts +14 -0
  336. package/out/src/utils/index.d.ts.map +1 -0
  337. package/out/src/utils/index.js +14 -0
  338. package/out/src/utils/index.js.map +1 -0
  339. package/out/src/utils/license.d.ts.map +1 -0
  340. package/out/src/utils/license.js.map +1 -0
  341. package/out/src/{parquet.d.ts → utils/parquet.d.ts} +2 -2
  342. package/out/src/utils/parquet.d.ts.map +1 -0
  343. package/out/src/{parquet.js → utils/parquet.js} +3 -15
  344. package/out/src/utils/parquet.js.map +1 -0
  345. package/out/src/{split.d.ts → utils/split.d.ts} +8 -1
  346. package/out/src/utils/split.d.ts.map +1 -0
  347. package/out/src/{split.js → utils/split.js} +7 -0
  348. package/out/src/utils/split.js.map +1 -0
  349. package/out/src/utils/tokenize.d.ts.map +1 -0
  350. package/out/src/utils/tokenize.js.map +1 -0
  351. package/out/src/{wof-json.d.ts → utils/wof-json.d.ts} +2 -2
  352. package/out/src/utils/wof-json.d.ts.map +1 -0
  353. package/out/src/{wof-json.js → utils/wof-json.js} +5 -8
  354. package/out/src/utils/wof-json.js.map +1 -0
  355. package/package.json +66 -20
  356. package/src/adapters/ban/adapter.ts +58 -68
  357. package/src/adapters/ban/street-decompose.ts +17 -25
  358. package/src/adapters/fcc-bdc/adapter.ts +5 -36
  359. package/src/adapters/geonames/adapter.ts +67 -76
  360. package/src/adapters/geonames-postal/adapter.ts +45 -54
  361. package/src/adapters/gnaf/adapter.ts +7 -11
  362. package/src/adapters/gnaf/assemble.ts +7 -11
  363. package/src/adapters/index.ts +3 -2
  364. package/src/adapters/openaddresses/adapter.ts +6 -12
  365. package/src/adapters/overture/adapter.ts +6 -10
  366. package/src/adapters/state-hi-schools/adapter.ts +53 -77
  367. package/src/adapters/state-ia-contractors/adapter.ts +55 -80
  368. package/src/adapters/state-ny-notaries/adapter.ts +64 -89
  369. package/src/adapters/state-tx-notaries/adapter.ts +63 -88
  370. package/src/adapters/synth-po-box/adapter.ts +12 -23
  371. package/src/adapters/tiger/adapter.ts +4 -3
  372. package/src/adapters/tiger/street-decompose.ts +21 -31
  373. package/src/adapters/usgov-hrsa-fqhc/adapter.ts +64 -88
  374. package/src/adapters/usgov-imls-pls/adapter.ts +57 -86
  375. package/src/adapters/usgov-irs-bmf/adapter.ts +55 -66
  376. package/src/adapters/usgov-nad/adapter.ts +7 -11
  377. package/src/adapters/usgov-nppes/adapter.ts +54 -79
  378. package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +48 -75
  379. package/src/{adapter.ts → adapters/utils/index.ts} +59 -3
  380. package/src/adapters/wof-admin-jp/adapter.ts +35 -9
  381. package/src/adapters/wof-admin-json/adapter.ts +3 -4
  382. package/src/adapters/wof-postalcode-json/adapter.ts +3 -4
  383. package/src/build.ts +12 -11
  384. package/src/index.ts +3 -17
  385. package/src/name-prone-us-suffixes.ts +12 -0
  386. package/src/parquet-wrapper/reader.ts +19 -0
  387. package/src/parquet-wrapper/schema.ts +4 -0
  388. package/src/runner.ts +7 -2
  389. package/src/shard-recipes/anchor-absorption.ts +5 -4
  390. package/src/shard-recipes/boundary-stress.ts +6 -2
  391. package/src/shard-recipes/country-balanced.ts +77 -91
  392. package/src/shard-recipes/fr-admin-split.ts +3 -3
  393. package/src/shard-recipes/fr-fragment.ts +10 -7
  394. package/src/shard-recipes/fr-lieudit.ts +3 -3
  395. package/src/shard-recipes/fr-order.ts +18 -83
  396. package/src/shard-recipes/german.ts +17 -80
  397. package/src/shard-recipes/house-venue.ts +2 -1
  398. package/src/shard-recipes/index.ts +4 -1
  399. package/src/shard-recipes/intersection.ts +37 -74
  400. package/src/shard-recipes/locale.ts +31 -39
  401. package/src/shard-recipes/no-fragment.ts +10 -7
  402. package/src/shard-recipes/no-street-led.ts +10 -7
  403. package/src/shard-recipes/no-street.ts +2 -1
  404. package/src/shard-recipes/po-box-cedex.ts +127 -161
  405. package/src/shard-recipes/po-box.ts +6 -1
  406. package/src/shard-recipes/scaffold.ts +125 -27
  407. package/src/shard-recipes/street-affix.ts +335 -134
  408. package/src/shard-recipes/street-bare.ts +4 -3
  409. package/src/shard-recipes/street.ts +3 -2
  410. package/src/shard-recipes/sub-venue-sources.ts +895 -0
  411. package/src/shard-recipes/sub-venue.ts +982 -0
  412. package/src/shard-recipes/unit.ts +27 -85
  413. package/src/{synthesize-boundary-stress.ts → synthesizers/boundary-stress.ts} +2 -2
  414. package/src/{synthesize-german.ts → synthesizers/german.ts} +2 -2
  415. package/src/{synthesize-house-venue.ts → synthesizers/house-venue.ts} +4 -4
  416. package/src/{synthesize-intersection.ts → synthesizers/intersection.ts} +1 -1
  417. package/src/{synthesize-no-street.ts → synthesizers/no-street.ts} +2 -2
  418. package/src/{synthesize-po-box.ts → synthesizers/po-box.ts} +1 -1
  419. package/src/{synthesize-street.ts → synthesizers/street.ts} +2 -2
  420. package/src/{synthesize.ts → synthesizers/utils.ts} +4 -18
  421. package/src/tools/align-shard.ts +4 -7
  422. package/src/tools/audit.ts +6 -5
  423. package/src/tools/corpus-stats.ts +25 -33
  424. package/src/tools/fetch/download.ts +7 -5
  425. package/src/tools/fetch/imls-pls.ts +4 -18
  426. package/src/tools/fetch/index.ts +14 -1
  427. package/src/tools/fetch/nad.ts +1 -1
  428. package/src/tools/fetch/nppes.ts +9 -16
  429. package/src/tools/fetch/openaddresses.ts +1 -1
  430. package/src/tools/fetch/ourairports.ts +166 -0
  431. package/src/tools/fetch/wikidata-subvenue.ts +386 -0
  432. package/src/tools/golden-expand.ts +18 -13
  433. package/src/tools/golden-promote.ts +15 -6
  434. package/src/tools/golden-relabel-street.ts +685 -0
  435. package/src/tools/index.ts +4 -0
  436. package/src/tools/jsonl-to-parquet.ts +3 -2
  437. package/src/tools/lint-shard-vocab.ts +3 -2
  438. package/src/tools/lint-shard.ts +33 -36
  439. package/src/tools/overlay-manifest.ts +2 -1
  440. package/src/tools/overture-subvenue.ts +215 -0
  441. package/src/tools/shard-kryptonite.ts +8 -9
  442. package/src/tools/shard-translit.ts +22 -11
  443. package/src/tools/sub-venue-lexicon.ts +1250 -0
  444. package/src/tools/sub-venue-promotions.ts +330 -0
  445. package/src/{align.ts → utils/align.ts} +1 -1
  446. package/src/{golden.ts → utils/golden.ts} +25 -11
  447. package/src/utils/index.ts +14 -0
  448. package/src/{parquet.ts → utils/parquet.ts} +5 -19
  449. package/src/{split.ts → utils/split.ts} +8 -2
  450. package/src/{wof-json.ts → utils/wof-json.ts} +5 -8
  451. package/out/src/adapter.d.ts.map +0 -1
  452. package/out/src/adapter.js.map +0 -1
  453. package/out/src/align.d.ts.map +0 -1
  454. package/out/src/align.js.map +0 -1
  455. package/out/src/format.d.ts +0 -14
  456. package/out/src/format.d.ts.map +0 -1
  457. package/out/src/format.js +0 -14
  458. package/out/src/format.js.map +0 -1
  459. package/out/src/golden.d.ts.map +0 -1
  460. package/out/src/golden.js.map +0 -1
  461. package/out/src/license.d.ts.map +0 -1
  462. package/out/src/license.js.map +0 -1
  463. package/out/src/parquet.d.ts.map +0 -1
  464. package/out/src/parquet.js.map +0 -1
  465. package/out/src/split.d.ts.map +0 -1
  466. package/out/src/split.js.map +0 -1
  467. package/out/src/synthesize-anchor-absorption.d.ts.map +0 -1
  468. package/out/src/synthesize-anchor-absorption.js.map +0 -1
  469. package/out/src/synthesize-boundary-stress.d.ts.map +0 -1
  470. package/out/src/synthesize-boundary-stress.js.map +0 -1
  471. package/out/src/synthesize-german.d.ts.map +0 -1
  472. package/out/src/synthesize-german.js.map +0 -1
  473. package/out/src/synthesize-house-venue.d.ts.map +0 -1
  474. package/out/src/synthesize-house-venue.js.map +0 -1
  475. package/out/src/synthesize-intersection.d.ts.map +0 -1
  476. package/out/src/synthesize-intersection.js.map +0 -1
  477. package/out/src/synthesize-no-street.d.ts.map +0 -1
  478. package/out/src/synthesize-no-street.js.map +0 -1
  479. package/out/src/synthesize-po-box.d.ts.map +0 -1
  480. package/out/src/synthesize-po-box.js.map +0 -1
  481. package/out/src/synthesize-street.d.ts.map +0 -1
  482. package/out/src/synthesize-street.js.map +0 -1
  483. package/out/src/synthesize.d.ts.map +0 -1
  484. package/out/src/synthesize.js.map +0 -1
  485. package/out/src/tokenize.d.ts.map +0 -1
  486. package/out/src/tokenize.js.map +0 -1
  487. package/out/src/wof-json.d.ts.map +0 -1
  488. package/out/src/wof-json.js.map +0 -1
  489. package/src/format.ts +0 -14
  490. /package/out/src/{align.js → utils/align.js} +0 -0
  491. /package/out/src/{license.d.ts → utils/license.d.ts} +0 -0
  492. /package/out/src/{license.js → utils/license.js} +0 -0
  493. /package/out/src/{tokenize.d.ts → utils/tokenize.d.ts} +0 -0
  494. /package/out/src/{tokenize.js → utils/tokenize.js} +0 -0
  495. /package/src/{synthesize-anchor-absorption.ts → synthesizers/anchor-absorption.ts} +0 -0
  496. /package/src/{license.ts → utils/license.ts} +0 -0
  497. /package/src/{tokenize.ts → utils/tokenize.ts} +0 -0
@@ -24,15 +24,14 @@
24
24
  * legacy script used.
25
25
  */
26
26
 
27
- import { spawnSync } from "node:child_process"
28
-
29
27
  import { COUNTRY_SURFACE_FORMS, CountryNames } from "@mailwoman/codex/country"
30
28
  import type { ComponentTag } from "@mailwoman/core/types"
29
+ import { dataRootPath } from "@mailwoman/core/utils"
30
+ import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
31
+ import type { CanonicalRow } from "@mailwoman/corpus/types"
32
+ import { alignRow } from "@mailwoman/corpus/utils"
31
33
 
32
- import { stableSourceID } from "../adapter.ts"
33
- import { alignRow } from "../align.ts"
34
- import type { CanonicalRow } from "../types.ts"
35
- import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
34
+ import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
36
35
 
37
36
  // v2: the country TOKEN is decoupled from the skeleton's locale and drawn from a BROAD pool — every
38
37
  // ISO canonical name + every curated surface form (endonyms/abbrevs). Surface forms are over-weighted
@@ -72,24 +71,72 @@ interface CountrySource {
72
71
  * countrywide extracts (FR/IT/NL) read region from the CSV when present.
73
72
  */
74
73
  const SOURCES: readonly CountrySource[] = [
75
- { zip: "/tmp/oa-cache/us__ia__statewide.zip", csv: "us/ia/statewide.csv", iso2: "US", region: "IA", order: "us" },
76
- { zip: "/tmp/oa-cache/us__il__cook.zip", csv: "us/il/cook.csv", iso2: "US", region: "IL", order: "us" },
77
- { zip: "/tmp/oa-cache/us__mt__statewide.zip", csv: "us/mt/statewide.csv", iso2: "US", region: "MT", order: "us" },
78
- { zip: "/tmp/oa-cache/us__sd__statewide.zip", csv: "us/sd/statewide.csv", iso2: "US", region: "SD", order: "us" },
79
- { zip: "/tmp/oa-cache/de__sn__statewide.zip", csv: "de/sn/statewide.csv", iso2: "DE", region: "", order: "eu" },
80
- { zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv", iso2: "FR", region: "", order: "fr" },
74
+ {
75
+ zip: dataRootPath("oa-cache", "us__ia__statewide.zip"),
76
+ csv: "us/ia/statewide.csv",
77
+ iso2: "US",
78
+ region: "IA",
79
+ order: "us",
80
+ },
81
+ { zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", iso2: "US", region: "IL", order: "us" },
82
+ {
83
+ zip: dataRootPath("oa-cache", "us__mt__statewide.zip"),
84
+ csv: "us/mt/statewide.csv",
85
+ iso2: "US",
86
+ region: "MT",
87
+ order: "us",
88
+ },
89
+ {
90
+ zip: dataRootPath("oa-cache", "us__sd__statewide.zip"),
91
+ csv: "us/sd/statewide.csv",
92
+ iso2: "US",
93
+ region: "SD",
94
+ order: "us",
95
+ },
96
+ {
97
+ zip: dataRootPath("oa-cache", "de__sn__statewide.zip"),
98
+ csv: "de/sn/statewide.csv",
99
+ iso2: "DE",
100
+ region: "",
101
+ order: "eu",
102
+ },
103
+ {
104
+ zip: dataRootPath("oa-cache", "fr__countrywide.zip"),
105
+ csv: "fr/countrywide.csv",
106
+ iso2: "FR",
107
+ region: "",
108
+ order: "fr",
109
+ },
81
110
  // ES uses the Spanish IGN schema, not the OA standard columns — skipped here (codex still recognizes
82
111
  // "España"/"Spain"). A dedicated IGN adapter is a follow-up.
83
- { zip: "/tmp/oa-cache/it__countrywide.zip", csv: "it/countrywide.csv", iso2: "IT", region: "", order: "eu" },
84
- { zip: "/tmp/oa-cache/nl__countrywide.zip", csv: "nl/countrywide.csv", iso2: "NL", region: "", order: "eu" },
112
+ {
113
+ zip: dataRootPath("oa-cache", "it__countrywide.zip"),
114
+ csv: "it/countrywide.csv",
115
+ iso2: "IT",
116
+ region: "",
117
+ order: "eu",
118
+ },
119
+ {
120
+ zip: dataRootPath("oa-cache", "nl__countrywide.zip"),
121
+ csv: "nl/countrywide.csv",
122
+ iso2: "NL",
123
+ region: "",
124
+ order: "eu",
125
+ },
85
126
  ]
86
127
 
87
128
  /**
88
129
  * Held-out for --golden: Vermont (US holdout) + Berlin (DE holdout) — geographic split, never trained.
89
130
  */
90
131
  const EVAL_SOURCES: readonly CountrySource[] = [
91
- { zip: "/tmp/oa-cache/us__vt__statewide.zip", csv: "us/vt/statewide.csv", iso2: "US", region: "VT", order: "us" },
92
- { zip: "/tmp/oa-cache/de__berlin.zip", csv: "de/berlin.csv", iso2: "DE", region: "", order: "eu" },
132
+ {
133
+ zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
134
+ csv: "us/vt/statewide.csv",
135
+ iso2: "US",
136
+ region: "VT",
137
+ order: "us",
138
+ },
139
+ { zip: dataRootPath("oa-cache", "de__berlin.zip"), csv: "de/berlin.csv", iso2: "DE", region: "", order: "eu" },
93
140
  ]
94
141
 
95
142
  /**
@@ -105,81 +152,20 @@ interface CountryTuple {
105
152
  order: string
106
153
  }
107
154
 
108
- function splitCSV(line: string): string[] {
109
- const out: string[] = []
110
-
111
- let cur = "",
112
- inQ = false
113
-
114
- for (let i = 0; i < line.length; i++) {
115
- const c = line[i]
116
-
117
- if (inQ) {
118
- if (c === '"') {
119
- if (line[i + 1] === '"') {
120
- cur += '"'
121
-
122
- i++
123
- } else {
124
- inQ = false
125
- }
126
- } else {
127
- cur += c
128
- }
129
- } else if (c === '"') {
130
- inQ = true
131
- } else if (c === ",") {
132
- out.push(cur)
133
- cur = ""
134
- } else {
135
- cur += c
136
- }
137
- }
138
-
139
- out.push(cur)
140
-
141
- return out
142
- }
143
-
144
- function readTuples(source: CountrySource, limit: number): CountryTuple[] {
145
- // countrywide extracts (FR/IT/NL) are GB-scale — cap the bytes with `head` (read ~8 lines per wanted
146
- // tuple to survive dedup/skips) so the toString stays under V8's string limit.
147
- const maxLines = Math.max(limit * 8, 20_000) + 1
148
-
149
- const r = spawnSync("bash", ["-c", `unzip -p "${source.zip}" "${source.csv}" | head -n ${maxLines}`], {
150
- maxBuffer: 1024 * 1024 * 1024,
151
- encoding: "buffer",
152
- })
153
-
154
- if (r.status !== 0) {
155
- console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
156
-
157
- return []
158
- }
159
-
160
- const lines = r.stdout.toString("utf8").split(/\r?\n/)
161
-
162
- if (lines.length < 2) return []
163
- const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
164
- const idx = (n: string): number => header.indexOf(n)
165
-
166
- const iNum = idx("number"),
167
- iStreet = idx("street"),
168
- iCity = idx("city"),
169
- iRegion = idx("region"),
170
- iPost = idx("postcode")
171
-
172
- const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
155
+ /**
156
+ * The countrywide extracts (FR/IT/NL) are GB-scale, so this reads only as far as `limit` distinct tuples: the `break`
157
+ * closes the reader, which releases the archive without inflating the rest of the member.
158
+ */
159
+ async function readTuples(source: CountrySource, limit: number): Promise<CountryTuple[]> {
173
160
  const tuples: CountryTuple[] = []
174
161
  const seen = new Set<string>()
175
162
 
176
- for (let li = 1; li < lines.length && tuples.length < limit; li++) {
177
- if (!lines[li]) continue
178
- const cells = splitCSV(lines[li]!)
163
+ for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
164
+ if (tuples.length >= limit) break
179
165
 
180
- const street = get(cells, iStreet),
181
- locality = get(cells, iCity),
182
- house_number = get(cells, iNum)
166
+ const street = row.street ?? "",
167
+ locality = row.city ?? "",
168
+ house_number = row.number ?? ""
183
169
 
184
170
  if (!street || !locality || !house_number) continue
185
171
  const key = `${house_number}|${street}|${locality}`.toLowerCase()
@@ -191,8 +177,8 @@ function readTuples(source: CountrySource, limit: number): CountryTuple[] {
191
177
  house_number,
192
178
  street,
193
179
  locality,
194
- region: get(cells, iRegion) || source.region,
195
- postcode: get(cells, iPost),
180
+ region: row.region || source.region,
181
+ postcode: row.postcode ?? "",
196
182
  iso2: source.iso2,
197
183
  order: source.order,
198
184
  })
@@ -450,7 +436,7 @@ export const countryBalancedRecipe: ShardRecipe = {
450
436
  const pool: CountryTuple[] = []
451
437
 
452
438
  for (const s of sources) {
453
- const t = readTuples(s, perSource)
439
+ const t = await readTuples(s, perSource)
454
440
 
455
441
  console.error(` ${s.csv} (${s.iso2}): ${t.length} tuples`)
456
442
 
@@ -460,7 +446,7 @@ export const countryBalancedRecipe: ShardRecipe = {
460
446
  }
461
447
 
462
448
  if (!pool.length) {
463
- throw new Error("No tuples — are the cached OA zips present in /tmp/oa-cache?")
449
+ throw new Error(`No tuples — are the cached OA zips present in ${dataRootPath("oa-cache")}?`)
464
450
  }
465
451
 
466
452
  let emitted = 0
@@ -32,11 +32,11 @@
32
32
 
33
33
  import { departementForCodePostal } from "@mailwoman/codex/fr"
34
34
  import type { ComponentTag } from "@mailwoman/core/types"
35
+ import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
36
+ import type { CanonicalRow } from "@mailwoman/corpus/types"
37
+ import { alignRow } from "@mailwoman/corpus/utils"
35
38
  import { CSVSpliterator, Delimiters } from "spliterator"
36
39
 
37
- import { stableSourceID } from "../adapter.ts"
38
- import { alignRow } from "../align.ts"
39
- import type { CanonicalRow } from "../types.ts"
40
40
  import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
41
41
 
42
42
  /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
@@ -68,7 +68,7 @@
68
68
  * blow-up). Disable that mask for any run including this shard.
69
69
  */
70
70
 
71
- import { readFileSync } from "node:fs"
71
+ import { TextSpliterator } from "spliterator"
72
72
 
73
73
  import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
74
74
  import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
@@ -187,12 +187,15 @@ export const frFragmentRecipe: ShardRecipe = {
187
187
  )
188
188
  }
189
189
 
190
- const excluded = new Set(
191
- readFileSync(excludePath, "utf8")
192
- .split("\n")
193
- .map((line) => line.trim())
194
- .filter((line) => line && !line.startsWith("#"))
195
- )
190
+ const excluded = new Set<string>()
191
+
192
+ for await (const line of TextSpliterator.fromAsync(excludePath)) {
193
+ const trimmed = line.trim()
194
+
195
+ if (trimmed && !trimmed.startsWith("#")) {
196
+ excluded.add(trimmed)
197
+ }
198
+ }
196
199
 
197
200
  if (!excluded.size) throw new Error(`fr-fragment: --exclude-surfaces "${excludePath}" listed no surfaces`)
198
201
 
@@ -31,11 +31,11 @@ import { extractBANAddrPoints } from "@mailwoman/ban/sdk"
31
31
  import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country"
32
32
  import type { ComponentTag } from "@mailwoman/core/types"
33
33
  import { dataRootPath } from "@mailwoman/core/utils"
34
+ import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
35
+ import type { CanonicalRow } from "@mailwoman/corpus/types"
36
+ import { alignRow } from "@mailwoman/corpus/utils"
34
37
 
35
- import { stableSourceID } from "../adapter.ts"
36
38
  import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
37
- import { alignRow } from "../align.ts"
38
- import type { CanonicalRow } from "../types.ts"
39
39
  import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
40
40
 
41
41
  /**
@@ -30,14 +30,13 @@
30
30
  * inlined the variant logic and never called it).
31
31
  */
32
32
 
33
- import { spawnSync } from "node:child_process"
34
-
35
33
  import type { ComponentTag } from "@mailwoman/core/types"
34
+ import { dataRootPath } from "@mailwoman/core/utils"
35
+ import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
36
+ import type { CanonicalRow } from "@mailwoman/corpus/types"
37
+ import { alignRow } from "@mailwoman/corpus/utils"
36
38
 
37
- import { stableSourceID } from "../adapter.ts"
38
- import { alignRow } from "../align.ts"
39
- import type { CanonicalRow } from "../types.ts"
40
- import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
39
+ import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
41
40
 
42
41
  /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
43
42
  cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
@@ -45,7 +44,7 @@ import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
45
44
  would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
46
45
  extracted as named constants above. */
47
46
 
48
- const SOURCE = { zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv" }
47
+ const SOURCE = { zip: dataRootPath("oa-cache", "fr__countrywide.zip"), csv: "fr/countrywide.csv" }
49
48
 
50
49
  /**
51
50
  * Ordinal suffixes used in French house numbers (BAN corpus), to cover the "8 bis" sub-mode.
@@ -71,84 +70,20 @@ interface FrTuple {
71
70
  }
72
71
 
73
72
  /**
74
- * Minimal RFC-4180-ish splitter (handles quoted fields with doubled-quote escaping).
75
- */
76
- function splitCSV(line: string): string[] {
77
- const out: string[] = []
78
- let cur = ""
79
- let inQ = false
80
-
81
- for (let i = 0; i < line.length; i++) {
82
- const c = line[i]
83
-
84
- if (inQ) {
85
- if (c === '"') {
86
- if (line[i + 1] === '"') {
87
- cur += '"'
88
-
89
- i++
90
- } else {
91
- inQ = false
92
- }
93
- } else {
94
- cur += c
95
- }
96
- } else if (c === '"') {
97
- inQ = true
98
- } else if (c === ",") {
99
- out.push(cur)
100
- cur = ""
101
- } else {
102
- cur += c
103
- }
104
- }
105
-
106
- out.push(cur)
107
-
108
- return out
109
- }
110
-
111
- /**
112
- * Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale; cap with `head` to stay under V8's
113
- * string limit. Only keeps rows with a house_number (the shard's core signal) and a postcode (required for
114
- * reversed-order rendering to be meaningful).
73
+ * Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale, so this reads only as far as `limit`
74
+ * distinct tuples — the `break` closes the reader and releases the archive. Only keeps rows with a house_number (the
75
+ * shard's core signal) and a postcode (required for reversed-order rendering to be meaningful).
115
76
  */
116
- function readTuples(limit: number): FrTuple[] {
117
- const maxLines = Math.max(limit * 8, 40_000) + 1
118
-
119
- const r = spawnSync("bash", ["-c", `unzip -p "${SOURCE.zip}" "${SOURCE.csv}" | head -n ${maxLines}`], {
120
- maxBuffer: 1024 * 1024 * 1024,
121
- encoding: "buffer",
122
- })
123
-
124
- if (r.status !== 0) {
125
- console.error(` WARN: unzip failed for ${SOURCE.zip} (status ${r.status})`)
126
-
127
- return []
128
- }
129
-
130
- const lines = r.stdout.toString("utf8").split(/\r?\n/)
131
-
132
- if (lines.length < 2) return []
133
- const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
134
- const idx = (name: string): number => header.indexOf(name)
135
-
136
- const iNum = idx("number"),
137
- iStreet = idx("street"),
138
- iCity = idx("city"),
139
- iPost = idx("postcode")
140
-
141
- const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
77
+ async function readTuples(limit: number): Promise<FrTuple[]> {
142
78
  const tuples: FrTuple[] = []
143
79
  const seen = new Set<string>()
144
80
 
145
- for (let li = 1; li < lines.length && tuples.length < limit; li++) {
146
- if (!lines[li]) continue
147
- const cells = splitCSV(lines[li]!)
148
- const street = get(cells, iStreet)
149
- const locality = get(cells, iCity)
150
- const house_number = get(cells, iNum)
151
- const postcode = get(cells, iPost)
81
+ for await (const row of readZippedCSVRecords(SOURCE.zip, SOURCE.csv)) {
82
+ if (tuples.length >= limit) break
83
+ const street = row.street ?? ""
84
+ const locality = row.city ?? ""
85
+ const house_number = row.number ?? ""
86
+ const postcode = row.postcode ?? ""
152
87
 
153
88
  // Require all four fields: HN is the signal; postcode drives reversed-order variants.
154
89
  if (!street || !locality || !house_number || !postcode) continue
@@ -211,12 +146,12 @@ export const frOrderRecipe: ShardRecipe = {
211
146
 
212
147
  // Over-read from the CSV so the dedup + filter pass can fill `count` rows.
213
148
  const poolLimit = Math.max(count * 8, 40_000)
214
- const pool = readTuples(poolLimit)
149
+ const pool = await readTuples(poolLimit)
215
150
 
216
151
  console.error(` ${SOURCE.csv}: ${pool.length} unique tuples (capped read)`)
217
152
 
218
153
  if (!pool.length) {
219
- throw new Error("No FR tuples found — is /tmp/oa-cache/fr__countrywide.zip present?")
154
+ throw new Error(`No FR tuples found — is ${SOURCE.zip} present?`)
220
155
  }
221
156
 
222
157
  let emitted = 0
@@ -15,12 +15,12 @@
15
15
  * docs/articles/evals/resolver-geo/2026-06-06-anchor-pilot.md (the order-artifact correction).
16
16
  */
17
17
 
18
- import { spawnSync } from "node:child_process"
18
+ import { dataRootPath } from "@mailwoman/core/utils"
19
+ import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
20
+ import { synthesizeGermanRow, type LocaleBaseTuple } from "@mailwoman/corpus/synthesizers/german"
21
+ import { alignRow } from "@mailwoman/corpus/utils"
19
22
 
20
- import { stableSourceID } from "../adapter.ts"
21
- import { alignRow } from "../align.ts"
22
- import { synthesizeGermanRow, type LocaleBaseTuple } from "../synthesize-german.ts"
23
- import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
23
+ import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
24
24
 
25
25
  /**
26
26
  * A German OA source (cached zip) + the Bundesland the file covers (OA's REGION column is empty for DE).
@@ -37,89 +37,26 @@ interface GermanSource {
37
37
  * Berlin (a city-state, region==locality); sn/statewide → Sachsen.
38
38
  */
39
39
  const SOURCES: GermanSource[] = [
40
- { zip: "/tmp/oa-cache/de__berlin.zip", csv: "de/berlin.csv", region: "Berlin" },
41
- { zip: "/tmp/oa-cache/de__sn__statewide.zip", csv: "de/sn/statewide.csv", region: "Sachsen" },
40
+ { zip: dataRootPath("oa-cache", "de__berlin.zip"), csv: "de/berlin.csv", region: "Berlin" },
41
+ { zip: dataRootPath("oa-cache", "de__sn__statewide.zip"), csv: "de/sn/statewide.csv", region: "Sachsen" },
42
42
  ]
43
43
 
44
44
  /**
45
- * Minimal RFC-4180-ish splitter (handles quoted fields).
45
+ * Stream real German tuples out of a cached OA zip.
46
46
  */
47
- function splitCSV(line: string): string[] {
48
- const out: string[] = []
49
- let cur = ""
50
- let inQ = false
51
-
52
- for (let i = 0; i < line.length; i++) {
53
- const c = line[i]!
54
-
55
- if (inQ) {
56
- if (c === '"') {
57
- if (line[i + 1] === '"') {
58
- cur += '"'
59
-
60
- i++
61
- } else {
62
- inQ = false
63
- }
64
- } else {
65
- cur += c
66
- }
67
- } else if (c === '"') {
68
- inQ = true
69
- } else if (c === ",") {
70
- out.push(cur)
71
- cur = ""
72
- } else {
73
- cur += c
74
- }
75
- }
76
-
77
- out.push(cur)
78
-
79
- return out
80
- }
81
-
82
- /**
83
- * Stream real German tuples out of a cached OA zip (buffered `unzip -p`).
84
- */
85
- function readGermanTuples(source: GermanSource): LocaleBaseTuple[] {
86
- const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
87
-
88
- if (r.status !== 0) {
89
- console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
90
-
91
- return []
92
- }
93
-
94
- const lines = r.stdout.toString("utf8").split(/\r?\n/)
95
-
96
- if (lines.length < 2) return []
97
- const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
98
- const idx = (name: string): number => header.indexOf(name)
99
-
100
- const iNum = idx("number"),
101
- iStreet = idx("street"),
102
- iCity = idx("city"),
103
- iRegion = idx("region"),
104
- iPost = idx("postcode")
105
-
106
- const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
47
+ async function readGermanTuples(source: GermanSource): Promise<LocaleBaseTuple[]> {
107
48
  const tuples: LocaleBaseTuple[] = []
108
49
  const seen = new Set<string>()
109
50
 
110
- for (let li = 1; li < lines.length; li++) {
111
- const lineStr = lines[li]
112
-
113
- if (!lineStr) continue
114
- const cells = splitCSV(lineStr)
115
- const street = get(cells, iStreet)
116
- const locality = get(cells, iCity)
51
+ for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
52
+ const street = row.street ?? ""
53
+ const locality = row.city ?? ""
117
54
 
118
55
  if (!street || !locality) continue
119
- const house_number = get(cells, iNum)
120
- const postcode = get(cells, iPost)
56
+ const house_number = row.number ?? ""
57
+ const postcode = row.postcode ?? ""
121
58
  // OA's REGION column is empty for DE — fall back to the source's Bundesland (set per file).
122
- const region = get(cells, iRegion) || source.region || ""
59
+ const region = row.region || source.region || ""
123
60
  const key = `${house_number}|${street}|${locality}|${postcode}`.toLowerCase()
124
61
 
125
62
  if (seen.has(key)) continue
@@ -155,7 +92,7 @@ export const germanRecipe: ShardRecipe = {
155
92
  const pool: LocaleBaseTuple[] = []
156
93
 
157
94
  for (const s of SOURCES) {
158
- const t = readGermanTuples(s)
95
+ const t = await readGermanTuples(s)
159
96
 
160
97
  console.error(` ${s.csv}: ${t.length} unique tuples`)
161
98
 
@@ -165,7 +102,7 @@ export const germanRecipe: ShardRecipe = {
165
102
  }
166
103
 
167
104
  if (!pool.length) {
168
- throw new Error("No German tuples found — are the cached zips present in /tmp/oa-cache?")
105
+ throw new Error(`No German tuples found — are the cached zips present in ${dataRootPath("oa-cache")}?`)
169
106
  }
170
107
 
171
108
  let emitted = 0
@@ -10,7 +10,8 @@
10
10
  * scripts/build-house-venue-shard.mjs.
11
11
  */
12
12
 
13
- import { synthesizeHouseVenueRow, type HouseVenueBaseTuple } from "../synthesize-house-venue.ts"
13
+ import { synthesizeHouseVenueRow, type HouseVenueBaseTuple } from "@mailwoman/corpus/synthesizers/house-venue"
14
+
14
15
  import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } from "./scaffold.ts"
15
16
 
16
17
  const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"
@@ -29,9 +29,10 @@ import { poBoxCedexRecipe } from "./po-box-cedex.ts"
29
29
  import { poBoxRecipe } from "./po-box.ts"
30
30
  import type { ShardRecipe } from "./scaffold.ts"
31
31
  import { siBareVillageRecipe } from "./si-bare-village.ts"
32
- import { streetAffixRecipe } from "./street-affix.ts"
32
+ import { streetAffixRecipe, suffixBoundaryRecipe } from "./street-affix.ts"
33
33
  import { streetBareRecipe } from "./street-bare.ts"
34
34
  import { streetRecipe } from "./street.ts"
35
+ import { subVenueRecipe } from "./sub-venue.ts"
35
36
  import { unitRecipe } from "./unit.ts"
36
37
 
37
38
  export * from "./scaffold.ts"
@@ -43,6 +44,7 @@ const RECIPES: readonly ShardRecipe[] = [
43
44
  streetRecipe,
44
45
  streetBareRecipe,
45
46
  streetAffixRecipe,
47
+ suffixBoundaryRecipe,
46
48
  noStreetRecipe,
47
49
  houseVenueRecipe,
48
50
  poBoxRecipe,
@@ -64,6 +66,7 @@ const RECIPES: readonly ShardRecipe[] = [
64
66
  countryBalancedRecipe,
65
67
  boundaryStressRecipe,
66
68
  anchorAbsorptionRecipe,
69
+ subVenueRecipe,
67
70
  ]
68
71
 
69
72
  /**