@mailwoman/corpus 8.6.0 → 9.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (497) hide show
  1. package/README.md +2 -2
  2. package/data/PROVENANCE.md +214 -0
  3. package/data/sub-venue-lexicon.json +18269 -0
  4. package/out/src/adapters/ban/adapter.d.ts +5 -4
  5. package/out/src/adapters/ban/adapter.d.ts.map +1 -1
  6. package/out/src/adapters/ban/adapter.js +62 -69
  7. package/out/src/adapters/ban/adapter.js.map +1 -1
  8. package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
  9. package/out/src/adapters/ban/street-decompose.js +18 -25
  10. package/out/src/adapters/ban/street-decompose.js.map +1 -1
  11. package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -7
  12. package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
  13. package/out/src/adapters/fcc-bdc/adapter.js +4 -26
  14. package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
  15. package/out/src/adapters/geonames/adapter.d.ts +1 -1
  16. package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
  17. package/out/src/adapters/geonames/adapter.js +72 -78
  18. package/out/src/adapters/geonames/adapter.js.map +1 -1
  19. package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
  20. package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
  21. package/out/src/adapters/geonames-postal/adapter.js +46 -51
  22. package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
  23. package/out/src/adapters/gnaf/adapter.d.ts +1 -1
  24. package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
  25. package/out/src/adapters/gnaf/adapter.js +6 -9
  26. package/out/src/adapters/gnaf/adapter.js.map +1 -1
  27. package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
  28. package/out/src/adapters/gnaf/assemble.js +7 -12
  29. package/out/src/adapters/gnaf/assemble.js.map +1 -1
  30. package/out/src/adapters/index.d.ts +1 -1
  31. package/out/src/adapters/index.d.ts.map +1 -1
  32. package/out/src/adapters/index.js +1 -1
  33. package/out/src/adapters/index.js.map +1 -1
  34. package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
  35. package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
  36. package/out/src/adapters/openaddresses/adapter.js +5 -10
  37. package/out/src/adapters/openaddresses/adapter.js.map +1 -1
  38. package/out/src/adapters/overture/adapter.d.ts +1 -1
  39. package/out/src/adapters/overture/adapter.d.ts.map +1 -1
  40. package/out/src/adapters/overture/adapter.js +5 -9
  41. package/out/src/adapters/overture/adapter.js.map +1 -1
  42. package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
  43. package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
  44. package/out/src/adapters/state-hi-schools/adapter.js +57 -75
  45. package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
  46. package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
  47. package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
  48. package/out/src/adapters/state-ia-contractors/adapter.js +60 -78
  49. package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
  50. package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
  51. package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
  52. package/out/src/adapters/state-ny-notaries/adapter.js +69 -87
  53. package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
  54. package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
  55. package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
  56. package/out/src/adapters/state-tx-notaries/adapter.js +71 -89
  57. package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
  58. package/out/src/adapters/synth-po-box/adapter.d.ts +3 -3
  59. package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
  60. package/out/src/adapters/synth-po-box/adapter.js +10 -18
  61. package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
  62. package/out/src/adapters/tiger/adapter.d.ts +1 -1
  63. package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
  64. package/out/src/adapters/tiger/adapter.js +2 -2
  65. package/out/src/adapters/tiger/adapter.js.map +1 -1
  66. package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
  67. package/out/src/adapters/tiger/street-decompose.js +18 -27
  68. package/out/src/adapters/tiger/street-decompose.js.map +1 -1
  69. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +4 -4
  70. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
  71. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +64 -82
  72. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
  73. package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
  74. package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
  75. package/out/src/adapters/usgov-imls-pls/adapter.js +62 -84
  76. package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
  77. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
  78. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
  79. package/out/src/adapters/usgov-irs-bmf/adapter.js +59 -66
  80. package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
  81. package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
  82. package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
  83. package/out/src/adapters/usgov-nad/adapter.js +6 -9
  84. package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
  85. package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
  86. package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
  87. package/out/src/adapters/usgov-nppes/adapter.js +60 -78
  88. package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
  89. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
  90. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
  91. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +53 -73
  92. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
  93. package/out/src/{adapter.d.ts → adapters/utils/index.d.ts} +43 -2
  94. package/out/src/adapters/utils/index.d.ts.map +1 -0
  95. package/out/src/{adapter.js → adapters/utils/index.js} +47 -2
  96. package/out/src/adapters/utils/index.js.map +1 -0
  97. package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
  98. package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
  99. package/out/src/adapters/wof-admin-jp/adapter.js +24 -6
  100. package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
  101. package/out/src/adapters/wof-admin-json/adapter.d.ts +2 -2
  102. package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
  103. package/out/src/adapters/wof-admin-json/adapter.js +2 -2
  104. package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
  105. package/out/src/adapters/wof-postalcode-json/adapter.d.ts +2 -2
  106. package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
  107. package/out/src/adapters/wof-postalcode-json/adapter.js +2 -2
  108. package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
  109. package/out/src/build.d.ts +3 -4
  110. package/out/src/build.d.ts.map +1 -1
  111. package/out/src/build.js +6 -8
  112. package/out/src/build.js.map +1 -1
  113. package/out/src/index.d.ts +3 -17
  114. package/out/src/index.d.ts.map +1 -1
  115. package/out/src/index.js +3 -17
  116. package/out/src/index.js.map +1 -1
  117. package/out/src/name-prone-us-suffixes.d.ts +12 -0
  118. package/out/src/name-prone-us-suffixes.d.ts.map +1 -0
  119. package/out/src/name-prone-us-suffixes.js +12 -0
  120. package/out/src/name-prone-us-suffixes.js.map +1 -0
  121. package/out/src/parquet-wrapper/reader.d.ts +8 -0
  122. package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
  123. package/out/src/parquet-wrapper/reader.js +16 -0
  124. package/out/src/parquet-wrapper/reader.js.map +1 -1
  125. package/out/src/parquet-wrapper/schema.d.ts +4 -0
  126. package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
  127. package/out/src/parquet-wrapper/schema.js.map +1 -1
  128. package/out/src/runner.d.ts +2 -2
  129. package/out/src/runner.d.ts.map +1 -1
  130. package/out/src/runner.js +1 -1
  131. package/out/src/runner.js.map +1 -1
  132. package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
  133. package/out/src/shard-recipes/anchor-absorption.js +5 -4
  134. package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
  135. package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
  136. package/out/src/shard-recipes/boundary-stress.js +2 -2
  137. package/out/src/shard-recipes/boundary-stress.js.map +1 -1
  138. package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
  139. package/out/src/shard-recipes/country-balanced.js +75 -75
  140. package/out/src/shard-recipes/country-balanced.js.map +1 -1
  141. package/out/src/shard-recipes/fr-admin-split.js +2 -2
  142. package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
  143. package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
  144. package/out/src/shard-recipes/fr-fragment.js +8 -5
  145. package/out/src/shard-recipes/fr-fragment.js.map +1 -1
  146. package/out/src/shard-recipes/fr-lieudit.js +2 -2
  147. package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
  148. package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
  149. package/out/src/shard-recipes/fr-order.js +18 -72
  150. package/out/src/shard-recipes/fr-order.js.map +1 -1
  151. package/out/src/shard-recipes/german.d.ts.map +1 -1
  152. package/out/src/shard-recipes/german.js +17 -70
  153. package/out/src/shard-recipes/german.js.map +1 -1
  154. package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
  155. package/out/src/shard-recipes/house-venue.js +1 -1
  156. package/out/src/shard-recipes/house-venue.js.map +1 -1
  157. package/out/src/shard-recipes/index.d.ts.map +1 -1
  158. package/out/src/shard-recipes/index.js +4 -1
  159. package/out/src/shard-recipes/index.js.map +1 -1
  160. package/out/src/shard-recipes/intersection.d.ts +2 -2
  161. package/out/src/shard-recipes/intersection.d.ts.map +1 -1
  162. package/out/src/shard-recipes/intersection.js +28 -72
  163. package/out/src/shard-recipes/intersection.js.map +1 -1
  164. package/out/src/shard-recipes/locale.d.ts +5 -5
  165. package/out/src/shard-recipes/locale.d.ts.map +1 -1
  166. package/out/src/shard-recipes/locale.js +25 -35
  167. package/out/src/shard-recipes/locale.js.map +1 -1
  168. package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
  169. package/out/src/shard-recipes/no-fragment.js +8 -5
  170. package/out/src/shard-recipes/no-fragment.js.map +1 -1
  171. package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
  172. package/out/src/shard-recipes/no-street-led.js +8 -5
  173. package/out/src/shard-recipes/no-street-led.js.map +1 -1
  174. package/out/src/shard-recipes/no-street.d.ts.map +1 -1
  175. package/out/src/shard-recipes/no-street.js +1 -1
  176. package/out/src/shard-recipes/no-street.js.map +1 -1
  177. package/out/src/shard-recipes/po-box-cedex.d.ts +3 -3
  178. package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
  179. package/out/src/shard-recipes/po-box-cedex.js +101 -127
  180. package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
  181. package/out/src/shard-recipes/po-box.d.ts.map +1 -1
  182. package/out/src/shard-recipes/po-box.js +1 -1
  183. package/out/src/shard-recipes/po-box.js.map +1 -1
  184. package/out/src/shard-recipes/scaffold.d.ts +58 -9
  185. package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
  186. package/out/src/shard-recipes/scaffold.js +87 -29
  187. package/out/src/shard-recipes/scaffold.js.map +1 -1
  188. package/out/src/shard-recipes/street-affix.d.ts +54 -0
  189. package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
  190. package/out/src/shard-recipes/street-affix.js +253 -111
  191. package/out/src/shard-recipes/street-affix.js.map +1 -1
  192. package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
  193. package/out/src/shard-recipes/street-bare.js +3 -3
  194. package/out/src/shard-recipes/street-bare.js.map +1 -1
  195. package/out/src/shard-recipes/street.d.ts.map +1 -1
  196. package/out/src/shard-recipes/street.js +2 -2
  197. package/out/src/shard-recipes/street.js.map +1 -1
  198. package/out/src/shard-recipes/sub-venue-sources.d.ts +228 -0
  199. package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -0
  200. package/out/src/shard-recipes/sub-venue-sources.js +664 -0
  201. package/out/src/shard-recipes/sub-venue-sources.js.map +1 -0
  202. package/out/src/shard-recipes/sub-venue.d.ts +269 -0
  203. package/out/src/shard-recipes/sub-venue.d.ts.map +1 -0
  204. package/out/src/shard-recipes/sub-venue.js +709 -0
  205. package/out/src/shard-recipes/sub-venue.js.map +1 -0
  206. package/out/src/shard-recipes/unit.d.ts +1 -1
  207. package/out/src/shard-recipes/unit.d.ts.map +1 -1
  208. package/out/src/shard-recipes/unit.js +26 -74
  209. package/out/src/shard-recipes/unit.js.map +1 -1
  210. package/out/src/{synthesize-anchor-absorption.d.ts → synthesizers/anchor-absorption.d.ts} +1 -1
  211. package/out/src/synthesizers/anchor-absorption.d.ts.map +1 -0
  212. package/out/src/{synthesize-anchor-absorption.js → synthesizers/anchor-absorption.js} +1 -1
  213. package/out/src/synthesizers/anchor-absorption.js.map +1 -0
  214. package/out/src/{synthesize-boundary-stress.d.ts → synthesizers/boundary-stress.d.ts} +3 -3
  215. package/out/src/synthesizers/boundary-stress.d.ts.map +1 -0
  216. package/out/src/{synthesize-boundary-stress.js → synthesizers/boundary-stress.js} +2 -2
  217. package/out/src/synthesizers/boundary-stress.js.map +1 -0
  218. package/out/src/{synthesize-german.d.ts → synthesizers/german.d.ts} +2 -2
  219. package/out/src/synthesizers/german.d.ts.map +1 -0
  220. package/out/src/{synthesize-german.js → synthesizers/german.js} +2 -2
  221. package/out/src/synthesizers/german.js.map +1 -0
  222. package/out/src/{synthesize-house-venue.d.ts → synthesizers/house-venue.d.ts} +4 -4
  223. package/out/src/synthesizers/house-venue.d.ts.map +1 -0
  224. package/out/src/{synthesize-house-venue.js → synthesizers/house-venue.js} +4 -4
  225. package/out/src/synthesizers/house-venue.js.map +1 -0
  226. package/out/src/{synthesize-intersection.d.ts → synthesizers/intersection.d.ts} +2 -2
  227. package/out/src/synthesizers/intersection.d.ts.map +1 -0
  228. package/out/src/{synthesize-intersection.js → synthesizers/intersection.js} +1 -1
  229. package/out/src/synthesizers/intersection.js.map +1 -0
  230. package/out/src/{synthesize-no-street.d.ts → synthesizers/no-street.d.ts} +3 -3
  231. package/out/src/synthesizers/no-street.d.ts.map +1 -0
  232. package/out/src/{synthesize-no-street.js → synthesizers/no-street.js} +2 -2
  233. package/out/src/synthesizers/no-street.js.map +1 -0
  234. package/out/src/{synthesize-po-box.d.ts → synthesizers/po-box.d.ts} +2 -2
  235. package/out/src/synthesizers/po-box.d.ts.map +1 -0
  236. package/out/src/{synthesize-po-box.js → synthesizers/po-box.js} +1 -1
  237. package/out/src/synthesizers/po-box.js.map +1 -0
  238. package/out/src/{synthesize-street.d.ts → synthesizers/street.d.ts} +2 -2
  239. package/out/src/synthesizers/street.d.ts.map +1 -0
  240. package/out/src/{synthesize-street.js → synthesizers/street.js} +2 -2
  241. package/out/src/synthesizers/street.js.map +1 -0
  242. package/out/src/{synthesize.d.ts → synthesizers/utils.d.ts} +3 -3
  243. package/out/src/synthesizers/utils.d.ts.map +1 -0
  244. package/out/src/{synthesize.js → synthesizers/utils.js} +4 -15
  245. package/out/src/synthesizers/utils.js.map +1 -0
  246. package/out/src/tools/align-shard.d.ts.map +1 -1
  247. package/out/src/tools/align-shard.js +4 -8
  248. package/out/src/tools/align-shard.js.map +1 -1
  249. package/out/src/tools/audit.d.ts.map +1 -1
  250. package/out/src/tools/audit.js +4 -4
  251. package/out/src/tools/audit.js.map +1 -1
  252. package/out/src/tools/corpus-stats.d.ts +2 -2
  253. package/out/src/tools/corpus-stats.d.ts.map +1 -1
  254. package/out/src/tools/corpus-stats.js +18 -31
  255. package/out/src/tools/corpus-stats.js.map +1 -1
  256. package/out/src/tools/fetch/download.d.ts.map +1 -1
  257. package/out/src/tools/fetch/download.js +5 -6
  258. package/out/src/tools/fetch/download.js.map +1 -1
  259. package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
  260. package/out/src/tools/fetch/imls-pls.js +3 -15
  261. package/out/src/tools/fetch/imls-pls.js.map +1 -1
  262. package/out/src/tools/fetch/index.d.ts +14 -1
  263. package/out/src/tools/fetch/index.d.ts.map +1 -1
  264. package/out/src/tools/fetch/index.js +14 -1
  265. package/out/src/tools/fetch/index.js.map +1 -1
  266. package/out/src/tools/fetch/nad.d.ts +1 -1
  267. package/out/src/tools/fetch/nad.js +1 -1
  268. package/out/src/tools/fetch/nppes.d.ts.map +1 -1
  269. package/out/src/tools/fetch/nppes.js +8 -14
  270. package/out/src/tools/fetch/nppes.js.map +1 -1
  271. package/out/src/tools/fetch/openaddresses.d.ts +1 -1
  272. package/out/src/tools/fetch/openaddresses.js +1 -1
  273. package/out/src/tools/fetch/ourairports.d.ts +45 -0
  274. package/out/src/tools/fetch/ourairports.d.ts.map +1 -0
  275. package/out/src/tools/fetch/ourairports.js +123 -0
  276. package/out/src/tools/fetch/ourairports.js.map +1 -0
  277. package/out/src/tools/fetch/wikidata-subvenue.d.ts +171 -0
  278. package/out/src/tools/fetch/wikidata-subvenue.d.ts.map +1 -0
  279. package/out/src/tools/fetch/wikidata-subvenue.js +275 -0
  280. package/out/src/tools/fetch/wikidata-subvenue.js.map +1 -0
  281. package/out/src/tools/golden-expand.d.ts.map +1 -1
  282. package/out/src/tools/golden-expand.js +18 -16
  283. package/out/src/tools/golden-expand.js.map +1 -1
  284. package/out/src/tools/golden-promote.d.ts.map +1 -1
  285. package/out/src/tools/golden-promote.js +12 -6
  286. package/out/src/tools/golden-promote.js.map +1 -1
  287. package/out/src/tools/golden-relabel-street.d.ts +196 -0
  288. package/out/src/tools/golden-relabel-street.d.ts.map +1 -0
  289. package/out/src/tools/golden-relabel-street.js +440 -0
  290. package/out/src/tools/golden-relabel-street.js.map +1 -0
  291. package/out/src/tools/index.d.ts +4 -0
  292. package/out/src/tools/index.d.ts.map +1 -1
  293. package/out/src/tools/index.js +4 -0
  294. package/out/src/tools/index.js.map +1 -1
  295. package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
  296. package/out/src/tools/jsonl-to-parquet.js +3 -2
  297. package/out/src/tools/jsonl-to-parquet.js.map +1 -1
  298. package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
  299. package/out/src/tools/lint-shard-vocab.js +3 -2
  300. package/out/src/tools/lint-shard-vocab.js.map +1 -1
  301. package/out/src/tools/lint-shard.d.ts +3 -3
  302. package/out/src/tools/lint-shard.d.ts.map +1 -1
  303. package/out/src/tools/lint-shard.js +23 -32
  304. package/out/src/tools/lint-shard.js.map +1 -1
  305. package/out/src/tools/overlay-manifest.d.ts.map +1 -1
  306. package/out/src/tools/overlay-manifest.js +2 -1
  307. package/out/src/tools/overlay-manifest.js.map +1 -1
  308. package/out/src/tools/overture-subvenue.d.ts +112 -0
  309. package/out/src/tools/overture-subvenue.d.ts.map +1 -0
  310. package/out/src/tools/overture-subvenue.js +144 -0
  311. package/out/src/tools/overture-subvenue.js.map +1 -0
  312. package/out/src/tools/shard-kryptonite.d.ts +1 -1
  313. package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
  314. package/out/src/tools/shard-kryptonite.js +6 -6
  315. package/out/src/tools/shard-kryptonite.js.map +1 -1
  316. package/out/src/tools/shard-translit.d.ts +6 -3
  317. package/out/src/tools/shard-translit.d.ts.map +1 -1
  318. package/out/src/tools/shard-translit.js +9 -8
  319. package/out/src/tools/shard-translit.js.map +1 -1
  320. package/out/src/tools/sub-venue-lexicon.d.ts +507 -0
  321. package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -0
  322. package/out/src/tools/sub-venue-lexicon.js +817 -0
  323. package/out/src/tools/sub-venue-lexicon.js.map +1 -0
  324. package/out/src/tools/sub-venue-promotions.d.ts +94 -0
  325. package/out/src/tools/sub-venue-promotions.d.ts.map +1 -0
  326. package/out/src/tools/sub-venue-promotions.js +266 -0
  327. package/out/src/tools/sub-venue-promotions.js.map +1 -0
  328. package/out/src/{align.d.ts → utils/align.d.ts} +1 -1
  329. package/out/src/utils/align.d.ts.map +1 -0
  330. package/out/src/utils/align.js.map +1 -0
  331. package/out/src/{golden.d.ts → utils/golden.d.ts} +5 -0
  332. package/out/src/utils/golden.d.ts.map +1 -0
  333. package/out/src/{golden.js → utils/golden.js} +24 -10
  334. package/out/src/utils/golden.js.map +1 -0
  335. package/out/src/utils/index.d.ts +14 -0
  336. package/out/src/utils/index.d.ts.map +1 -0
  337. package/out/src/utils/index.js +14 -0
  338. package/out/src/utils/index.js.map +1 -0
  339. package/out/src/utils/license.d.ts.map +1 -0
  340. package/out/src/utils/license.js.map +1 -0
  341. package/out/src/{parquet.d.ts → utils/parquet.d.ts} +2 -2
  342. package/out/src/utils/parquet.d.ts.map +1 -0
  343. package/out/src/{parquet.js → utils/parquet.js} +3 -15
  344. package/out/src/utils/parquet.js.map +1 -0
  345. package/out/src/{split.d.ts → utils/split.d.ts} +8 -1
  346. package/out/src/utils/split.d.ts.map +1 -0
  347. package/out/src/{split.js → utils/split.js} +7 -0
  348. package/out/src/utils/split.js.map +1 -0
  349. package/out/src/utils/tokenize.d.ts.map +1 -0
  350. package/out/src/utils/tokenize.js.map +1 -0
  351. package/out/src/{wof-json.d.ts → utils/wof-json.d.ts} +2 -2
  352. package/out/src/utils/wof-json.d.ts.map +1 -0
  353. package/out/src/{wof-json.js → utils/wof-json.js} +5 -8
  354. package/out/src/utils/wof-json.js.map +1 -0
  355. package/package.json +66 -20
  356. package/src/adapters/ban/adapter.ts +58 -68
  357. package/src/adapters/ban/street-decompose.ts +17 -25
  358. package/src/adapters/fcc-bdc/adapter.ts +5 -36
  359. package/src/adapters/geonames/adapter.ts +67 -76
  360. package/src/adapters/geonames-postal/adapter.ts +45 -54
  361. package/src/adapters/gnaf/adapter.ts +7 -11
  362. package/src/adapters/gnaf/assemble.ts +7 -11
  363. package/src/adapters/index.ts +3 -2
  364. package/src/adapters/openaddresses/adapter.ts +6 -12
  365. package/src/adapters/overture/adapter.ts +6 -10
  366. package/src/adapters/state-hi-schools/adapter.ts +53 -77
  367. package/src/adapters/state-ia-contractors/adapter.ts +55 -80
  368. package/src/adapters/state-ny-notaries/adapter.ts +64 -89
  369. package/src/adapters/state-tx-notaries/adapter.ts +63 -88
  370. package/src/adapters/synth-po-box/adapter.ts +12 -23
  371. package/src/adapters/tiger/adapter.ts +4 -3
  372. package/src/adapters/tiger/street-decompose.ts +21 -31
  373. package/src/adapters/usgov-hrsa-fqhc/adapter.ts +64 -88
  374. package/src/adapters/usgov-imls-pls/adapter.ts +57 -86
  375. package/src/adapters/usgov-irs-bmf/adapter.ts +55 -66
  376. package/src/adapters/usgov-nad/adapter.ts +7 -11
  377. package/src/adapters/usgov-nppes/adapter.ts +54 -79
  378. package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +48 -75
  379. package/src/{adapter.ts → adapters/utils/index.ts} +59 -3
  380. package/src/adapters/wof-admin-jp/adapter.ts +35 -9
  381. package/src/adapters/wof-admin-json/adapter.ts +3 -4
  382. package/src/adapters/wof-postalcode-json/adapter.ts +3 -4
  383. package/src/build.ts +12 -11
  384. package/src/index.ts +3 -17
  385. package/src/name-prone-us-suffixes.ts +12 -0
  386. package/src/parquet-wrapper/reader.ts +19 -0
  387. package/src/parquet-wrapper/schema.ts +4 -0
  388. package/src/runner.ts +7 -2
  389. package/src/shard-recipes/anchor-absorption.ts +5 -4
  390. package/src/shard-recipes/boundary-stress.ts +6 -2
  391. package/src/shard-recipes/country-balanced.ts +77 -91
  392. package/src/shard-recipes/fr-admin-split.ts +3 -3
  393. package/src/shard-recipes/fr-fragment.ts +10 -7
  394. package/src/shard-recipes/fr-lieudit.ts +3 -3
  395. package/src/shard-recipes/fr-order.ts +18 -83
  396. package/src/shard-recipes/german.ts +17 -80
  397. package/src/shard-recipes/house-venue.ts +2 -1
  398. package/src/shard-recipes/index.ts +4 -1
  399. package/src/shard-recipes/intersection.ts +37 -74
  400. package/src/shard-recipes/locale.ts +31 -39
  401. package/src/shard-recipes/no-fragment.ts +10 -7
  402. package/src/shard-recipes/no-street-led.ts +10 -7
  403. package/src/shard-recipes/no-street.ts +2 -1
  404. package/src/shard-recipes/po-box-cedex.ts +127 -161
  405. package/src/shard-recipes/po-box.ts +6 -1
  406. package/src/shard-recipes/scaffold.ts +125 -27
  407. package/src/shard-recipes/street-affix.ts +335 -134
  408. package/src/shard-recipes/street-bare.ts +4 -3
  409. package/src/shard-recipes/street.ts +3 -2
  410. package/src/shard-recipes/sub-venue-sources.ts +895 -0
  411. package/src/shard-recipes/sub-venue.ts +982 -0
  412. package/src/shard-recipes/unit.ts +27 -85
  413. package/src/{synthesize-boundary-stress.ts → synthesizers/boundary-stress.ts} +2 -2
  414. package/src/{synthesize-german.ts → synthesizers/german.ts} +2 -2
  415. package/src/{synthesize-house-venue.ts → synthesizers/house-venue.ts} +4 -4
  416. package/src/{synthesize-intersection.ts → synthesizers/intersection.ts} +1 -1
  417. package/src/{synthesize-no-street.ts → synthesizers/no-street.ts} +2 -2
  418. package/src/{synthesize-po-box.ts → synthesizers/po-box.ts} +1 -1
  419. package/src/{synthesize-street.ts → synthesizers/street.ts} +2 -2
  420. package/src/{synthesize.ts → synthesizers/utils.ts} +4 -18
  421. package/src/tools/align-shard.ts +4 -7
  422. package/src/tools/audit.ts +6 -5
  423. package/src/tools/corpus-stats.ts +25 -33
  424. package/src/tools/fetch/download.ts +7 -5
  425. package/src/tools/fetch/imls-pls.ts +4 -18
  426. package/src/tools/fetch/index.ts +14 -1
  427. package/src/tools/fetch/nad.ts +1 -1
  428. package/src/tools/fetch/nppes.ts +9 -16
  429. package/src/tools/fetch/openaddresses.ts +1 -1
  430. package/src/tools/fetch/ourairports.ts +166 -0
  431. package/src/tools/fetch/wikidata-subvenue.ts +386 -0
  432. package/src/tools/golden-expand.ts +18 -13
  433. package/src/tools/golden-promote.ts +15 -6
  434. package/src/tools/golden-relabel-street.ts +685 -0
  435. package/src/tools/index.ts +4 -0
  436. package/src/tools/jsonl-to-parquet.ts +3 -2
  437. package/src/tools/lint-shard-vocab.ts +3 -2
  438. package/src/tools/lint-shard.ts +33 -36
  439. package/src/tools/overlay-manifest.ts +2 -1
  440. package/src/tools/overture-subvenue.ts +215 -0
  441. package/src/tools/shard-kryptonite.ts +8 -9
  442. package/src/tools/shard-translit.ts +22 -11
  443. package/src/tools/sub-venue-lexicon.ts +1250 -0
  444. package/src/tools/sub-venue-promotions.ts +330 -0
  445. package/src/{align.ts → utils/align.ts} +1 -1
  446. package/src/{golden.ts → utils/golden.ts} +25 -11
  447. package/src/utils/index.ts +14 -0
  448. package/src/{parquet.ts → utils/parquet.ts} +5 -19
  449. package/src/{split.ts → utils/split.ts} +8 -2
  450. package/src/{wof-json.ts → utils/wof-json.ts} +5 -8
  451. package/out/src/adapter.d.ts.map +0 -1
  452. package/out/src/adapter.js.map +0 -1
  453. package/out/src/align.d.ts.map +0 -1
  454. package/out/src/align.js.map +0 -1
  455. package/out/src/format.d.ts +0 -14
  456. package/out/src/format.d.ts.map +0 -1
  457. package/out/src/format.js +0 -14
  458. package/out/src/format.js.map +0 -1
  459. package/out/src/golden.d.ts.map +0 -1
  460. package/out/src/golden.js.map +0 -1
  461. package/out/src/license.d.ts.map +0 -1
  462. package/out/src/license.js.map +0 -1
  463. package/out/src/parquet.d.ts.map +0 -1
  464. package/out/src/parquet.js.map +0 -1
  465. package/out/src/split.d.ts.map +0 -1
  466. package/out/src/split.js.map +0 -1
  467. package/out/src/synthesize-anchor-absorption.d.ts.map +0 -1
  468. package/out/src/synthesize-anchor-absorption.js.map +0 -1
  469. package/out/src/synthesize-boundary-stress.d.ts.map +0 -1
  470. package/out/src/synthesize-boundary-stress.js.map +0 -1
  471. package/out/src/synthesize-german.d.ts.map +0 -1
  472. package/out/src/synthesize-german.js.map +0 -1
  473. package/out/src/synthesize-house-venue.d.ts.map +0 -1
  474. package/out/src/synthesize-house-venue.js.map +0 -1
  475. package/out/src/synthesize-intersection.d.ts.map +0 -1
  476. package/out/src/synthesize-intersection.js.map +0 -1
  477. package/out/src/synthesize-no-street.d.ts.map +0 -1
  478. package/out/src/synthesize-no-street.js.map +0 -1
  479. package/out/src/synthesize-po-box.d.ts.map +0 -1
  480. package/out/src/synthesize-po-box.js.map +0 -1
  481. package/out/src/synthesize-street.d.ts.map +0 -1
  482. package/out/src/synthesize-street.js.map +0 -1
  483. package/out/src/synthesize.d.ts.map +0 -1
  484. package/out/src/synthesize.js.map +0 -1
  485. package/out/src/tokenize.d.ts.map +0 -1
  486. package/out/src/tokenize.js.map +0 -1
  487. package/out/src/wof-json.d.ts.map +0 -1
  488. package/out/src/wof-json.js.map +0 -1
  489. package/src/format.ts +0 -14
  490. /package/out/src/{align.js → utils/align.js} +0 -0
  491. /package/out/src/{license.d.ts → utils/license.d.ts} +0 -0
  492. /package/out/src/{license.js → utils/license.js} +0 -0
  493. /package/out/src/{tokenize.d.ts → utils/tokenize.d.ts} +0 -0
  494. /package/out/src/{tokenize.js → utils/tokenize.js} +0 -0
  495. /package/src/{synthesize-anchor-absorption.ts → synthesizers/anchor-absorption.ts} +0 -0
  496. /package/src/{license.ts → utils/license.ts} +0 -0
  497. /package/src/{tokenize.ts → utils/tokenize.ts} +0 -0
@@ -20,9 +20,9 @@
20
20
  * locality-hash holdout (hash%10===0).
21
21
  *
22
22
  * Prerequisites (read once, before the generation loop — these do NOT consume `random`): the cached
23
- * OA zips in /tmp/oa-cache, the GeoNames Canada dump at /tmp/geonames-cache/CA.zip, and the
24
- * GeoNames POSTAL-CODE dumps for AU/NZ (/tmp/geonames-cache/{AU,NZ}-postal.zip). See the legacy
25
- * script header for the exact curl commands.
23
+ * OA zips in `$MAILWOMAN_DATA_ROOT/oa-cache`, the GeoNames Canada dump at
24
+ * `$MAILWOMAN_DATA_ROOT/geonames/CA.zip`, and the GeoNames POSTAL-CODE dumps for AU/NZ
25
+ * (`$MAILWOMAN_DATA_ROOT/geonames-postal/{AU,NZ}.zip`).
26
26
  *
27
27
  * Byte-fidelity: the legacy script seeded its own mulberry32 from `--seed`
28
28
  * (`mulberry32(opts.seed)`); this recipe re-creates the SAME generator
@@ -30,22 +30,32 @@
30
30
  * reproduces the legacy run byte-for-byte.
31
31
  */
32
32
 
33
- import { spawnSync } from "node:child_process"
33
+ import { existsSync } from "node:fs"
34
34
 
35
35
  import { isAuDeliveryService, isAuPostcode, isAuStateAbbreviation } from "@mailwoman/codex/au"
36
36
  import { FSA_LETTER_TO_PROVINCE, normalizeCaPostalCode } from "@mailwoman/codex/ca"
37
37
  import { isCedex } from "@mailwoman/codex/fr"
38
- import { isNzDeliveryService, isNzPostcode } from "@mailwoman/codex/nz"
38
+ import { isNZDeliveryService, isNZPostcode } from "@mailwoman/codex/nz"
39
39
  import { isPOBox } from "@mailwoman/codex/us"
40
-
41
- import { alignRow } from "../align.ts"
40
+ import { readZipEntry } from "@mailwoman/core/fs/zip"
41
+ import { dataRootPath } from "@mailwoman/core/utils"
42
42
  import {
43
43
  maybeNoisifyBoxNumber,
44
44
  PO_BOX_LOCALE_TEMPLATES,
45
45
  synthesizeMilitaryPoBoxRow,
46
46
  type LocaleTemplate,
47
- } from "../synthesize-po-box.ts"
48
- import { makeMulberry32, shardSourceID, type CanonicalShardRow, type ShardRecipe } from "./scaffold.ts"
47
+ } from "@mailwoman/corpus/synthesizers/po-box"
48
+ import { alignRow } from "@mailwoman/corpus/utils"
49
+ import { TextSpliterator, TSVSpliterator } from "spliterator"
50
+
51
+ import {
52
+ makeMulberry32,
53
+ readCSVRecords,
54
+ readZippedCSVRecords,
55
+ shardSourceID,
56
+ type CanonicalShardRow,
57
+ type ShardRecipe,
58
+ } from "./scaffold.ts"
49
59
 
50
60
  // ── Base-skeleton sources ────────────────────────────────────────────────────────────────────────
51
61
  // Same OA cache as the unit/affix shards. US train = every NON-Vermont state; US eval = Vermont (the
@@ -58,20 +68,29 @@ import { makeMulberry32, shardSourceID, type CanonicalShardRow, type ShardRecipe
58
68
  extracted as named constants above. */
59
69
 
60
70
  const US_TRAIN_SOURCES = [
61
- { zip: "/tmp/oa-cache/us__ca__berkeley.zip", csv: "us/ca/berkeley.csv", region: "CA" },
62
- { zip: "/tmp/oa-cache/us__ca__marin.zip", csv: "us/ca/marin.csv", region: "CA" },
63
- { zip: "/tmp/oa-cache/us__dc__statewide.zip", csv: "us/dc/statewide.csv", region: "DC" },
64
- { zip: "/tmp/oa-cache/us__ia__statewide.zip", csv: "us/ia/statewide.csv", region: "IA" },
65
- { zip: "/tmp/oa-cache/us__il__cook.zip", csv: "us/il/cook.csv", region: "IL" },
66
- { zip: "/tmp/oa-cache/us__mt__statewide.zip", csv: "us/mt/statewide.csv", region: "MT" },
67
- { zip: "/tmp/oa-cache/us__sd__statewide.zip", csv: "us/sd/statewide.csv", region: "SD" },
71
+ { zip: dataRootPath("oa-cache", "us__ca__berkeley.zip"), csv: "us/ca/berkeley.csv", region: "CA" },
72
+ { zip: dataRootPath("oa-cache", "us__ca__marin.zip"), csv: "us/ca/marin.csv", region: "CA" },
73
+ { zip: dataRootPath("oa-cache", "us__dc__statewide.zip"), csv: "us/dc/statewide.csv", region: "DC" },
74
+ { zip: dataRootPath("oa-cache", "us__ia__statewide.zip"), csv: "us/ia/statewide.csv", region: "IA" },
75
+ { zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", region: "IL" },
76
+ { zip: dataRootPath("oa-cache", "us__mt__statewide.zip"), csv: "us/mt/statewide.csv", region: "MT" },
77
+ { zip: dataRootPath("oa-cache", "us__sd__statewide.zip"), csv: "us/sd/statewide.csv", region: "SD" },
68
78
  ]
69
79
 
70
- const US_EVAL_SOURCE = { zip: "/tmp/oa-cache/us__vt__statewide.zip", csv: "us/vt/statewide.csv", region: "VT" }
71
- const FR_SOURCE = { zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv" }
72
- const GEONAMES_CA = "/tmp/geonames-cache/CA.zip"
73
- const GEONAMES_POSTAL_AU = { zip: "/tmp/geonames-cache/AU-postal.zip", txt: "AU.txt" }
74
- const GEONAMES_POSTAL_NZ = { zip: "/tmp/geonames-cache/NZ-postal.zip", txt: "NZ.txt" }
80
+ const US_EVAL_SOURCE = {
81
+ zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
82
+ csv: "us/vt/statewide.csv",
83
+ region: "VT",
84
+ }
85
+
86
+ const FR_SOURCE = { zip: dataRootPath("oa-cache", "fr__countrywide.zip"), csv: "fr/countrywide.csv" }
87
+ // GeoNames has two per-country exports with different schemas, and this recipe reads both. The MAIN
88
+ // dump (feature class + population columns) lives under `geonames/`; the POSTAL-CODE dump lives
89
+ // under `geonames-postal/`, where the directory carries the "postal" distinction the filename used
90
+ // to.
91
+ const GEONAMES_CA = dataRootPath("geonames", "CA.zip")
92
+ const GEONAMES_POSTAL_AU = { zip: dataRootPath("geonames-postal", "AU.zip"), txt: "AU.txt" }
93
+ const GEONAMES_POSTAL_NZ = { zip: dataRootPath("geonames-postal", "NZ.zip"), txt: "NZ.txt" }
75
94
 
76
95
  /**
77
96
  * ── Surface vocabulary (codex + corpus templates — see the header) ──────────────────────────────.
@@ -172,20 +191,20 @@ interface USTuple {
172
191
  postcode: string
173
192
  }
174
193
 
175
- interface FrTuple {
194
+ interface FRTuple {
176
195
  house_number: string
177
196
  street: string
178
197
  locality: string
179
198
  postcode: string
180
199
  }
181
200
 
182
- interface AuTuple {
201
+ interface AUTuple {
183
202
  locality: string
184
203
  region: string
185
204
  postcode: string
186
205
  }
187
206
 
188
- interface NzTuple {
207
+ interface NZTuple {
189
208
  locality: string
190
209
  postcode: string
191
210
  }
@@ -214,44 +233,6 @@ function localityHash(name: string): number {
214
233
 
215
234
  const isHoldoutLocality = (name: string): boolean => localityHash(name) % 10 === 0
216
235
 
217
- /**
218
- * Minimal RFC-4180-ish splitter (handles quoted fields).
219
- */
220
- function splitCSV(line: string): string[] {
221
- const out: string[] = []
222
- let cur = ""
223
- let inQ = false
224
-
225
- for (let i = 0; i < line.length; i++) {
226
- const c = line[i]!
227
-
228
- if (inQ) {
229
- if (c === '"') {
230
- if (line[i + 1] === '"') {
231
- cur += '"'
232
-
233
- i++
234
- } else {
235
- inQ = false
236
- }
237
- } else {
238
- cur += c
239
- }
240
- } else if (c === '"') {
241
- inQ = true
242
- } else if (c === ",") {
243
- out.push(cur)
244
- cur = ""
245
- } else {
246
- cur += c
247
- }
248
- }
249
-
250
- out.push(cur)
251
-
252
- return out
253
- }
254
-
255
236
  const MAX_LOCALITY_LENGTH = 40
256
237
 
257
238
  const cleanLocality = (loc: string) =>
@@ -260,93 +241,75 @@ const cleanLocality = (loc: string) =>
260
241
  /**
261
242
  * Stream real US tuples (number/street/city/postcode) out of a cached OA zip.
262
243
  */
263
- function readUsTuples(source: { zip: string; csv: string; region: string }): USTuple[] {
264
- const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
265
-
266
- if (r.status !== 0) {
267
- console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
268
-
269
- return []
270
- }
271
-
272
- const lines = r.stdout.toString("utf8").split(/\r?\n/)
273
-
274
- if (lines.length < 2) return []
275
- const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
276
- const idx = (name: string) => header.indexOf(name)
277
-
278
- const iNum = idx("number"),
279
- iStreet = idx("street"),
280
- iCity = idx("city"),
281
- iPost = idx("postcode")
282
-
283
- const get = (cells: string[], i: number) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
244
+ async function readUsTuples(source: { zip: string; csv: string; region: string }): Promise<USTuple[]> {
284
245
  const tuples: USTuple[] = []
285
246
  const seen = new Set<string>()
286
247
 
287
- for (let li = 1; li < lines.length; li++) {
288
- if (!lines[li]) continue
289
- const cells = splitCSV(lines[li]!)
290
- const locality = get(cells, iCity)
248
+ for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
249
+ const locality = row.city ?? ""
291
250
 
292
251
  if (!cleanLocality(locality)) continue
293
252
  const key = locality.toLowerCase()
294
253
 
295
- const street = get(cells, iStreet),
296
- house_number = get(cells, iNum)
254
+ const street = row.street ?? "",
255
+ house_number = row.number ?? ""
297
256
 
298
257
  // One tuple per (locality, street) pair keeps the pool varied without ballooning memory.
299
258
  const pairKey = `${key}|${street}`.toLowerCase()
300
259
 
301
260
  if (seen.has(pairKey)) continue
302
261
  seen.add(pairKey)
303
- tuples.push({ house_number, street, locality, region: source.region, postcode: get(cells, iPost) })
262
+ tuples.push({ house_number, street, locality, region: source.region, postcode: row.postcode ?? "" })
304
263
  }
305
264
 
306
265
  return tuples
307
266
  }
308
267
 
309
268
  /**
310
- * Stride-sampled FR tuples (number/street/city/postcode). The countrywide CSV is 2.5 GB and insee-ordered; `awk NR%K`
311
- * strides the whole country instead of reading one département. Quoted commas survive because awk only FILTERS lines —
312
- * parsing stays in splitCSV.
269
+ * The FR stride: keep line 3, then every 211th. Both numbers are the awk pre-filter's, kept exactly — they select which
270
+ * rows land in the shard, so changing either re-rolls the sample.
313
271
  */
314
- function readFrTuples(limit: number): FrTuple[] {
315
- const r = spawnSync(
316
- "bash",
317
- ["-c", `unzip -p "${FR_SOURCE.zip}" "${FR_SOURCE.csv}" | awk 'NR==1 || NR%211==3' | head -n ${limit + 1}`],
318
- { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" }
319
- )
272
+ const FR_STRIDE = 211
273
+ const FR_STRIDE_OFFSET = 3
320
274
 
321
- if (r.status !== 0 && !r.stdout.length) {
322
- console.error(` WARN: unzip failed for ${FR_SOURCE.zip}`)
275
+ /**
276
+ * GeoNames population floor for a Canadian locality to enter the pool.
277
+ */
278
+ const CA_LOCALITY_MIN_POPULATION = 1000
279
+
280
+ /**
281
+ * Stride-sampled FR tuples (number/street/city/postcode). The countrywide CSV is 2.5 GB and insee-ordered; `awk NR%K`
282
+ * strides the whole country instead of reading one département.
283
+ *
284
+ * The stride counts PHYSICAL lines, so a record carrying a newline inside a quoted field is two lines to the stride and
285
+ * can be cut in half by it. That is a sampling artefact of striding before the parse, not of the parse: whichever lines
286
+ * survive are re-assembled into records by the CSV reader, and a halved record fails the field checks below. Hence
287
+ * `skipEmpty: false` and no quote handling on the line split — a blank line still advances the count, exactly as it did
288
+ * when this was `awk NR`, and the rows the stride selects are the rows every existing FR shard was built from.
289
+ */
290
+ async function readFrTuples(limit: number): Promise<FRTuple[]> {
291
+ if (!existsSync(FR_SOURCE.zip)) {
292
+ console.error(` WARN: ${FR_SOURCE.zip} is not cached — skipping ${FR_SOURCE.csv}`)
323
293
 
324
294
  return []
325
295
  }
326
296
 
327
- const lines = r.stdout.toString("utf8").split(/\r?\n/)
297
+ const encoder = new TextEncoder()
328
298
 
329
- if (lines.length < 2) return []
330
- const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
331
- const idx = (n: string) => header.indexOf(n)
299
+ const strided = TextSpliterator.fromAsync(readZipEntry(FR_SOURCE.zip, FR_SOURCE.csv), { skipEmpty: false })
300
+ // `index` is 0-based where awk's NR is 1-based, hence the +1: keep the header, then every 211th line at offset 3.
301
+ .filter((_line, index) => index === 0 || (index + 1) % FR_STRIDE === FR_STRIDE_OFFSET)
302
+ .take(limit + 1)
303
+ .map((line) => encoder.encode(line + "\n"))
332
304
 
333
- const iNum = idx("number"),
334
- iStreet = idx("street"),
335
- iCity = idx("city"),
336
- iPost = idx("postcode")
337
-
338
- const get = (cells: string[], i: number) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
339
- const tuples: FrTuple[] = []
305
+ const tuples: FRTuple[] = []
340
306
  const seen = new Set<string>()
341
307
 
342
- for (let li = 1; li < lines.length; li++) {
343
- if (!lines[li]) continue
344
- const cells = splitCSV(lines[li]!)
345
-
346
- const locality = get(cells, iCity),
347
- postcode = get(cells, iPost),
348
- street = get(cells, iStreet),
349
- house_number = get(cells, iNum)
308
+ for await (const row of readCSVRecords(strided)) {
309
+ const locality = row.city ?? "",
310
+ postcode = row.postcode ?? "",
311
+ street = row.street ?? "",
312
+ house_number = row.number ?? ""
350
313
 
351
314
  if (!cleanLocality(locality) || !/^\d{5}$/.test(postcode) || !street || !house_number) continue
352
315
  const key = `${locality}|${street}`.toLowerCase()
@@ -363,23 +326,29 @@ function readFrTuples(limit: number): FrTuple[] {
363
326
  * Canadian locality pools from the GeoNames dump (CC-BY 4.0): feature class P, admin1 10 (Québec) / 08 (Ontario),
364
327
  * population > 1000. GeoNames is the provenance-tracked source — no hand list.
365
328
  */
366
- function readCaLocalities(admin1: string): string[] {
367
- const r = spawnSync(
368
- "bash",
369
- [
370
- "-c",
371
- `unzip -p "${GEONAMES_CA}" CA.txt | awk -F'\\t' '$7=="P" && $9=="CA" && $11=="${admin1}" && $15>1000 {print $2}'`,
372
- ],
373
- { maxBuffer: 1024 * 1024 * 256, encoding: "utf8" }
374
- )
375
-
376
- if (r.status !== 0) {
377
- console.error(` WARN: GeoNames read failed (admin1=${admin1}) — is ${GEONAMES_CA} present?`)
329
+ async function readCaLocalities(admin1: string): Promise<string[]> {
330
+ if (!existsSync(GEONAMES_CA)) {
331
+ console.error(` WARN: ${GEONAMES_CA} is not cached — skipping CA localities (admin1=${admin1})`)
378
332
 
379
333
  return []
380
334
  }
381
335
 
382
- return [...new Set(r.stdout.split("\n").filter(cleanLocality))]
336
+ const localities = new Set<string>()
337
+
338
+ // GeoNames main dump columns, 0-based: 1 name, 6 feature class, 8 country, 10 admin1, 14 population.
339
+ for await (const columns of TSVSpliterator.fromAsync(readZipEntry(GEONAMES_CA, "CA.txt"), { header: false })) {
340
+ if (columns[6] !== "P" || columns[8] !== "CA" || columns[10] !== admin1) continue
341
+
342
+ if (Number(columns[14]) <= CA_LOCALITY_MIN_POPULATION) continue
343
+
344
+ const name = columns[1] ?? ""
345
+
346
+ if (cleanLocality(name)) {
347
+ localities.add(name)
348
+ }
349
+ }
350
+
351
+ return [...localities]
383
352
  }
384
353
 
385
354
  /**
@@ -389,25 +358,22 @@ function readCaLocalities(admin1: string): string[] {
389
358
  * validated against the codex 4-digit shape — a dump row that fails the contract is skipped, not emitted as a junk
390
359
  * label.
391
360
  */
392
- function readPostalTuples(
361
+ async function readPostalTuples(
393
362
  source: { zip: string; txt: string },
394
363
  opts: { withState: boolean }
395
- ): Array<AuTuple | NzTuple> {
396
- const r = spawnSync("unzip", ["-p", source.zip, source.txt], { maxBuffer: 1024 * 1024 * 64, encoding: "utf8" })
397
-
398
- if (r.status !== 0) {
399
- console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
364
+ ): Promise<Array<AUTuple | NZTuple>> {
365
+ if (!existsSync(source.zip)) {
366
+ console.error(` WARN: ${source.zip} is not cached — skipping ${source.txt}`)
400
367
 
401
368
  return []
402
369
  }
403
370
 
404
- const tuples: Array<AuTuple | NzTuple> = []
371
+ const tuples: Array<AUTuple | NZTuple> = []
405
372
  const seen = new Set<string>()
406
- const validPostcode = opts.withState ? isAuPostcode : isNzPostcode
373
+ const validPostcode = opts.withState ? isAuPostcode : isNZPostcode
407
374
 
408
- for (const line of r.stdout.split("\n")) {
409
- if (!line) continue
410
- const cols = line.split("\t")
375
+ // A GeoNames dump carries no header row, so every line is data.
376
+ for await (const cols of TSVSpliterator.fromAsync(readZipEntry(source.zip, source.txt), { header: false })) {
411
377
  const postcode = (cols[1] ?? "").trim()
412
378
  const locality = (cols[2] ?? "").trim()
413
379
  const region = (cols[4] ?? "").trim()
@@ -514,7 +480,7 @@ function makeCedex(random: () => number): string {
514
480
  /**
515
481
  * Compose an AU or NZ delivery-service phrase from the codex-sourced leaders. Same contract as makePoBoxPhrase: a
516
482
  * phrase built from a codex-known designator and a clean id must round-trip the codex matcher (isAuDeliveryService /
517
- * isNzDeliveryService) — a failure is a generation bug, loud. Noisy ids (commas / embedded spaces) are corpus-designed
483
+ * isNZDeliveryService) — a failure is a generation bug, loud. Noisy ids (commas / embedded spaces) are corpus-designed
518
484
  * adversarial forms, exempt.
519
485
  */
520
486
  function makeAuNzPoBoxPhrase(
@@ -532,14 +498,14 @@ function makeAuNzPoBoxPhrase(
532
498
  let num = maybeNoisifyBoxNumber(pickBoxNumber(random), random)
533
499
 
534
500
  // NZ CMB identifiers are alpha-led per the ADV358 example ("CMB B99").
535
- if (leader === "CMB" && validate === isNzDeliveryService) {
501
+ if (leader === "CMB" && validate === isNZDeliveryService) {
536
502
  num = `B${num}`
537
503
  }
538
504
 
539
505
  const phrase = `${caseDial(random, leader)} ${num}`
540
506
  // The "clean id" shape differs per system: ADV358 identifiers carry no separators at all, the AU
541
507
  // AMAS id (like the US one) tolerates dashes. Noisy ids outside the clean shape are exempt.
542
- const cleanID = validate === isNzDeliveryService ? /^[\dA-Za-z]+$/ : /^[\dA-Za-z][\dA-Za-z-]*$/
508
+ const cleanID = validate === isNZDeliveryService ? /^[\dA-Za-z]+$/ : /^[\dA-Za-z][\dA-Za-z-]*$/
543
509
 
544
510
  if (cleanID.test(num) && !validate(phrase)) {
545
511
  throw new Error(`generated a phrase the codex matcher rejects: "${phrase}"`)
@@ -614,7 +580,7 @@ function renderPmbUs(random: () => number, t: USTuple): Rendered {
614
580
  return { fmt: "pmb-bare", raw: `${road} ${phrase}`, components: { house_number: hn, street, po_box: phrase } }
615
581
  }
616
582
 
617
- function renderBpFr(random: () => number, t: FrTuple): Rendered {
583
+ function renderBpFr(random: () => number, t: FRTuple): Rendered {
618
584
  const phrase = makePoBoxPhrase(random, FR_LEADERS)
619
585
  const { locality, postcode: pc } = t
620
586
  const upper = random() < 0.5
@@ -651,7 +617,7 @@ function renderBpFr(random: () => number, t: FrTuple): Rendered {
651
617
  }
652
618
  }
653
619
 
654
- function renderCedexFr(random: () => number, t: FrTuple): Rendered {
620
+ function renderCedexFr(random: () => number, t: FRTuple): Rendered {
655
621
  const cedex = makeCedex(random)
656
622
  const { house_number: hn, street, locality, postcode: pc } = t
657
623
  const loc = random() < 0.6 ? locality.toUpperCase() : locality
@@ -703,7 +669,7 @@ function renderCaEn(random: () => number, loc: string): Rendered {
703
669
  return { fmt: "ca-en-bare", raw: phrase, components: { po_box: phrase } }
704
670
  }
705
671
 
706
- function renderAuPoBox(random: () => number, t: AuTuple): Rendered {
672
+ function renderAUPoBox(random: () => number, t: AUTuple): Rendered {
707
673
  const phrase = makeAuNzPoBoxPhrase(random, AU_LEADERS_CURRENT, AU_LEADERS_LEGACY, isAuDeliveryService)
708
674
  const { locality, region: reg, postcode: pc } = t
709
675
  const r = random()
@@ -737,8 +703,8 @@ function renderAuPoBox(random: () => number, t: AuTuple): Rendered {
737
703
  return { fmt: "au-venue", raw: `${v}, ${phrase}, ${loc} ${reg} ${pc}`, components: { venue: v, ...base } }
738
704
  }
739
705
 
740
- function renderNzPoBox(random: () => number, t: NzTuple): Rendered {
741
- const phrase = makeAuNzPoBoxPhrase(random, NZ_LEADERS_COMMON, NZ_LEADERS_RARE, isNzDeliveryService)
706
+ function renderNZPoBox(random: () => number, t: NZTuple): Rendered {
707
+ const phrase = makeAuNzPoBoxPhrase(random, NZ_LEADERS_COMMON, NZ_LEADERS_RARE, isNZDeliveryService)
742
708
  const { locality, postcode: pc } = t
743
709
  // NZ addresses are written mixed-case ("PO Box 4099, Timaru 7942") — no region line (ADV358).
744
710
  const base = { po_box: phrase, locality, postcode: pc }
@@ -795,7 +761,7 @@ export const poBoxCedexRecipe: ShardRecipe = {
795
761
  const usPool: USTuple[] = []
796
762
 
797
763
  for (const s of opts.golden ? [US_EVAL_SOURCE] : US_TRAIN_SOURCES) {
798
- const t = readUsTuples(s)
764
+ const t = await readUsTuples(s)
799
765
 
800
766
  console.error(` ${s.csv}: ${t.length} tuples`)
801
767
 
@@ -805,21 +771,21 @@ export const poBoxCedexRecipe: ShardRecipe = {
805
771
  }
806
772
 
807
773
  // FR + CA pools: stable locality-hash holdout (golden gets hash%10==0, train the rest).
808
- const frAll = readFrTuples(80_000)
774
+ const frAll = await readFrTuples(80_000)
809
775
  const frPool = frAll.filter((t) => isHoldoutLocality(t.locality) === opts.golden)
810
776
 
811
777
  console.error(` ${FR_SOURCE.csv}: ${frAll.length} tuples (${frPool.length} after holdout split)`)
812
778
 
813
- const qcAll = readCaLocalities("10")
814
- const onAll = readCaLocalities("08")
779
+ const qcAll = await readCaLocalities("10")
780
+ const onAll = await readCaLocalities("08")
815
781
  const qcPool = qcAll.filter((l) => isHoldoutLocality(l) === opts.golden)
816
782
  const onPool = onAll.filter((l) => isHoldoutLocality(l) === opts.golden)
817
783
 
818
784
  console.error(` GeoNames CA: QC ${qcAll.length}→${qcPool.length}, ON ${onAll.length}→${onPool.length}`)
819
785
 
820
786
  // AU/NZ pools: same stable locality-hash holdout as FR/CA.
821
- const auAll = readPostalTuples(GEONAMES_POSTAL_AU, { withState: true }) as AuTuple[]
822
- const nzAll = readPostalTuples(GEONAMES_POSTAL_NZ, { withState: false }) as NzTuple[]
787
+ const auAll = (await readPostalTuples(GEONAMES_POSTAL_AU, { withState: true })) as AUTuple[]
788
+ const nzAll = (await readPostalTuples(GEONAMES_POSTAL_NZ, { withState: false })) as NZTuple[]
823
789
  const auPool = auAll.filter((t) => isHoldoutLocality(t.locality) === opts.golden)
824
790
  const nzPool = nzAll.filter((t) => isHoldoutLocality(t.locality) === opts.golden)
825
791
 
@@ -827,7 +793,7 @@ export const poBoxCedexRecipe: ShardRecipe = {
827
793
 
828
794
  if (!usPool.length || !frPool.length || !qcPool.length || !onPool.length || !auPool.length || !nzPool.length) {
829
795
  throw new Error(
830
- "A base pool is empty — check /tmp/oa-cache and /tmp/geonames-cache (CA.zip, AU-postal.zip, NZ-postal.zip)."
796
+ `A base pool is empty — check ${dataRootPath("oa-cache")}, ${GEONAMES_CA}, ${GEONAMES_POSTAL_AU.zip}, and ${GEONAMES_POSTAL_NZ.zip}.`
831
797
  )
832
798
  }
833
799
 
@@ -877,11 +843,11 @@ export const poBoxCedexRecipe: ShardRecipe = {
877
843
  country = "CA"
878
844
  locale = "fr-CA"
879
845
  } else if (cls === "po-box-au") {
880
- rendered = renderAuPoBox(random, pick(random, auPool))
846
+ rendered = renderAUPoBox(random, pick(random, auPool))
881
847
  country = "AU"
882
848
  locale = "en-AU"
883
849
  } else if (cls === "po-box-nz") {
884
- rendered = renderNzPoBox(random, pick(random, nzPool))
850
+ rendered = renderNZPoBox(random, pick(random, nzPool))
885
851
  country = "NZ"
886
852
  locale = "en-NZ"
887
853
  } else if (cls === "po-box-us-military") {
@@ -9,7 +9,12 @@
9
9
  * scripts/build-po-box-shard.mjs.
10
10
  */
11
11
 
12
- import { synthesizeMilitaryPoBoxRow, synthesizePoBoxRow, type PoBoxBaseTuple } from "../synthesize-po-box.ts"
12
+ import {
13
+ synthesizeMilitaryPoBoxRow,
14
+ synthesizePoBoxRow,
15
+ type PoBoxBaseTuple,
16
+ } from "@mailwoman/corpus/synthesizers/po-box"
17
+
13
18
  import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } from "./scaffold.ts"
14
19
 
15
20
  const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"