@mailwoman/corpus 8.0.0 → 8.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (483) hide show
  1. package/out/src/adapter.d.ts +15 -5
  2. package/out/src/adapter.d.ts.map +1 -1
  3. package/out/src/adapter.js +11 -9
  4. package/out/src/adapter.js.map +1 -1
  5. package/out/src/adapters/ban/adapter.d.ts +7 -0
  6. package/out/src/adapters/ban/adapter.d.ts.map +1 -1
  7. package/out/src/adapters/ban/adapter.js +9 -2
  8. package/out/src/adapters/ban/adapter.js.map +1 -1
  9. package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
  10. package/out/src/adapters/ban/street-decompose.js +5 -1
  11. package/out/src/adapters/ban/street-decompose.js.map +1 -1
  12. package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
  13. package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
  14. package/out/src/adapters/fcc-bdc/adapter.js +15 -2
  15. package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
  16. package/out/src/adapters/geonames/adapter.d.ts +11 -0
  17. package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
  18. package/out/src/adapters/geonames/adapter.js +24 -5
  19. package/out/src/adapters/geonames/adapter.js.map +1 -1
  20. package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
  21. package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
  22. package/out/src/adapters/geonames-postal/adapter.js +14 -1
  23. package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
  24. package/out/src/adapters/gnaf/adapter.d.ts +10 -1
  25. package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
  26. package/out/src/adapters/gnaf/adapter.js +11 -2
  27. package/out/src/adapters/gnaf/adapter.js.map +1 -1
  28. package/out/src/adapters/gnaf/assemble.d.ts +18 -6
  29. package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
  30. package/out/src/adapters/gnaf/assemble.js +16 -6
  31. package/out/src/adapters/gnaf/assemble.js.map +1 -1
  32. package/out/src/adapters/index.d.ts.map +1 -1
  33. package/out/src/adapters/index.js.map +1 -1
  34. package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
  35. package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
  36. package/out/src/adapters/openaddresses/adapter.js +18 -3
  37. package/out/src/adapters/openaddresses/adapter.js.map +1 -1
  38. package/out/src/adapters/overture/adapter.d.ts +11 -0
  39. package/out/src/adapters/overture/adapter.d.ts.map +1 -1
  40. package/out/src/adapters/overture/adapter.js +12 -1
  41. package/out/src/adapters/overture/adapter.js.map +1 -1
  42. package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
  43. package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
  44. package/out/src/adapters/state-hi-schools/adapter.js +11 -0
  45. package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
  46. package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
  47. package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
  48. package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
  49. package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
  50. package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
  51. package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
  52. package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
  53. package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
  54. package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
  55. package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
  56. package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
  57. package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
  58. package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
  59. package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
  60. package/out/src/adapters/synth-po-box/adapter.js +13 -2
  61. package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
  62. package/out/src/adapters/tiger/adapter.d.ts +14 -1
  63. package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
  64. package/out/src/adapters/tiger/adapter.js +19 -4
  65. package/out/src/adapters/tiger/adapter.js.map +1 -1
  66. package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
  67. package/out/src/adapters/tiger/street-decompose.js.map +1 -1
  68. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
  69. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
  70. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
  71. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
  72. package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
  73. package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
  74. package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
  75. package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
  76. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
  77. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
  78. package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
  79. package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
  80. package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
  81. package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
  82. package/out/src/adapters/usgov-nad/adapter.js +12 -1
  83. package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
  84. package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
  85. package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
  86. package/out/src/adapters/usgov-nppes/adapter.js +11 -0
  87. package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
  88. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
  89. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
  90. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
  91. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
  92. package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
  93. package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
  94. package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
  95. package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
  96. package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
  97. package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
  98. package/out/src/adapters/wof-admin-json/adapter.js +15 -5
  99. package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
  100. package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
  101. package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
  102. package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
  103. package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
  104. package/out/src/align.d.ts +12 -4
  105. package/out/src/align.d.ts.map +1 -1
  106. package/out/src/align.js +9 -5
  107. package/out/src/align.js.map +1 -1
  108. package/out/src/build.d.ts +24 -8
  109. package/out/src/build.d.ts.map +1 -1
  110. package/out/src/build.js +3 -3
  111. package/out/src/build.js.map +1 -1
  112. package/out/src/codex/us-fips-state.d.ts +6 -2
  113. package/out/src/codex/us-fips-state.d.ts.map +1 -1
  114. package/out/src/codex/us-fips-state.js +3 -1
  115. package/out/src/codex/us-fips-state.js.map +1 -1
  116. package/out/src/golden.d.ts +21 -7
  117. package/out/src/golden.d.ts.map +1 -1
  118. package/out/src/golden.js +18 -10
  119. package/out/src/golden.js.map +1 -1
  120. package/out/src/license.d.ts +3 -1
  121. package/out/src/license.d.ts.map +1 -1
  122. package/out/src/license.js +4 -2
  123. package/out/src/license.js.map +1 -1
  124. package/out/src/parquet-wrapper/reader.d.ts +4 -2
  125. package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
  126. package/out/src/parquet-wrapper/reader.js +5 -4
  127. package/out/src/parquet-wrapper/reader.js.map +1 -1
  128. package/out/src/parquet-wrapper/schema.d.ts +14 -10
  129. package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
  130. package/out/src/parquet-wrapper/schema.js +6 -2
  131. package/out/src/parquet-wrapper/schema.js.map +1 -1
  132. package/out/src/parquet-wrapper/writer.d.ts +12 -4
  133. package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
  134. package/out/src/parquet-wrapper/writer.js +12 -4
  135. package/out/src/parquet-wrapper/writer.js.map +1 -1
  136. package/out/src/parquet.d.ts +21 -7
  137. package/out/src/parquet.d.ts.map +1 -1
  138. package/out/src/parquet.js +15 -4
  139. package/out/src/parquet.js.map +1 -1
  140. package/out/src/runner.d.ts +39 -13
  141. package/out/src/runner.d.ts.map +1 -1
  142. package/out/src/runner.js +7 -3
  143. package/out/src/runner.js.map +1 -1
  144. package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
  145. package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
  146. package/out/src/shard-recipes/anchor-absorption.js +14 -5
  147. package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
  148. package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
  149. package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
  150. package/out/src/shard-recipes/boundary-stress.js +13 -7
  151. package/out/src/shard-recipes/boundary-stress.js.map +1 -1
  152. package/out/src/shard-recipes/country-balanced.d.ts +4 -0
  153. package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
  154. package/out/src/shard-recipes/country-balanced.js +39 -11
  155. package/out/src/shard-recipes/country-balanced.js.map +1 -1
  156. package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
  157. package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
  158. package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
  159. package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
  160. package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
  161. package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
  162. package/out/src/shard-recipes/fr-admin-split.js +14 -3
  163. package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
  164. package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
  165. package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
  166. package/out/src/shard-recipes/fr-bare-street.js +4 -0
  167. package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
  168. package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
  169. package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
  170. package/out/src/shard-recipes/fr-fragment.js +25 -8
  171. package/out/src/shard-recipes/fr-fragment.js.map +1 -1
  172. package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
  173. package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
  174. package/out/src/shard-recipes/fr-lieudit.js +21 -8
  175. package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
  176. package/out/src/shard-recipes/fr-order.d.ts +4 -0
  177. package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
  178. package/out/src/shard-recipes/fr-order.js +27 -8
  179. package/out/src/shard-recipes/fr-order.js.map +1 -1
  180. package/out/src/shard-recipes/german.d.ts +4 -0
  181. package/out/src/shard-recipes/german.d.ts.map +1 -1
  182. package/out/src/shard-recipes/german.js +16 -6
  183. package/out/src/shard-recipes/german.js.map +1 -1
  184. package/out/src/shard-recipes/house-venue.d.ts +4 -0
  185. package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
  186. package/out/src/shard-recipes/house-venue.js +4 -0
  187. package/out/src/shard-recipes/house-venue.js.map +1 -1
  188. package/out/src/shard-recipes/index.d.ts +9 -3
  189. package/out/src/shard-recipes/index.d.ts.map +1 -1
  190. package/out/src/shard-recipes/index.js +12 -4
  191. package/out/src/shard-recipes/index.js.map +1 -1
  192. package/out/src/shard-recipes/intersection.d.ts +4 -0
  193. package/out/src/shard-recipes/intersection.d.ts.map +1 -1
  194. package/out/src/shard-recipes/intersection.js +27 -13
  195. package/out/src/shard-recipes/intersection.js.map +1 -1
  196. package/out/src/shard-recipes/locale.d.ts +4 -0
  197. package/out/src/shard-recipes/locale.d.ts.map +1 -1
  198. package/out/src/shard-recipes/locale.js +9 -4
  199. package/out/src/shard-recipes/locale.js.map +1 -1
  200. package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
  201. package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
  202. package/out/src/shard-recipes/nl-postcode.js +8 -2
  203. package/out/src/shard-recipes/nl-postcode.js.map +1 -1
  204. package/out/src/shard-recipes/no-fragment.d.ts +4 -0
  205. package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
  206. package/out/src/shard-recipes/no-fragment.js +8 -2
  207. package/out/src/shard-recipes/no-fragment.js.map +1 -1
  208. package/out/src/shard-recipes/no-street-led.d.ts +4 -0
  209. package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
  210. package/out/src/shard-recipes/no-street-led.js +5 -1
  211. package/out/src/shard-recipes/no-street-led.js.map +1 -1
  212. package/out/src/shard-recipes/no-street.d.ts +4 -0
  213. package/out/src/shard-recipes/no-street.d.ts.map +1 -1
  214. package/out/src/shard-recipes/no-street.js +4 -0
  215. package/out/src/shard-recipes/no-street.js.map +1 -1
  216. package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
  217. package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
  218. package/out/src/shard-recipes/po-box-cedex.js +99 -43
  219. package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
  220. package/out/src/shard-recipes/po-box.d.ts +4 -0
  221. package/out/src/shard-recipes/po-box.d.ts.map +1 -1
  222. package/out/src/shard-recipes/po-box.js +4 -0
  223. package/out/src/shard-recipes/po-box.js.map +1 -1
  224. package/out/src/shard-recipes/scaffold.d.ts +48 -16
  225. package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
  226. package/out/src/shard-recipes/scaffold.js +10 -6
  227. package/out/src/shard-recipes/scaffold.js.map +1 -1
  228. package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
  229. package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
  230. package/out/src/shard-recipes/si-bare-village.js +4 -0
  231. package/out/src/shard-recipes/si-bare-village.js.map +1 -1
  232. package/out/src/shard-recipes/street-affix.d.ts +4 -0
  233. package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
  234. package/out/src/shard-recipes/street-affix.js +29 -10
  235. package/out/src/shard-recipes/street-affix.js.map +1 -1
  236. package/out/src/shard-recipes/street-bare.d.ts +4 -0
  237. package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
  238. package/out/src/shard-recipes/street-bare.js +4 -0
  239. package/out/src/shard-recipes/street-bare.js.map +1 -1
  240. package/out/src/shard-recipes/street.d.ts +4 -0
  241. package/out/src/shard-recipes/street.d.ts.map +1 -1
  242. package/out/src/shard-recipes/street.js +4 -0
  243. package/out/src/shard-recipes/street.js.map +1 -1
  244. package/out/src/shard-recipes/unit.d.ts +4 -0
  245. package/out/src/shard-recipes/unit.d.ts.map +1 -1
  246. package/out/src/shard-recipes/unit.js +45 -14
  247. package/out/src/shard-recipes/unit.js.map +1 -1
  248. package/out/src/split.d.ts +18 -6
  249. package/out/src/split.d.ts.map +1 -1
  250. package/out/src/split.js +4 -2
  251. package/out/src/split.js.map +1 -1
  252. package/out/src/synthesize-anchor-absorption.d.ts +7 -1
  253. package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
  254. package/out/src/synthesize-anchor-absorption.js +28 -14
  255. package/out/src/synthesize-anchor-absorption.js.map +1 -1
  256. package/out/src/synthesize-boundary-stress.d.ts +3 -1
  257. package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
  258. package/out/src/synthesize-boundary-stress.js +35 -28
  259. package/out/src/synthesize-boundary-stress.js.map +1 -1
  260. package/out/src/synthesize-german.d.ts +15 -5
  261. package/out/src/synthesize-german.d.ts.map +1 -1
  262. package/out/src/synthesize-german.js +9 -3
  263. package/out/src/synthesize-german.js.map +1 -1
  264. package/out/src/synthesize-house-venue.d.ts.map +1 -1
  265. package/out/src/synthesize-house-venue.js +15 -15
  266. package/out/src/synthesize-house-venue.js.map +1 -1
  267. package/out/src/synthesize-intersection.d.ts +9 -3
  268. package/out/src/synthesize-intersection.d.ts.map +1 -1
  269. package/out/src/synthesize-intersection.js +25 -6
  270. package/out/src/synthesize-intersection.js.map +1 -1
  271. package/out/src/synthesize-no-street.d.ts +3 -1
  272. package/out/src/synthesize-no-street.d.ts.map +1 -1
  273. package/out/src/synthesize-no-street.js +5 -6
  274. package/out/src/synthesize-no-street.js.map +1 -1
  275. package/out/src/synthesize-po-box.d.ts +15 -5
  276. package/out/src/synthesize-po-box.d.ts.map +1 -1
  277. package/out/src/synthesize-po-box.js +21 -7
  278. package/out/src/synthesize-po-box.js.map +1 -1
  279. package/out/src/synthesize-street.d.ts +3 -1
  280. package/out/src/synthesize-street.d.ts.map +1 -1
  281. package/out/src/synthesize-street.js +6 -1
  282. package/out/src/synthesize-street.js.map +1 -1
  283. package/out/src/synthesize.d.ts +39 -13
  284. package/out/src/synthesize.d.ts.map +1 -1
  285. package/out/src/synthesize.js +66 -36
  286. package/out/src/synthesize.js.map +1 -1
  287. package/out/src/tokenize.d.ts +15 -5
  288. package/out/src/tokenize.d.ts.map +1 -1
  289. package/out/src/tokenize.js.map +1 -1
  290. package/out/src/tools/align-shard.d.ts.map +1 -1
  291. package/out/src/tools/align-shard.js +3 -1
  292. package/out/src/tools/align-shard.js.map +1 -1
  293. package/out/src/tools/audit.d.ts.map +1 -1
  294. package/out/src/tools/audit.js +20 -8
  295. package/out/src/tools/audit.js.map +1 -1
  296. package/out/src/tools/corpus-stats.d.ts.map +1 -1
  297. package/out/src/tools/corpus-stats.js.map +1 -1
  298. package/out/src/tools/fetch/ban.d.ts.map +1 -1
  299. package/out/src/tools/fetch/ban.js +17 -10
  300. package/out/src/tools/fetch/ban.js.map +1 -1
  301. package/out/src/tools/fetch/download.d.ts +30 -10
  302. package/out/src/tools/fetch/download.d.ts.map +1 -1
  303. package/out/src/tools/fetch/download.js +26 -6
  304. package/out/src/tools/fetch/download.js.map +1 -1
  305. package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
  306. package/out/src/tools/fetch/hrsa.js.map +1 -1
  307. package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
  308. package/out/src/tools/fetch/imls-pls.js +18 -18
  309. package/out/src/tools/fetch/imls-pls.js.map +1 -1
  310. package/out/src/tools/fetch/index.d.ts +3 -1
  311. package/out/src/tools/fetch/index.d.ts.map +1 -1
  312. package/out/src/tools/fetch/index.js +3 -1
  313. package/out/src/tools/fetch/index.js.map +1 -1
  314. package/out/src/tools/fetch/nad.d.ts +21 -7
  315. package/out/src/tools/fetch/nad.d.ts.map +1 -1
  316. package/out/src/tools/fetch/nad.js +5 -5
  317. package/out/src/tools/fetch/nad.js.map +1 -1
  318. package/out/src/tools/fetch/nppes.d.ts.map +1 -1
  319. package/out/src/tools/fetch/nppes.js +9 -15
  320. package/out/src/tools/fetch/nppes.js.map +1 -1
  321. package/out/src/tools/fetch/openaddresses.d.ts +3 -1
  322. package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
  323. package/out/src/tools/fetch/openaddresses.js +32 -23
  324. package/out/src/tools/fetch/openaddresses.js.map +1 -1
  325. package/out/src/tools/fetch/ppd.d.ts +3 -1
  326. package/out/src/tools/fetch/ppd.d.ts.map +1 -1
  327. package/out/src/tools/fetch/ppd.js +14 -3
  328. package/out/src/tools/fetch/ppd.js.map +1 -1
  329. package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
  330. package/out/src/tools/fetch/state-hi-schools.js +13 -6
  331. package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
  332. package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
  333. package/out/src/tools/fetch/state-sources.js +8 -3
  334. package/out/src/tools/fetch/state-sources.js.map +1 -1
  335. package/out/src/tools/fetch/tiger-full.d.ts +9 -3
  336. package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
  337. package/out/src/tools/fetch/tiger-full.js +35 -20
  338. package/out/src/tools/fetch/tiger-full.js.map +1 -1
  339. package/out/src/tools/gb-title-case.d.ts.map +1 -1
  340. package/out/src/tools/gb-title-case.js.map +1 -1
  341. package/out/src/tools/golden-expand.d.ts +24 -8
  342. package/out/src/tools/golden-expand.d.ts.map +1 -1
  343. package/out/src/tools/golden-expand.js +20 -12
  344. package/out/src/tools/golden-expand.js.map +1 -1
  345. package/out/src/tools/golden-promote.d.ts +18 -6
  346. package/out/src/tools/golden-promote.d.ts.map +1 -1
  347. package/out/src/tools/golden-promote.js +22 -7
  348. package/out/src/tools/golden-promote.js.map +1 -1
  349. package/out/src/tools/ingest-csv.d.ts +3 -1
  350. package/out/src/tools/ingest-csv.d.ts.map +1 -1
  351. package/out/src/tools/ingest-csv.js +17 -19
  352. package/out/src/tools/ingest-csv.js.map +1 -1
  353. package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
  354. package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
  355. package/out/src/tools/jsonl-to-parquet.js +11 -5
  356. package/out/src/tools/jsonl-to-parquet.js.map +1 -1
  357. package/out/src/tools/lint-shard-vocab.d.ts +36 -12
  358. package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
  359. package/out/src/tools/lint-shard-vocab.js +25 -11
  360. package/out/src/tools/lint-shard-vocab.js.map +1 -1
  361. package/out/src/tools/lint-shard.d.ts +30 -10
  362. package/out/src/tools/lint-shard.d.ts.map +1 -1
  363. package/out/src/tools/lint-shard.js +33 -20
  364. package/out/src/tools/lint-shard.js.map +1 -1
  365. package/out/src/tools/overlay-manifest.d.ts.map +1 -1
  366. package/out/src/tools/overlay-manifest.js +6 -4
  367. package/out/src/tools/overlay-manifest.js.map +1 -1
  368. package/out/src/tools/shard-kryptonite.d.ts +6 -2
  369. package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
  370. package/out/src/tools/shard-kryptonite.js +1 -1
  371. package/out/src/tools/shard-kryptonite.js.map +1 -1
  372. package/out/src/tools/shard-translit.d.ts +9 -3
  373. package/out/src/tools/shard-translit.d.ts.map +1 -1
  374. package/out/src/tools/shard-translit.js +2 -2
  375. package/out/src/tools/shard-translit.js.map +1 -1
  376. package/out/src/types.d.ts +51 -17
  377. package/out/src/types.d.ts.map +1 -1
  378. package/out/src/wof-json.d.ts +9 -3
  379. package/out/src/wof-json.d.ts.map +1 -1
  380. package/out/src/wof-json.js +6 -4
  381. package/out/src/wof-json.js.map +1 -1
  382. package/package.json +7 -6
  383. package/src/adapter.ts +24 -13
  384. package/src/adapters/ban/adapter.ts +12 -2
  385. package/src/adapters/ban/street-decompose.ts +7 -1
  386. package/src/adapters/fcc-bdc/adapter.ts +19 -3
  387. package/src/adapters/geonames/adapter.ts +28 -5
  388. package/src/adapters/geonames-postal/adapter.ts +17 -1
  389. package/src/adapters/gnaf/adapter.ts +15 -2
  390. package/src/adapters/gnaf/assemble.ts +40 -10
  391. package/src/adapters/index.ts +13 -0
  392. package/src/adapters/openaddresses/adapter.ts +22 -3
  393. package/src/adapters/overture/adapter.ts +18 -2
  394. package/src/adapters/state-hi-schools/adapter.ts +15 -0
  395. package/src/adapters/state-ia-contractors/adapter.ts +15 -0
  396. package/src/adapters/state-ny-notaries/adapter.ts +15 -0
  397. package/src/adapters/state-tx-notaries/adapter.ts +18 -2
  398. package/src/adapters/synth-po-box/adapter.ts +19 -2
  399. package/src/adapters/tiger/adapter.ts +23 -4
  400. package/src/adapters/tiger/street-decompose.ts +1 -0
  401. package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
  402. package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
  403. package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
  404. package/src/adapters/usgov-nad/adapter.ts +14 -1
  405. package/src/adapters/usgov-nppes/adapter.ts +13 -0
  406. package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
  407. package/src/adapters/wof-admin-jp/adapter.ts +20 -3
  408. package/src/adapters/wof-admin-json/adapter.ts +22 -7
  409. package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
  410. package/src/align.ts +21 -8
  411. package/src/build.ts +48 -11
  412. package/src/codex/us-fips-state.ts +6 -2
  413. package/src/golden.ts +28 -13
  414. package/src/license.ts +4 -2
  415. package/src/parquet-wrapper/reader.ts +6 -4
  416. package/src/parquet-wrapper/schema.ts +14 -10
  417. package/src/parquet-wrapper/writer.ts +12 -4
  418. package/src/parquet.ts +39 -8
  419. package/src/runner.ts +49 -15
  420. package/src/shard-recipes/anchor-absorption.ts +21 -5
  421. package/src/shard-recipes/boundary-stress.ts +20 -7
  422. package/src/shard-recipes/country-balanced.ts +78 -13
  423. package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
  424. package/src/shard-recipes/fr-admin-split.ts +30 -5
  425. package/src/shard-recipes/fr-bare-street.ts +10 -0
  426. package/src/shard-recipes/fr-fragment.ts +33 -8
  427. package/src/shard-recipes/fr-lieudit.ts +35 -9
  428. package/src/shard-recipes/fr-order.ts +47 -9
  429. package/src/shard-recipes/german.ts +32 -6
  430. package/src/shard-recipes/house-venue.ts +7 -0
  431. package/src/shard-recipes/index.ts +12 -4
  432. package/src/shard-recipes/intersection.ts +63 -14
  433. package/src/shard-recipes/locale.ts +37 -4
  434. package/src/shard-recipes/nl-postcode.ts +12 -2
  435. package/src/shard-recipes/no-fragment.ts +12 -2
  436. package/src/shard-recipes/no-street-led.ts +11 -1
  437. package/src/shard-recipes/no-street.ts +7 -0
  438. package/src/shard-recipes/po-box-cedex.ts +145 -45
  439. package/src/shard-recipes/po-box.ts +7 -0
  440. package/src/shard-recipes/scaffold.ts +52 -20
  441. package/src/shard-recipes/si-bare-village.ts +9 -0
  442. package/src/shard-recipes/street-affix.ts +88 -13
  443. package/src/shard-recipes/street-bare.ts +10 -0
  444. package/src/shard-recipes/street.ts +7 -0
  445. package/src/shard-recipes/unit.ts +79 -16
  446. package/src/split.ts +28 -7
  447. package/src/synthesize-anchor-absorption.ts +41 -14
  448. package/src/synthesize-boundary-stress.ts +57 -30
  449. package/src/synthesize-german.ts +30 -7
  450. package/src/synthesize-house-venue.ts +22 -16
  451. package/src/synthesize-intersection.ts +34 -7
  452. package/src/synthesize-no-street.ts +18 -10
  453. package/src/synthesize-po-box.ts +42 -10
  454. package/src/synthesize-street.ts +10 -2
  455. package/src/synthesize.ts +81 -38
  456. package/src/tokenize.ts +15 -5
  457. package/src/tools/align-shard.ts +6 -1
  458. package/src/tools/audit.ts +57 -11
  459. package/src/tools/corpus-stats.ts +9 -0
  460. package/src/tools/fetch/ban.ts +32 -10
  461. package/src/tools/fetch/download.ts +49 -12
  462. package/src/tools/fetch/hrsa.ts +3 -0
  463. package/src/tools/fetch/imls-pls.ts +28 -18
  464. package/src/tools/fetch/index.ts +3 -1
  465. package/src/tools/fetch/nad.ts +39 -12
  466. package/src/tools/fetch/nppes.ts +17 -15
  467. package/src/tools/fetch/openaddresses.ts +52 -24
  468. package/src/tools/fetch/ppd.ts +22 -3
  469. package/src/tools/fetch/state-hi-schools.ts +17 -6
  470. package/src/tools/fetch/state-sources.ts +16 -3
  471. package/src/tools/fetch/tiger-full.ts +66 -23
  472. package/src/tools/gb-title-case.ts +1 -0
  473. package/src/tools/golden-expand.ts +64 -20
  474. package/src/tools/golden-promote.ts +53 -13
  475. package/src/tools/ingest-csv.ts +41 -26
  476. package/src/tools/jsonl-to-parquet.ts +30 -10
  477. package/src/tools/lint-shard-vocab.ts +75 -24
  478. package/src/tools/lint-shard.ts +75 -31
  479. package/src/tools/overlay-manifest.ts +7 -4
  480. package/src/tools/shard-kryptonite.ts +11 -3
  481. package/src/tools/shard-translit.ts +20 -5
  482. package/src/types.ts +51 -17
  483. package/src/wof-json.ts +18 -7
@@ -37,6 +37,12 @@ import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
37
37
  // v2: the country TOKEN is decoupled from the skeleton's locale and drawn from a BROAD pool — every
38
38
  // ISO canonical name + every curated surface form (endonyms/abbrevs). Surface forms are over-weighted
39
39
  // so endonyms/abbrevs ("Deutschland","USA","NL") get strong signal.
40
+ /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
41
+ cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
42
+ output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
43
+ would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
44
+ extracted as named constants above. */
45
+
40
46
  const COUNTRY_FORM_POOL = (() => {
41
47
  const surface = Object.values(COUNTRY_SURFACE_FORMS).flat() // endonyms + abbrevs + canonical (curated)
42
48
  const names = [...CountryNames]
@@ -44,9 +50,15 @@ const COUNTRY_FORM_POOL = (() => {
44
50
  // all ~249 ISO canonical English names (breadth)
45
51
  return { surface, names }
46
52
  })()
47
- const COUNTRY_ABSENT_PROB = 0.3 // negatives: rows with NO country token → teach golden precision
48
53
 
49
- /** A cached OpenAddresses extract + the implied iso2/region/render-order. */
54
+ /**
55
+ * Negatives: rows with NO country token → teach golden precision.
56
+ */
57
+ const COUNTRY_ABSENT_PROB = 0.3
58
+
59
+ /**
60
+ * A cached OpenAddresses extract + the implied iso2/region/render-order.
61
+ */
50
62
  interface CountrySource {
51
63
  zip: string
52
64
  csv: string
@@ -55,8 +67,10 @@ interface CountrySource {
55
67
  order: string
56
68
  }
57
69
 
58
- // Multi-locale OA sources. region = implied admin where the extract is single-region (US states, DE
59
- // Saxony); countrywide extracts (FR/IT/NL) read region from the CSV when present.
70
+ /**
71
+ * Multi-locale OA sources. region = implied admin where the extract is single-region (US states, DE Saxony);
72
+ * countrywide extracts (FR/IT/NL) read region from the CSV when present.
73
+ */
60
74
  const SOURCES: readonly CountrySource[] = [
61
75
  { zip: "/tmp/oa-cache/us__ia__statewide.zip", csv: "us/ia/statewide.csv", iso2: "US", region: "IA", order: "us" },
62
76
  { zip: "/tmp/oa-cache/us__il__cook.zip", csv: "us/il/cook.csv", iso2: "US", region: "IL", order: "us" },
@@ -69,13 +83,18 @@ const SOURCES: readonly CountrySource[] = [
69
83
  { zip: "/tmp/oa-cache/it__countrywide.zip", csv: "it/countrywide.csv", iso2: "IT", region: "", order: "eu" },
70
84
  { zip: "/tmp/oa-cache/nl__countrywide.zip", csv: "nl/countrywide.csv", iso2: "NL", region: "", order: "eu" },
71
85
  ]
72
- // Held-out for --golden: Vermont (US holdout) + Berlin (DE holdout) — geographic split, never trained.
86
+
87
+ /**
88
+ * Held-out for --golden: Vermont (US holdout) + Berlin (DE holdout) — geographic split, never trained.
89
+ */
73
90
  const EVAL_SOURCES: readonly CountrySource[] = [
74
91
  { zip: "/tmp/oa-cache/us__vt__statewide.zip", csv: "us/vt/statewide.csv", iso2: "US", region: "VT", order: "us" },
75
92
  { zip: "/tmp/oa-cache/de__berlin.zip", csv: "de/berlin.csv", iso2: "DE", region: "", order: "eu" },
76
93
  ]
77
94
 
78
- /** A real tuple read out of a cached OA zip (+ the source's iso2/render-order). */
95
+ /**
96
+ * A real tuple read out of a cached OA zip (+ the source's iso2/render-order).
97
+ */
79
98
  interface CountryTuple {
80
99
  house_number: string
81
100
  street: string
@@ -88,6 +107,7 @@ interface CountryTuple {
88
107
 
89
108
  function splitCSV(line: string): string[] {
90
109
  const out: string[] = []
110
+
91
111
  let cur = "",
92
112
  inQ = false
93
113
 
@@ -98,6 +118,7 @@ function splitCSV(line: string): string[] {
98
118
  if (c === '"') {
99
119
  if (line[i + 1] === '"') {
100
120
  cur += '"'
121
+
101
122
  i++
102
123
  } else {
103
124
  inQ = false
@@ -114,6 +135,7 @@ function splitCSV(line: string): string[] {
114
135
  cur += c
115
136
  }
116
137
  }
138
+
117
139
  out.push(cur)
118
140
 
119
141
  return out
@@ -122,7 +144,8 @@ function splitCSV(line: string): string[] {
122
144
  function readTuples(source: CountrySource, limit: number): CountryTuple[] {
123
145
  // countrywide extracts (FR/IT/NL) are GB-scale — cap the bytes with `head` (read ~8 lines per wanted
124
146
  // tuple to survive dedup/skips) so the toString stays under V8's string limit.
125
- const maxLines = Math.max(limit * 8, 20000) + 1
147
+ const maxLines = Math.max(limit * 8, 20_000) + 1
148
+
126
149
  const r = spawnSync("bash", ["-c", `unzip -p "${source.zip}" "${source.csv}" | head -n ${maxLines}`], {
127
150
  maxBuffer: 1024 * 1024 * 1024,
128
151
  encoding: "buffer",
@@ -133,16 +156,19 @@ function readTuples(source: CountrySource, limit: number): CountryTuple[] {
133
156
 
134
157
  return []
135
158
  }
159
+
136
160
  const lines = r.stdout.toString("utf8").split(/\r?\n/)
137
161
 
138
162
  if (lines.length < 2) return []
139
163
  const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
140
164
  const idx = (n: string): number => header.indexOf(n)
165
+
141
166
  const iNum = idx("number"),
142
167
  iStreet = idx("street"),
143
168
  iCity = idx("city"),
144
169
  iRegion = idx("region"),
145
170
  iPost = idx("postcode")
171
+
146
172
  const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
147
173
  const tuples: CountryTuple[] = []
148
174
  const seen = new Set<string>()
@@ -150,6 +176,7 @@ function readTuples(source: CountrySource, limit: number): CountryTuple[] {
150
176
  for (let li = 1; li < lines.length && tuples.length < limit; li++) {
151
177
  if (!lines[li]) continue
152
178
  const cells = splitCSV(lines[li]!)
179
+
153
180
  const street = get(cells, iStreet),
154
181
  locality = get(cells, iCity),
155
182
  house_number = get(cells, iNum)
@@ -159,6 +186,7 @@ function readTuples(source: CountrySource, limit: number): CountryTuple[] {
159
186
 
160
187
  if (seen.has(key)) continue
161
188
  seen.add(key)
189
+
162
190
  tuples.push({
163
191
  house_number,
164
192
  street,
@@ -173,7 +201,9 @@ function readTuples(source: CountrySource, limit: number): CountryTuple[] {
173
201
  return tuples
174
202
  }
175
203
 
176
- /** Pick a country token from the BROAD pool, or null (a country-absent negative). v2. */
204
+ /**
205
+ * Pick a country token from the BROAD pool, or null (a country-absent negative). v2.
206
+ */
177
207
  function pickCountry(random: () => number): string | null {
178
208
  if (random() < COUNTRY_ABSENT_PROB) return null // negative — teaches "trailing token != always country"
179
209
  // 60% curated surface forms (endonym/abbrev variety), 40% broad ISO canonical names (coverage).
@@ -182,7 +212,9 @@ function pickCountry(random: () => number): string | null {
182
212
  return pool[Math.floor(random() * pool.length)]!
183
213
  }
184
214
 
185
- /** Render the address body in native-ish order. `country` null → a country-ABSENT negative row. */
215
+ /**
216
+ * Render the address body in native-ish order. `country` null → a country-ABSENT negative row.
217
+ */
186
218
  function renderCountry(
187
219
  random: () => number,
188
220
  t: CountryTuple,
@@ -198,6 +230,7 @@ function renderCountry(
198
230
  if (pc) {
199
231
  components.postcode = pc
200
232
  }
233
+
201
234
  let body: string
202
235
 
203
236
  if (order === "us") {
@@ -217,6 +250,7 @@ function renderCountry(
217
250
  // postcode is NOT a country (counters the v1 golden over-firing).
218
251
  return { fmt: "negative", raw: body, components }
219
252
  }
253
+
220
254
  const withC: Partial<Record<ComponentTag, string>> = { ...components, country }
221
255
  const r = random()
222
256
 
@@ -242,6 +276,7 @@ interface Homograph {
242
276
  cities: readonly string[]
243
277
  us: { role: "region" | "locality"; locality: string; region: string; postcodes: readonly string[] }
244
278
  }
279
+
245
280
  const HOMOGRAPHS: readonly Homograph[] = [
246
281
  {
247
282
  surface: "Georgia",
@@ -280,12 +315,14 @@ const HOMOGRAPHS: readonly Homograph[] = [
280
315
  us: { role: "locality", locality: "Turkey", region: "TX", postcodes: ["79261", "28393"] },
281
316
  },
282
317
  ]
318
+
283
319
  // 2-letter codes that are BOTH a US state abbrev AND an ISO country code → must read as region in US ctx.
284
320
  interface AbbrevRegion {
285
321
  code: string
286
322
  localities: readonly string[]
287
323
  postcodes: readonly string[]
288
324
  }
325
+
289
326
  const ABBREV_REGIONS: readonly AbbrevRegion[] = [
290
327
  { code: "CA", localities: ["Los Angeles", "Sacramento", "San Diego"], postcodes: ["90012", "95814", "92101"] }, // California / Canada
291
328
  { code: "GA", localities: ["Atlanta", "Savannah", "Macon"], postcodes: ["30309", "31401", "31201"] }, // Georgia(US) / Georgia
@@ -294,6 +331,7 @@ const ABBREV_REGIONS: readonly AbbrevRegion[] = [
294
331
  { code: "PA", localities: ["Philadelphia", "Pittsburgh"], postcodes: ["19103", "15222"] }, // Pennsylvania / Panama
295
332
  { code: "AL", localities: ["Birmingham", "Montgomery"], postcodes: ["35203", "36104"] }, // Alabama / Albania
296
333
  ]
334
+
297
335
  const STREET_POOL: readonly string[] = [
298
336
  "Main Street",
299
337
  "Oak Avenue",
@@ -306,6 +344,7 @@ const STREET_POOL: readonly string[] = [
306
344
  "2nd Avenue",
307
345
  "Maple Drive",
308
346
  ]
347
+
309
348
  const pick = <T>(random: () => number, arr: readonly T[]): T => arr[Math.floor(random() * arr.length)]!
310
349
  const houseNo = (random: () => number): string => String(1 + Math.floor(random() * 998))
311
350
 
@@ -320,6 +359,7 @@ function renderHomograph(random: () => number): {
320
359
  iso2: string
321
360
  } {
322
361
  const h = pick(random, HOMOGRAPHS)
362
+
323
363
  const hn = houseNo(random),
324
364
  street = pick(random, STREET_POOL)
325
365
 
@@ -327,12 +367,14 @@ function renderHomograph(random: () => number): {
327
367
  const city = pick(random, h.cities)
328
368
  const withStreet = random() < 0.6
329
369
  const raw = withStreet ? `${hn} ${street}, ${city}, ${h.surface}` : `${city}, ${h.surface}`
370
+
330
371
  const components: Partial<Record<ComponentTag, string>> = withStreet
331
372
  ? { house_number: hn, street, locality: city, country: h.surface }
332
373
  : { locality: city, country: h.surface }
333
374
 
334
375
  return { fmt: "homograph-country", raw, components, iso2: h.iso2 }
335
376
  }
377
+
336
378
  const pc = pick(random, h.us.postcodes)
337
379
 
338
380
  if (h.us.role === "region") {
@@ -354,7 +396,9 @@ function renderHomograph(random: () => number): {
354
396
  }
355
397
  }
356
398
 
357
- /** An abbrev-as-region negative: "123 Main St, Los Angeles, CA 90012" → region CA, NO country. */
399
+ /**
400
+ * An abbrev-as-region negative: "123 Main St, Los Angeles, CA 90012" → region CA, NO country.
401
+ */
358
402
  function renderAbbrevRegion(random: () => number): {
359
403
  fmt: string
360
404
  raw: string
@@ -362,6 +406,7 @@ function renderAbbrevRegion(random: () => number): {
362
406
  iso2: string
363
407
  } {
364
408
  const a = pick(random, ABBREV_REGIONS)
409
+
365
410
  const hn = houseNo(random),
366
411
  street = pick(random, STREET_POOL),
367
412
  locality = pick(random, a.localities),
@@ -375,9 +420,19 @@ function renderAbbrevRegion(random: () => number): {
375
420
  }
376
421
  }
377
422
 
378
- const HOMOGRAPH_FRAC = 0.22 // share of rows that are homograph contrast pairs
379
- const ABBREV_FRAC = 0.08 // share that are code-as-region negatives (cumulative with HOMOGRAPH_FRAC)
423
+ /**
424
+ * Share of rows that are homograph contrast pairs.
425
+ */
426
+ const HOMOGRAPH_FRAC = 0.22
427
+ /**
428
+ * Share that are code-as-region negatives (cumulative with HOMOGRAPH_FRAC)
429
+ */
430
+ const ABBREV_FRAC = 0.08
380
431
 
432
+ /**
433
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
434
+ * and `description` below for the surface form it generates.
435
+ */
381
436
  export const countryBalancedRecipe: ShardRecipe = {
382
437
  name: "country-balanced",
383
438
  description: "Balanced model-first country rows (#464): OA skeletons + ISO surface forms + homograph contrast pairs",
@@ -396,6 +451,7 @@ export const countryBalancedRecipe: ShardRecipe = {
396
451
 
397
452
  for (const s of sources) {
398
453
  const t = readTuples(s, perSource)
454
+
399
455
  console.error(` ${s.csv} (${s.iso2}): ${t.length} tuples`)
400
456
 
401
457
  for (const x of t) {
@@ -403,7 +459,7 @@ export const countryBalancedRecipe: ShardRecipe = {
403
459
  }
404
460
  }
405
461
 
406
- if (pool.length === 0) {
462
+ if (!pool.length) {
407
463
  throw new Error("No tuples — are the cached OA zips present in /tmp/oa-cache?")
408
464
  }
409
465
 
@@ -435,17 +491,22 @@ export const countryBalancedRecipe: ShardRecipe = {
435
491
 
436
492
  if (country && !rendered.raw.includes(country)) {
437
493
  skipped++
494
+
438
495
  continue
439
496
  }
440
497
  }
498
+
441
499
  const { raw, components } = rendered
442
500
  const localeTag = rowISO2 === "US" ? "en-US" : `${rowISO2.toLowerCase()}-${rowISO2}`
443
501
 
444
502
  if (opts.golden) {
445
503
  write(JSON.stringify({ raw, components, country: rowISO2 }) + "\n")
504
+
446
505
  emitted++
506
+
447
507
  continue
448
508
  }
509
+
449
510
  const canonical: CanonicalRow = {
450
511
  raw,
451
512
  components,
@@ -456,13 +517,17 @@ export const countryBalancedRecipe: ShardRecipe = {
456
517
  corpus_version: "0.4.0",
457
518
  license: "OpenAddresses multi-locale skeletons + injected ISO-3166 country surface forms (codex)",
458
519
  }
520
+
459
521
  const aligned = alignRow(canonical)
460
522
 
461
523
  if (aligned.kind !== "labeled" || !aligned.row) {
462
524
  skipped++
525
+
463
526
  continue
464
527
  }
528
+
465
529
  write(JSON.stringify({ ...aligned.row, synth_method: "country", synth_base_id: null }) + "\n")
530
+
466
531
  emitted++
467
532
  }
468
533
 
@@ -16,6 +16,10 @@
16
16
 
17
17
  import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
18
18
 
19
+ /**
20
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
21
+ * and `description` below for the surface form it generates.
22
+ */
19
23
  export const czPcFirstPrepositionRecipe: ShardRecipe = {
20
24
  name: "cz-pcfirst-preposition",
21
25
  description:
@@ -36,10 +40,13 @@ export const czPcFirstPrepositionRecipe: ShardRecipe = {
36
40
 
37
41
  if (!street || !city || !number || !postcode) {
38
42
  skipped++
43
+
39
44
  continue
40
45
  }
46
+
41
47
  const order = read % 3
42
48
  let raw: string
49
+
43
50
  const components: Record<string, string> = {
44
51
  street,
45
52
  house_number: number,
@@ -54,11 +61,13 @@ export const czPcFirstPrepositionRecipe: ShardRecipe = {
54
61
  } else {
55
62
  raw = `${city}, ${postcode}, ${street} ${number}`
56
63
  }
64
+
57
65
  const source_id = shardSourceID("synth-cz-pcfirst-preposition", {
58
66
  ...components,
59
67
  o: String(order),
60
68
  v: String(read),
61
69
  })
70
+
62
71
  const canonical = {
63
72
  raw,
64
73
  components,
@@ -39,10 +39,18 @@ import { alignRow } from "../align.ts"
39
39
  import type { CanonicalRow } from "../types.ts"
40
40
  import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
41
41
 
42
+ /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
43
+ cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
44
+ output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
45
+ would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
46
+ extracted as named constants above. */
47
+
42
48
  const DEFAULT_COMMUNES = "/tmp/reg/fr-communes.tsv"
43
49
  const LICENSE = "BAN (Base Adresse Nationale) commune+postcode tuples, rendered admin-split — see ingest SOURCE"
44
50
 
45
- /** One distinct commune row from the TSV, with the département derived from its postcode. */
51
+ /**
52
+ * One distinct commune row from the TSV, with the département derived from its postcode.
53
+ */
46
54
  interface CommuneRow {
47
55
  commune: string
48
56
  postcode: string
@@ -51,14 +59,18 @@ interface CommuneRow {
51
59
  lat: string | undefined
52
60
  }
53
61
 
54
- /** One rendered admin-split variant. */
62
+ /**
63
+ * One rendered admin-split variant.
64
+ */
55
65
  interface AdminSplitVariant {
56
66
  raw: string
57
67
  components: Partial<Record<ComponentTag, string>>
58
68
  order: string
59
69
  }
60
70
 
61
- /** Read the distinct commune TSV (commune, postcode, lon, lat); derive the département name. */
71
+ /**
72
+ * Read the distinct commune TSV (commune, postcode, lon, lat); derive the département name.
73
+ */
62
74
  async function readCommunes(path: string): Promise<CommuneRow[]> {
63
75
  const rows: CommuneRow[] = []
64
76
 
@@ -129,6 +141,10 @@ function render(random: () => number, c: CommuneRow): AdminSplitVariant {
129
141
  return out
130
142
  }
131
143
 
144
+ /**
145
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
146
+ * and `description` below for the surface form it generates.
147
+ */
132
148
  export const frAdminSplitRecipe: ShardRecipe = {
133
149
  name: "fr-admin-split",
134
150
  description: "FR admin-split rows: BAN communes → split département into `region` (+ canonical-FR preservation)",
@@ -139,14 +155,15 @@ export const frAdminSplitRecipe: ShardRecipe = {
139
155
  async run(opts, write) {
140
156
  // Legacy build-fr-admin-split-shard.mjs seeded `mulberry32(opts.seed)`.
141
157
  const random = makeMulberry32(opts.seed)
142
- const count = opts.count ?? 60000
158
+ const count = opts.count ?? 60_000
143
159
  const source = opts.sourceName ?? "synth-fr-admin-split"
144
160
  const communesPath = opts.communes ?? DEFAULT_COMMUNES
145
161
 
146
162
  const pool = await readCommunes(communesPath)
163
+
147
164
  console.error(` ${communesPath}: ${pool.length} communes with derived département`)
148
165
 
149
- if (pool.length === 0) {
166
+ if (!pool.length) {
150
167
  throw new Error("No communes — build the TSV from BAN first (see the recipe header).")
151
168
  }
152
169
 
@@ -165,14 +182,17 @@ export const frAdminSplitRecipe: ShardRecipe = {
165
182
 
166
183
  if (!values.every((v) => raw.includes(v))) {
167
184
  skipped++
185
+
168
186
  continue
169
187
  }
170
188
 
171
189
  if (opts.golden) {
172
190
  // Held-out eval slice for the centroid gate — carries the truth coordinate.
173
191
  write(JSON.stringify({ raw, components, country: "FR", lat: Number(base.lat), lon: Number(base.lon) }) + "\n")
192
+
174
193
  emitted++
175
194
  orderCounts[order] = (orderCounts[order] ?? 0) + 1
195
+
176
196
  continue
177
197
  }
178
198
 
@@ -181,6 +201,7 @@ export const frAdminSplitRecipe: ShardRecipe = {
181
201
  region: components.region,
182
202
  postcode: components.postcode,
183
203
  })
204
+
184
205
  const canonical: CanonicalRow = {
185
206
  raw,
186
207
  components,
@@ -191,16 +212,20 @@ export const frAdminSplitRecipe: ShardRecipe = {
191
212
  corpus_version: "0.5.0",
192
213
  license: LICENSE,
193
214
  }
215
+
194
216
  const aligned = alignRow(canonical)
195
217
 
196
218
  if (aligned.kind !== "labeled" || !aligned.row) {
197
219
  skipped++
220
+
198
221
  continue
199
222
  }
223
+
200
224
  write(
201
225
  JSON.stringify({ ...aligned.row, synth_method: "fr-admin-split", synth_order: order, synth_base_id: null }) +
202
226
  "\n"
203
227
  )
228
+
204
229
  emitted++
205
230
  orderCounts[order] = (orderCounts[order] ?? 0) + 1
206
231
  }
@@ -23,6 +23,10 @@
23
23
  import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
24
24
  import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
25
25
 
26
+ /**
27
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
28
+ * and `description` below for the surface form it generates.
29
+ */
26
30
  export const frBareStreetRecipe: ShardRecipe = {
27
31
  name: "fr-bare-street",
28
32
  description:
@@ -43,24 +47,30 @@ export const frBareStreetRecipe: ShardRecipe = {
43
47
 
44
48
  if (!fullStreet || !number || !locality) {
45
49
  skipped++
50
+
46
51
  continue
47
52
  }
53
+
48
54
  const { prefix, street } = decomposeFrStreet(fullStreet)
49
55
 
50
56
  // The failing class is the prefix-led FR street; a no-prefix nom_voie ("La Ville Mois") isn't it.
51
57
  if (!prefix || !street) {
52
58
  skipped++
59
+
53
60
  continue
54
61
  }
62
+
55
63
  const components: Record<string, string> = {
56
64
  house_number: number,
57
65
  street_prefix: prefix,
58
66
  street,
59
67
  locality,
60
68
  }
69
+
61
70
  // BARE comma form, number-before (FR's dominant order), NO postcode — the whole point.
62
71
  const raw = `${number} ${prefix} ${street}, ${locality}`
63
72
  const source_id = shardSourceID("synth-fr-bare-street", { ...components, v: String(read) })
73
+
64
74
  const canonical = {
65
75
  raw,
66
76
  components,
@@ -73,19 +73,30 @@ import { readFileSync } from "node:fs"
73
73
  import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
74
74
  import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
75
75
 
76
- /** House numbers, weighted toward the small values that dominate real BAN rows. */
76
+ /**
77
+ * House numbers, weighted toward the small values that dominate real BAN rows.
78
+ */
79
+ /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
80
+ cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
81
+ output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
82
+ would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
83
+ extracted as named constants above. */
84
+
77
85
  const HOUSE_NUMBERS = [
78
86
  1, 1, 2, 2, 3, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 20, 21, 24, 27, 30, 33, 42, 57, 68, 84, 102, 115, 140,
79
87
  ]
80
- /** FR alphanumeric house-number forms. `bis`/`ter` are separated; a bare letter is suffixed. */
88
+
89
+ /**
90
+ * FR alphanumeric house-number forms. `bis`/`ter` are separated; a bare letter is suffixed.
91
+ */
81
92
  const ALNUM_SUFFIXES = ["bis", "ter", "A", "B"]
82
93
 
83
94
  const norm = (value: string): string =>
84
95
  value
85
96
  .normalize("NFD")
86
- .replace(/[̀-ͯ]/g, "")
97
+ .replaceAll(/[̀-ͯ]/g, "")
87
98
  .toLowerCase()
88
- .replace(/\s+/g, " ")
99
+ .replaceAll(/\s+/g, " ")
89
100
  .trim()
90
101
 
91
102
  /**
@@ -130,12 +141,21 @@ export function frTitleCase(value: string): string {
130
141
  .join(" ")
131
142
  }
132
143
 
133
- /** Does the street name carry a particle? Decides the particle vs bare classification. */
144
+ /**
145
+ * Does the street name carry a particle? Decides the particle vs bare classification.
146
+ */
134
147
  const PARTICLE = /\b(de la|de l'|du|des|de|d'|le|la|les)\b/i
135
- /** Does the street name carry date material (a year, or a day + French month)? */
148
+
149
+ /**
150
+ * Does the street name carry date material (a year, or a day + French month)?
151
+ */
136
152
  const DATEISH =
137
153
  /\b(1[0-9]|20)\d{2}\b|\b\d{1,2}\s+(janvier|f[ée]vrier|mars|avril|mai|juin|juillet|ao[ûu]t|septembre|octobre|novembre|d[ée]cembre)\b/i
138
154
 
155
+ /**
156
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
157
+ * and `description` below for the surface form it generates.
158
+ */
139
159
  export const frFragmentRecipe: ShardRecipe = {
140
160
  name: "fr-fragment",
141
161
  description:
@@ -199,12 +219,14 @@ export const frFragmentRecipe: ShardRecipe = {
199
219
 
200
220
  if (!fullStreet) {
201
221
  skipped++
222
+
202
223
  continue
203
224
  }
204
225
 
205
226
  // THE SPLIT. A surface on the fragment board never enters training.
206
227
  if (excluded.has(norm(fullStreet))) {
207
228
  contaminated++
229
+
208
230
  continue
209
231
  }
210
232
 
@@ -214,6 +236,7 @@ export const frFragmentRecipe: ShardRecipe = {
214
236
  // is a different problem and would muddy the signal.
215
237
  if (!prefix || !street) {
216
238
  skipped++
239
+
217
240
  continue
218
241
  }
219
242
 
@@ -226,11 +249,13 @@ export const frFragmentRecipe: ShardRecipe = {
226
249
  const number = HOUSE_NUMBERS[Math.floor(random() * HOUSE_NUMBERS.length)]!
227
250
  const alnum = random() < 0.25
228
251
  const suffix = ALNUM_SUFFIXES[Math.floor(random() * ALNUM_SUFFIXES.length)]!
252
+
229
253
  const houseNumber = alnum
230
254
  ? suffix === "bis" || suffix === "ter"
231
255
  ? `${number} ${suffix}`
232
256
  : `${number}${suffix}`
233
257
  : String(number)
258
+
234
259
  components.house_number = houseNumber
235
260
  raw = `${houseNumber} ${prefix} ${street}`
236
261
  klass = alnum ? "alnum-housenumber" : "street-housenumber"
@@ -260,11 +285,11 @@ export const frFragmentRecipe: ShardRecipe = {
260
285
  }
261
286
  }
262
287
 
263
- // ---- the counter-distribution: bare localities, NO street anywhere in the row -------------
288
+ // MARK: counter-distribution — bare localities
264
289
  // Minted last so the locality pool is complete. Without these the model can satisfy every row
265
290
  // above by flipping its default from "bare => locality" to "bare => street", which trades one
266
291
  // broken prior for another and would show up as bare-locality collapsing on the board.
267
- const pool = [...localities].sort()
292
+ const pool = [...localities].toSorted()
268
293
  const wanted = Math.round((emitted / Math.max(1, 1 - bareLocalityProb)) * bareLocalityProb)
269
294
 
270
295
  for (let i = 0; i < wanted && pool.length; i++) {