@mailwoman/corpus 8.0.0 → 8.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (483) hide show
  1. package/out/src/adapter.d.ts +15 -5
  2. package/out/src/adapter.d.ts.map +1 -1
  3. package/out/src/adapter.js +11 -9
  4. package/out/src/adapter.js.map +1 -1
  5. package/out/src/adapters/ban/adapter.d.ts +7 -0
  6. package/out/src/adapters/ban/adapter.d.ts.map +1 -1
  7. package/out/src/adapters/ban/adapter.js +9 -2
  8. package/out/src/adapters/ban/adapter.js.map +1 -1
  9. package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
  10. package/out/src/adapters/ban/street-decompose.js +5 -1
  11. package/out/src/adapters/ban/street-decompose.js.map +1 -1
  12. package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
  13. package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
  14. package/out/src/adapters/fcc-bdc/adapter.js +15 -2
  15. package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
  16. package/out/src/adapters/geonames/adapter.d.ts +11 -0
  17. package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
  18. package/out/src/adapters/geonames/adapter.js +24 -5
  19. package/out/src/adapters/geonames/adapter.js.map +1 -1
  20. package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
  21. package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
  22. package/out/src/adapters/geonames-postal/adapter.js +14 -1
  23. package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
  24. package/out/src/adapters/gnaf/adapter.d.ts +10 -1
  25. package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
  26. package/out/src/adapters/gnaf/adapter.js +11 -2
  27. package/out/src/adapters/gnaf/adapter.js.map +1 -1
  28. package/out/src/adapters/gnaf/assemble.d.ts +18 -6
  29. package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
  30. package/out/src/adapters/gnaf/assemble.js +16 -6
  31. package/out/src/adapters/gnaf/assemble.js.map +1 -1
  32. package/out/src/adapters/index.d.ts.map +1 -1
  33. package/out/src/adapters/index.js.map +1 -1
  34. package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
  35. package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
  36. package/out/src/adapters/openaddresses/adapter.js +18 -3
  37. package/out/src/adapters/openaddresses/adapter.js.map +1 -1
  38. package/out/src/adapters/overture/adapter.d.ts +11 -0
  39. package/out/src/adapters/overture/adapter.d.ts.map +1 -1
  40. package/out/src/adapters/overture/adapter.js +12 -1
  41. package/out/src/adapters/overture/adapter.js.map +1 -1
  42. package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
  43. package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
  44. package/out/src/adapters/state-hi-schools/adapter.js +11 -0
  45. package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
  46. package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
  47. package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
  48. package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
  49. package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
  50. package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
  51. package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
  52. package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
  53. package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
  54. package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
  55. package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
  56. package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
  57. package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
  58. package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
  59. package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
  60. package/out/src/adapters/synth-po-box/adapter.js +13 -2
  61. package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
  62. package/out/src/adapters/tiger/adapter.d.ts +14 -1
  63. package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
  64. package/out/src/adapters/tiger/adapter.js +19 -4
  65. package/out/src/adapters/tiger/adapter.js.map +1 -1
  66. package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
  67. package/out/src/adapters/tiger/street-decompose.js.map +1 -1
  68. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
  69. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
  70. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
  71. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
  72. package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
  73. package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
  74. package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
  75. package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
  76. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
  77. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
  78. package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
  79. package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
  80. package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
  81. package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
  82. package/out/src/adapters/usgov-nad/adapter.js +12 -1
  83. package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
  84. package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
  85. package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
  86. package/out/src/adapters/usgov-nppes/adapter.js +11 -0
  87. package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
  88. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
  89. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
  90. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
  91. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
  92. package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
  93. package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
  94. package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
  95. package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
  96. package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
  97. package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
  98. package/out/src/adapters/wof-admin-json/adapter.js +15 -5
  99. package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
  100. package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
  101. package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
  102. package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
  103. package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
  104. package/out/src/align.d.ts +12 -4
  105. package/out/src/align.d.ts.map +1 -1
  106. package/out/src/align.js +9 -5
  107. package/out/src/align.js.map +1 -1
  108. package/out/src/build.d.ts +24 -8
  109. package/out/src/build.d.ts.map +1 -1
  110. package/out/src/build.js +3 -3
  111. package/out/src/build.js.map +1 -1
  112. package/out/src/codex/us-fips-state.d.ts +6 -2
  113. package/out/src/codex/us-fips-state.d.ts.map +1 -1
  114. package/out/src/codex/us-fips-state.js +3 -1
  115. package/out/src/codex/us-fips-state.js.map +1 -1
  116. package/out/src/golden.d.ts +21 -7
  117. package/out/src/golden.d.ts.map +1 -1
  118. package/out/src/golden.js +18 -10
  119. package/out/src/golden.js.map +1 -1
  120. package/out/src/license.d.ts +3 -1
  121. package/out/src/license.d.ts.map +1 -1
  122. package/out/src/license.js +4 -2
  123. package/out/src/license.js.map +1 -1
  124. package/out/src/parquet-wrapper/reader.d.ts +4 -2
  125. package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
  126. package/out/src/parquet-wrapper/reader.js +5 -4
  127. package/out/src/parquet-wrapper/reader.js.map +1 -1
  128. package/out/src/parquet-wrapper/schema.d.ts +14 -10
  129. package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
  130. package/out/src/parquet-wrapper/schema.js +6 -2
  131. package/out/src/parquet-wrapper/schema.js.map +1 -1
  132. package/out/src/parquet-wrapper/writer.d.ts +12 -4
  133. package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
  134. package/out/src/parquet-wrapper/writer.js +12 -4
  135. package/out/src/parquet-wrapper/writer.js.map +1 -1
  136. package/out/src/parquet.d.ts +21 -7
  137. package/out/src/parquet.d.ts.map +1 -1
  138. package/out/src/parquet.js +15 -4
  139. package/out/src/parquet.js.map +1 -1
  140. package/out/src/runner.d.ts +39 -13
  141. package/out/src/runner.d.ts.map +1 -1
  142. package/out/src/runner.js +7 -3
  143. package/out/src/runner.js.map +1 -1
  144. package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
  145. package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
  146. package/out/src/shard-recipes/anchor-absorption.js +14 -5
  147. package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
  148. package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
  149. package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
  150. package/out/src/shard-recipes/boundary-stress.js +13 -7
  151. package/out/src/shard-recipes/boundary-stress.js.map +1 -1
  152. package/out/src/shard-recipes/country-balanced.d.ts +4 -0
  153. package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
  154. package/out/src/shard-recipes/country-balanced.js +39 -11
  155. package/out/src/shard-recipes/country-balanced.js.map +1 -1
  156. package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
  157. package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
  158. package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
  159. package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
  160. package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
  161. package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
  162. package/out/src/shard-recipes/fr-admin-split.js +14 -3
  163. package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
  164. package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
  165. package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
  166. package/out/src/shard-recipes/fr-bare-street.js +4 -0
  167. package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
  168. package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
  169. package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
  170. package/out/src/shard-recipes/fr-fragment.js +25 -8
  171. package/out/src/shard-recipes/fr-fragment.js.map +1 -1
  172. package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
  173. package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
  174. package/out/src/shard-recipes/fr-lieudit.js +21 -8
  175. package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
  176. package/out/src/shard-recipes/fr-order.d.ts +4 -0
  177. package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
  178. package/out/src/shard-recipes/fr-order.js +27 -8
  179. package/out/src/shard-recipes/fr-order.js.map +1 -1
  180. package/out/src/shard-recipes/german.d.ts +4 -0
  181. package/out/src/shard-recipes/german.d.ts.map +1 -1
  182. package/out/src/shard-recipes/german.js +16 -6
  183. package/out/src/shard-recipes/german.js.map +1 -1
  184. package/out/src/shard-recipes/house-venue.d.ts +4 -0
  185. package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
  186. package/out/src/shard-recipes/house-venue.js +4 -0
  187. package/out/src/shard-recipes/house-venue.js.map +1 -1
  188. package/out/src/shard-recipes/index.d.ts +9 -3
  189. package/out/src/shard-recipes/index.d.ts.map +1 -1
  190. package/out/src/shard-recipes/index.js +12 -4
  191. package/out/src/shard-recipes/index.js.map +1 -1
  192. package/out/src/shard-recipes/intersection.d.ts +4 -0
  193. package/out/src/shard-recipes/intersection.d.ts.map +1 -1
  194. package/out/src/shard-recipes/intersection.js +27 -13
  195. package/out/src/shard-recipes/intersection.js.map +1 -1
  196. package/out/src/shard-recipes/locale.d.ts +4 -0
  197. package/out/src/shard-recipes/locale.d.ts.map +1 -1
  198. package/out/src/shard-recipes/locale.js +9 -4
  199. package/out/src/shard-recipes/locale.js.map +1 -1
  200. package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
  201. package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
  202. package/out/src/shard-recipes/nl-postcode.js +8 -2
  203. package/out/src/shard-recipes/nl-postcode.js.map +1 -1
  204. package/out/src/shard-recipes/no-fragment.d.ts +4 -0
  205. package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
  206. package/out/src/shard-recipes/no-fragment.js +8 -2
  207. package/out/src/shard-recipes/no-fragment.js.map +1 -1
  208. package/out/src/shard-recipes/no-street-led.d.ts +4 -0
  209. package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
  210. package/out/src/shard-recipes/no-street-led.js +5 -1
  211. package/out/src/shard-recipes/no-street-led.js.map +1 -1
  212. package/out/src/shard-recipes/no-street.d.ts +4 -0
  213. package/out/src/shard-recipes/no-street.d.ts.map +1 -1
  214. package/out/src/shard-recipes/no-street.js +4 -0
  215. package/out/src/shard-recipes/no-street.js.map +1 -1
  216. package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
  217. package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
  218. package/out/src/shard-recipes/po-box-cedex.js +99 -43
  219. package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
  220. package/out/src/shard-recipes/po-box.d.ts +4 -0
  221. package/out/src/shard-recipes/po-box.d.ts.map +1 -1
  222. package/out/src/shard-recipes/po-box.js +4 -0
  223. package/out/src/shard-recipes/po-box.js.map +1 -1
  224. package/out/src/shard-recipes/scaffold.d.ts +48 -16
  225. package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
  226. package/out/src/shard-recipes/scaffold.js +10 -6
  227. package/out/src/shard-recipes/scaffold.js.map +1 -1
  228. package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
  229. package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
  230. package/out/src/shard-recipes/si-bare-village.js +4 -0
  231. package/out/src/shard-recipes/si-bare-village.js.map +1 -1
  232. package/out/src/shard-recipes/street-affix.d.ts +4 -0
  233. package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
  234. package/out/src/shard-recipes/street-affix.js +29 -10
  235. package/out/src/shard-recipes/street-affix.js.map +1 -1
  236. package/out/src/shard-recipes/street-bare.d.ts +4 -0
  237. package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
  238. package/out/src/shard-recipes/street-bare.js +4 -0
  239. package/out/src/shard-recipes/street-bare.js.map +1 -1
  240. package/out/src/shard-recipes/street.d.ts +4 -0
  241. package/out/src/shard-recipes/street.d.ts.map +1 -1
  242. package/out/src/shard-recipes/street.js +4 -0
  243. package/out/src/shard-recipes/street.js.map +1 -1
  244. package/out/src/shard-recipes/unit.d.ts +4 -0
  245. package/out/src/shard-recipes/unit.d.ts.map +1 -1
  246. package/out/src/shard-recipes/unit.js +45 -14
  247. package/out/src/shard-recipes/unit.js.map +1 -1
  248. package/out/src/split.d.ts +18 -6
  249. package/out/src/split.d.ts.map +1 -1
  250. package/out/src/split.js +4 -2
  251. package/out/src/split.js.map +1 -1
  252. package/out/src/synthesize-anchor-absorption.d.ts +7 -1
  253. package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
  254. package/out/src/synthesize-anchor-absorption.js +28 -14
  255. package/out/src/synthesize-anchor-absorption.js.map +1 -1
  256. package/out/src/synthesize-boundary-stress.d.ts +3 -1
  257. package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
  258. package/out/src/synthesize-boundary-stress.js +35 -28
  259. package/out/src/synthesize-boundary-stress.js.map +1 -1
  260. package/out/src/synthesize-german.d.ts +15 -5
  261. package/out/src/synthesize-german.d.ts.map +1 -1
  262. package/out/src/synthesize-german.js +9 -3
  263. package/out/src/synthesize-german.js.map +1 -1
  264. package/out/src/synthesize-house-venue.d.ts.map +1 -1
  265. package/out/src/synthesize-house-venue.js +15 -15
  266. package/out/src/synthesize-house-venue.js.map +1 -1
  267. package/out/src/synthesize-intersection.d.ts +9 -3
  268. package/out/src/synthesize-intersection.d.ts.map +1 -1
  269. package/out/src/synthesize-intersection.js +25 -6
  270. package/out/src/synthesize-intersection.js.map +1 -1
  271. package/out/src/synthesize-no-street.d.ts +3 -1
  272. package/out/src/synthesize-no-street.d.ts.map +1 -1
  273. package/out/src/synthesize-no-street.js +5 -6
  274. package/out/src/synthesize-no-street.js.map +1 -1
  275. package/out/src/synthesize-po-box.d.ts +15 -5
  276. package/out/src/synthesize-po-box.d.ts.map +1 -1
  277. package/out/src/synthesize-po-box.js +21 -7
  278. package/out/src/synthesize-po-box.js.map +1 -1
  279. package/out/src/synthesize-street.d.ts +3 -1
  280. package/out/src/synthesize-street.d.ts.map +1 -1
  281. package/out/src/synthesize-street.js +6 -1
  282. package/out/src/synthesize-street.js.map +1 -1
  283. package/out/src/synthesize.d.ts +39 -13
  284. package/out/src/synthesize.d.ts.map +1 -1
  285. package/out/src/synthesize.js +66 -36
  286. package/out/src/synthesize.js.map +1 -1
  287. package/out/src/tokenize.d.ts +15 -5
  288. package/out/src/tokenize.d.ts.map +1 -1
  289. package/out/src/tokenize.js.map +1 -1
  290. package/out/src/tools/align-shard.d.ts.map +1 -1
  291. package/out/src/tools/align-shard.js +3 -1
  292. package/out/src/tools/align-shard.js.map +1 -1
  293. package/out/src/tools/audit.d.ts.map +1 -1
  294. package/out/src/tools/audit.js +20 -8
  295. package/out/src/tools/audit.js.map +1 -1
  296. package/out/src/tools/corpus-stats.d.ts.map +1 -1
  297. package/out/src/tools/corpus-stats.js.map +1 -1
  298. package/out/src/tools/fetch/ban.d.ts.map +1 -1
  299. package/out/src/tools/fetch/ban.js +17 -10
  300. package/out/src/tools/fetch/ban.js.map +1 -1
  301. package/out/src/tools/fetch/download.d.ts +30 -10
  302. package/out/src/tools/fetch/download.d.ts.map +1 -1
  303. package/out/src/tools/fetch/download.js +26 -6
  304. package/out/src/tools/fetch/download.js.map +1 -1
  305. package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
  306. package/out/src/tools/fetch/hrsa.js.map +1 -1
  307. package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
  308. package/out/src/tools/fetch/imls-pls.js +18 -18
  309. package/out/src/tools/fetch/imls-pls.js.map +1 -1
  310. package/out/src/tools/fetch/index.d.ts +3 -1
  311. package/out/src/tools/fetch/index.d.ts.map +1 -1
  312. package/out/src/tools/fetch/index.js +3 -1
  313. package/out/src/tools/fetch/index.js.map +1 -1
  314. package/out/src/tools/fetch/nad.d.ts +21 -7
  315. package/out/src/tools/fetch/nad.d.ts.map +1 -1
  316. package/out/src/tools/fetch/nad.js +5 -5
  317. package/out/src/tools/fetch/nad.js.map +1 -1
  318. package/out/src/tools/fetch/nppes.d.ts.map +1 -1
  319. package/out/src/tools/fetch/nppes.js +9 -15
  320. package/out/src/tools/fetch/nppes.js.map +1 -1
  321. package/out/src/tools/fetch/openaddresses.d.ts +3 -1
  322. package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
  323. package/out/src/tools/fetch/openaddresses.js +32 -23
  324. package/out/src/tools/fetch/openaddresses.js.map +1 -1
  325. package/out/src/tools/fetch/ppd.d.ts +3 -1
  326. package/out/src/tools/fetch/ppd.d.ts.map +1 -1
  327. package/out/src/tools/fetch/ppd.js +14 -3
  328. package/out/src/tools/fetch/ppd.js.map +1 -1
  329. package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
  330. package/out/src/tools/fetch/state-hi-schools.js +13 -6
  331. package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
  332. package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
  333. package/out/src/tools/fetch/state-sources.js +8 -3
  334. package/out/src/tools/fetch/state-sources.js.map +1 -1
  335. package/out/src/tools/fetch/tiger-full.d.ts +9 -3
  336. package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
  337. package/out/src/tools/fetch/tiger-full.js +35 -20
  338. package/out/src/tools/fetch/tiger-full.js.map +1 -1
  339. package/out/src/tools/gb-title-case.d.ts.map +1 -1
  340. package/out/src/tools/gb-title-case.js.map +1 -1
  341. package/out/src/tools/golden-expand.d.ts +24 -8
  342. package/out/src/tools/golden-expand.d.ts.map +1 -1
  343. package/out/src/tools/golden-expand.js +20 -12
  344. package/out/src/tools/golden-expand.js.map +1 -1
  345. package/out/src/tools/golden-promote.d.ts +18 -6
  346. package/out/src/tools/golden-promote.d.ts.map +1 -1
  347. package/out/src/tools/golden-promote.js +22 -7
  348. package/out/src/tools/golden-promote.js.map +1 -1
  349. package/out/src/tools/ingest-csv.d.ts +3 -1
  350. package/out/src/tools/ingest-csv.d.ts.map +1 -1
  351. package/out/src/tools/ingest-csv.js +17 -19
  352. package/out/src/tools/ingest-csv.js.map +1 -1
  353. package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
  354. package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
  355. package/out/src/tools/jsonl-to-parquet.js +11 -5
  356. package/out/src/tools/jsonl-to-parquet.js.map +1 -1
  357. package/out/src/tools/lint-shard-vocab.d.ts +36 -12
  358. package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
  359. package/out/src/tools/lint-shard-vocab.js +25 -11
  360. package/out/src/tools/lint-shard-vocab.js.map +1 -1
  361. package/out/src/tools/lint-shard.d.ts +30 -10
  362. package/out/src/tools/lint-shard.d.ts.map +1 -1
  363. package/out/src/tools/lint-shard.js +33 -20
  364. package/out/src/tools/lint-shard.js.map +1 -1
  365. package/out/src/tools/overlay-manifest.d.ts.map +1 -1
  366. package/out/src/tools/overlay-manifest.js +6 -4
  367. package/out/src/tools/overlay-manifest.js.map +1 -1
  368. package/out/src/tools/shard-kryptonite.d.ts +6 -2
  369. package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
  370. package/out/src/tools/shard-kryptonite.js +1 -1
  371. package/out/src/tools/shard-kryptonite.js.map +1 -1
  372. package/out/src/tools/shard-translit.d.ts +9 -3
  373. package/out/src/tools/shard-translit.d.ts.map +1 -1
  374. package/out/src/tools/shard-translit.js +2 -2
  375. package/out/src/tools/shard-translit.js.map +1 -1
  376. package/out/src/types.d.ts +51 -17
  377. package/out/src/types.d.ts.map +1 -1
  378. package/out/src/wof-json.d.ts +9 -3
  379. package/out/src/wof-json.d.ts.map +1 -1
  380. package/out/src/wof-json.js +6 -4
  381. package/out/src/wof-json.js.map +1 -1
  382. package/package.json +7 -6
  383. package/src/adapter.ts +24 -13
  384. package/src/adapters/ban/adapter.ts +12 -2
  385. package/src/adapters/ban/street-decompose.ts +7 -1
  386. package/src/adapters/fcc-bdc/adapter.ts +19 -3
  387. package/src/adapters/geonames/adapter.ts +28 -5
  388. package/src/adapters/geonames-postal/adapter.ts +17 -1
  389. package/src/adapters/gnaf/adapter.ts +15 -2
  390. package/src/adapters/gnaf/assemble.ts +40 -10
  391. package/src/adapters/index.ts +13 -0
  392. package/src/adapters/openaddresses/adapter.ts +22 -3
  393. package/src/adapters/overture/adapter.ts +18 -2
  394. package/src/adapters/state-hi-schools/adapter.ts +15 -0
  395. package/src/adapters/state-ia-contractors/adapter.ts +15 -0
  396. package/src/adapters/state-ny-notaries/adapter.ts +15 -0
  397. package/src/adapters/state-tx-notaries/adapter.ts +18 -2
  398. package/src/adapters/synth-po-box/adapter.ts +19 -2
  399. package/src/adapters/tiger/adapter.ts +23 -4
  400. package/src/adapters/tiger/street-decompose.ts +1 -0
  401. package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
  402. package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
  403. package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
  404. package/src/adapters/usgov-nad/adapter.ts +14 -1
  405. package/src/adapters/usgov-nppes/adapter.ts +13 -0
  406. package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
  407. package/src/adapters/wof-admin-jp/adapter.ts +20 -3
  408. package/src/adapters/wof-admin-json/adapter.ts +22 -7
  409. package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
  410. package/src/align.ts +21 -8
  411. package/src/build.ts +48 -11
  412. package/src/codex/us-fips-state.ts +6 -2
  413. package/src/golden.ts +28 -13
  414. package/src/license.ts +4 -2
  415. package/src/parquet-wrapper/reader.ts +6 -4
  416. package/src/parquet-wrapper/schema.ts +14 -10
  417. package/src/parquet-wrapper/writer.ts +12 -4
  418. package/src/parquet.ts +39 -8
  419. package/src/runner.ts +49 -15
  420. package/src/shard-recipes/anchor-absorption.ts +21 -5
  421. package/src/shard-recipes/boundary-stress.ts +20 -7
  422. package/src/shard-recipes/country-balanced.ts +78 -13
  423. package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
  424. package/src/shard-recipes/fr-admin-split.ts +30 -5
  425. package/src/shard-recipes/fr-bare-street.ts +10 -0
  426. package/src/shard-recipes/fr-fragment.ts +33 -8
  427. package/src/shard-recipes/fr-lieudit.ts +35 -9
  428. package/src/shard-recipes/fr-order.ts +47 -9
  429. package/src/shard-recipes/german.ts +32 -6
  430. package/src/shard-recipes/house-venue.ts +7 -0
  431. package/src/shard-recipes/index.ts +12 -4
  432. package/src/shard-recipes/intersection.ts +63 -14
  433. package/src/shard-recipes/locale.ts +37 -4
  434. package/src/shard-recipes/nl-postcode.ts +12 -2
  435. package/src/shard-recipes/no-fragment.ts +12 -2
  436. package/src/shard-recipes/no-street-led.ts +11 -1
  437. package/src/shard-recipes/no-street.ts +7 -0
  438. package/src/shard-recipes/po-box-cedex.ts +145 -45
  439. package/src/shard-recipes/po-box.ts +7 -0
  440. package/src/shard-recipes/scaffold.ts +52 -20
  441. package/src/shard-recipes/si-bare-village.ts +9 -0
  442. package/src/shard-recipes/street-affix.ts +88 -13
  443. package/src/shard-recipes/street-bare.ts +10 -0
  444. package/src/shard-recipes/street.ts +7 -0
  445. package/src/shard-recipes/unit.ts +79 -16
  446. package/src/split.ts +28 -7
  447. package/src/synthesize-anchor-absorption.ts +41 -14
  448. package/src/synthesize-boundary-stress.ts +57 -30
  449. package/src/synthesize-german.ts +30 -7
  450. package/src/synthesize-house-venue.ts +22 -16
  451. package/src/synthesize-intersection.ts +34 -7
  452. package/src/synthesize-no-street.ts +18 -10
  453. package/src/synthesize-po-box.ts +42 -10
  454. package/src/synthesize-street.ts +10 -2
  455. package/src/synthesize.ts +81 -38
  456. package/src/tokenize.ts +15 -5
  457. package/src/tools/align-shard.ts +6 -1
  458. package/src/tools/audit.ts +57 -11
  459. package/src/tools/corpus-stats.ts +9 -0
  460. package/src/tools/fetch/ban.ts +32 -10
  461. package/src/tools/fetch/download.ts +49 -12
  462. package/src/tools/fetch/hrsa.ts +3 -0
  463. package/src/tools/fetch/imls-pls.ts +28 -18
  464. package/src/tools/fetch/index.ts +3 -1
  465. package/src/tools/fetch/nad.ts +39 -12
  466. package/src/tools/fetch/nppes.ts +17 -15
  467. package/src/tools/fetch/openaddresses.ts +52 -24
  468. package/src/tools/fetch/ppd.ts +22 -3
  469. package/src/tools/fetch/state-hi-schools.ts +17 -6
  470. package/src/tools/fetch/state-sources.ts +16 -3
  471. package/src/tools/fetch/tiger-full.ts +66 -23
  472. package/src/tools/gb-title-case.ts +1 -0
  473. package/src/tools/golden-expand.ts +64 -20
  474. package/src/tools/golden-promote.ts +53 -13
  475. package/src/tools/ingest-csv.ts +41 -26
  476. package/src/tools/jsonl-to-parquet.ts +30 -10
  477. package/src/tools/lint-shard-vocab.ts +75 -24
  478. package/src/tools/lint-shard.ts +75 -31
  479. package/src/tools/overlay-manifest.ts +7 -4
  480. package/src/tools/shard-kryptonite.ts +11 -3
  481. package/src/tools/shard-translit.ts +20 -5
  482. package/src/types.ts +51 -17
  483. package/src/wof-json.ts +18 -7
@@ -31,7 +31,9 @@ export class ParquetWriter extends BaseParquetWriter {
31
31
  const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);
32
32
  return new ParquetWriter(schema, envelopeWriter, opts);
33
33
  }
34
- /** Convenience method to create a new buffered parquet writer that writes to the specified file. */
34
+ /**
35
+ * Convenience method to create a new buffered parquet writer that writes to the specified file.
36
+ */
35
37
  static async openFile(schemaLike, sourcePath, opts) {
36
38
  if (typeof sourcePath === "string") {
37
39
  await fs.mkdir(path.dirname(sourcePath), { recursive: true });
@@ -40,15 +42,21 @@ export class ParquetWriter extends BaseParquetWriter {
40
42
  return ParquetWriter.openStream(schemaLike, outputStream, opts);
41
43
  }
42
44
  // @note This fixes invalid Markdown in the base class JSDoc.
43
- /** Set a metadata key-value pair on the writer. */
45
+ /**
46
+ * Set a metadata key-value pair on the writer.
47
+ */
44
48
  setMetadata(key, value) {
45
49
  return super.setMetadata(key, value);
46
50
  }
47
- /** Append a row to the buffer. If the buffer is full, the data will be written to disk. */
51
+ /**
52
+ * Append a row to the buffer. If the buffer is full, the data will be written to disk.
53
+ */
48
54
  async appendRow(row) {
49
55
  return super.appendRow(row);
50
56
  }
51
- /** Flush all buffered data to disk, close the file, and release resources. */
57
+ /**
58
+ * Flush all buffered data to disk, close the file, and release resources.
59
+ */
52
60
  async close() {
53
61
  await this.#flushing;
54
62
  if (this.closed)
@@ -1 +1 @@
1
- {"version":3,"file":"writer.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACtC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAA;AAEjC,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEpE,OAAO,EAAE,MAAM,EAA2B,MAAM,kCAAkC,CAAA;AAClF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAE1E,OAAO,EAEN,aAAa,EAEb,4BAA4B,GAC5B,MAAM,aAAa,CAAA;AAEpB;;GAEG;AACH,MAAM,OAAO,aAA2C,SAAQ,iBAAiB;IAEtE,MAAM,CAAU,qBAAqB,GAAG,IAAI,4BAA4B,EAAE,CAAA;IACpF,SAAS,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAA;IAE5C,MAAM,CAAU,KAAK,CAAC,UAAU,CAC/B,UAAyD,EACzD,YAAgC,EAChC,OAAsB,EAAE;QAExB,MAAM,MAAM,GACX,UAAU,YAAY,aAAa;YAClC,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,aAAa,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAA;QAEtE,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,CAAA;QAEzF,OAAO,IAAI,aAAa,CAAI,MAAM,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;IAC1D,CAAC;IAED,oGAAoG;IACpG,MAAM,CAAU,KAAK,CAAC,QAAQ,CAC7B,UAAyD,EACzD,UAAiC,EACjC,IAAoB;QAEpB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACpC,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9D,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;QAEnD,OAAO,aAAa,CAAC,UAAU,CAAI,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC,CAAA;IACnE,CAAC;IAED,6DAA6D;IAC7D,mDAAmD;IACnC,WAAW,CAAC,GAAW,EAAE,KAAa;QACrD,OAAO,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IACrC,CAAC;IAED,2FAA2F;IAC3E,KAAK,CAAC,SAAS,CAAC,GAAM;QACrC,OAAO,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;IAC5B,CAAC;IAED,8EAA8E;IAC9D,KAAK,CAAC,KAAK;QAC1B,MAAM,IAAI,CAAC,SAAS,CAAA;QAEpB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAM;QAEvB,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,aAAa,EAAQ,CAAA;QAElE,KAAK,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAEnC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAA;QAExB,OAAO,IAAI,CAAC,SAAS,CAAA;IACtB,CAAC;IAEM,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,OAAO,IAAI,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;IAEM,KAAK,CAAC,OAAO;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAA;IACnC,CAAC"}
1
+ {"version":3,"file":"writer.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACtC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAA;AAEjC,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEpE,OAAO,EAAE,MAAM,EAA2B,MAAM,kCAAkC,CAAA;AAClF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAE1E,OAAO,EAEN,aAAa,EAEb,4BAA4B,GAC5B,MAAM,aAAa,CAAA;AAEpB;;GAEG;AACH,MAAM,OAAO,aAA2C,SAAQ,iBAAiB;IAEtE,MAAM,CAAU,qBAAqB,GAAG,IAAI,4BAA4B,EAAE,CAAA;IACpF,SAAS,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAA;IAE5C,MAAM,CAAU,KAAK,CAAC,UAAU,CAC/B,UAAyD,EACzD,YAAgC,EAChC,OAAsB,EAAE;QAExB,MAAM,MAAM,GACX,UAAU,YAAY,aAAa;YAClC,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,aAAa,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAA;QAEtE,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,CAAA;QAEzF,OAAO,IAAI,aAAa,CAAI,MAAM,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,MAAM,CAAU,KAAK,CAAC,QAAQ,CAC7B,UAAyD,EACzD,UAAiC,EACjC,IAAoB;QAEpB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACpC,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9D,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;QAEnD,OAAO,aAAa,CAAC,UAAU,CAAI,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC,CAAA;IACnE,CAAC;IAED,6DAA6D;IAC7D;;OAEG;IACa,WAAW,CAAC,GAAW,EAAE,KAAa;QACrD,OAAO,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IACrC,CAAC;IAED;;OAEG;IACa,KAAK,CAAC,SAAS,CAAC,GAAM;QACrC,OAAO,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;IAC5B,CAAC;IAED;;OAEG;IACa,KAAK,CAAC,KAAK;QAC1B,MAAM,IAAI,CAAC,SAAS,CAAA;QAEpB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAM;QAEvB,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,aAAa,EAAQ,CAAA;QAElE,KAAK,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAEnC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAA;QAExB,OAAO,IAAI,CAAC,SAAS,CAAA;IACtB,CAAC;IAEM,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,OAAO,IAAI,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;IAEM,KAAK,CAAC,OAAO;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAA;IACnC,CAAC"}
@@ -39,9 +39,13 @@
39
39
  import { type ParquetSchemaDefinition } from "./parquet-wrapper/index.ts";
40
40
  import type { SplitName } from "./split.ts";
41
41
  import type { LabeledRow } from "./types.ts";
42
- /** Row groups flush at this many rows (parquetjs internal cadence within a shard). */
42
+ /**
43
+ * Row groups flush at this many rows (parquetjs internal cadence within a shard).
44
+ */
43
45
  export declare const ROW_GROUP_SIZE = 50000;
44
- /** Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0. */
46
+ /**
47
+ * Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0.
48
+ */
45
49
  export declare const SHARD_COMPRESSION: "SNAPPY";
46
50
  /**
47
51
  * A single Parquet-style row shape. The `[key: string]: unknown` index signature is required for compatibility with
@@ -64,14 +68,18 @@ export interface ParquetRow {
64
68
  synth_base_id: string | null;
65
69
  [key: string]: unknown;
66
70
  }
67
- /** Column names emitted into every shard. Matches `ParquetRow`. */
71
+ /**
72
+ * Column names emitted into every shard. Matches `ParquetRow`.
73
+ */
68
74
  export declare const PARQUET_COLUMNS: readonly ["raw", "tokens", "labels", "span_starts", "span_ends", "span_tags", "country", "locale", "source", "source_id", "corpus_version", "license", "synth_method", "synth_base_id"];
69
75
  /**
70
76
  * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
71
77
  * tokens/labels arrays. Compression is per-column SNAPPY.
72
78
  */
73
79
  export declare const LABELED_ROW_SCHEMA: ParquetSchemaDefinition<ParquetRow>;
74
- /** Per-shard metadata captured in `MANIFEST.json`. */
80
+ /**
81
+ * Per-shard metadata captured in `MANIFEST.json`.
82
+ */
75
83
  export interface ShardDescriptor {
76
84
  split: SplitName;
77
85
  path: string;
@@ -93,11 +101,17 @@ export interface ShardManifest {
93
101
  total_rows: number;
94
102
  }
95
103
  export interface WriteShardsOptions {
96
- /** Root output directory; corpus version dir is created beneath. */
104
+ /**
105
+ * Root output directory; corpus version dir is created beneath.
106
+ */
97
107
  outputDir: string;
98
- /** Corpus version stamped onto rows + into the output directory name. */
108
+ /**
109
+ * Corpus version stamped onto rows + into the output directory name.
110
+ */
99
111
  corpusVersion: string;
100
- /** Max rows per `.parquet` shard. Default 1_000_000 per the Phase 1 plan. */
112
+ /**
113
+ * Max rows per `.parquet` shard. Default 1_000_000 per the Phase 1 plan.
114
+ */
101
115
  rowsPerShard?: number;
102
116
  }
103
117
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"parquet.d.ts","sourceRoot":"","sources":["../../src/parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAOH,OAAO,EAAiB,KAAK,uBAAuB,EAAE,MAAM,4BAA4B,CAAA;AACxF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAE5C,sFAAsF;AACtF,eAAO,MAAM,cAAc,QAAS,CAAA;AAEpC,mFAAmF;AACnF,eAAO,MAAM,iBAAiB,EAAG,QAAiB,CAAA;AAElD;;;GAGG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,WAAW,EAAE,SAAS,MAAM,EAAE,CAAA;IAC9B,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,CAAA;IACtB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAED,mEAAmE;AACnE,eAAO,MAAM,eAAe,yLAelB,CAAA;AAEV;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,uBAAuB,CAAC,UAAU,CAkBlE,CAAA;AAED,sDAAsD;AACtD,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,SAAS,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,SAAS,CAAA;IACjB,WAAW,EAAE,OAAO,iBAAiB,CAAA;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,eAAe,EAAE,MAAM,CAAA;IACvB,cAAc,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,aAAa;IAC7B,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,cAAc,EAAE,MAAM,CAAA;IACtB,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,eAAe,EAAE,CAAA;IACzB,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,UAAU,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,kBAAkB;IAClC,oEAAoE;IACpE,SAAS,EAAE,MAAM,CAAA;IAEjB,yEAAyE;IACzE,aAAa,EAAE,MAAM,CAAA;IAErB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;AAEhF;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,UAAU,CAqCxD;AAqCD;;;;;;;;GAQG;AACH,wBAAsB,WAAW,CAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,aAAa,CAAC,CAgG1G"}
1
+ {"version":3,"file":"parquet.d.ts","sourceRoot":"","sources":["../../src/parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAOH,OAAO,EAAiB,KAAK,uBAAuB,EAAE,MAAM,4BAA4B,CAAA;AACxF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAE5C;;GAEG;AACH,eAAO,MAAM,cAAc,QAAS,CAAA;AAEpC;;GAEG;AACH,eAAO,MAAM,iBAAiB,EAAG,QAAiB,CAAA;AAElD;;;GAGG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,WAAW,EAAE,SAAS,MAAM,EAAE,CAAA;IAC9B,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,CAAA;IACtB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAED;;GAEG;AACH,eAAO,MAAM,eAAe,yLAelB,CAAA;AAMV;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,uBAAuB,CAAC,UAAU,CAkBlE,CAAA;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,SAAS,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,SAAS,CAAA;IACjB,WAAW,EAAE,OAAO,iBAAiB,CAAA;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,eAAe,EAAE,MAAM,CAAA;IACvB,cAAc,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,aAAa;IAC7B,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,cAAc,EAAE,MAAM,CAAA;IACtB,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,eAAe,EAAE,CAAA;IACzB,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,UAAU,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;AAEhF;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,UAAU,CAqCxD;AAqCD;;;;;;;;GAQG;AACH,wBAAsB,WAAW,CAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,aAAa,CAAC,CA2G1G"}
@@ -41,11 +41,17 @@ import { createReadStream } from "node:fs";
41
41
  import { mkdir, stat, writeFile } from "node:fs/promises";
42
42
  import { join } from "node:path";
43
43
  import { ParquetWriter } from "./parquet-wrapper/index.js";
44
- /** Row groups flush at this many rows (parquetjs internal cadence within a shard). */
44
+ /**
45
+ * Row groups flush at this many rows (parquetjs internal cadence within a shard).
46
+ */
45
47
  export const ROW_GROUP_SIZE = 50_000;
46
- /** Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0. */
48
+ /**
49
+ * Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0.
50
+ */
47
51
  export const SHARD_COMPRESSION = "SNAPPY";
48
- /** Column names emitted into every shard. Matches `ParquetRow`. */
52
+ /**
53
+ * Column names emitted into every shard. Matches `ParquetRow`.
54
+ */
49
55
  export const PARQUET_COLUMNS = [
50
56
  "raw",
51
57
  "tokens",
@@ -62,6 +68,9 @@ export const PARQUET_COLUMNS = [
62
68
  "synth_method",
63
69
  "synth_base_id",
64
70
  ];
71
+ /* oxlint-disable unicorn/text-encoding-identifier-case -- `"UTF8"` below is a ParquetType enum member,
72
+ not a text-encoding identifier. Lowercasing it does not type-check against ParquetSchemaDefinition,
73
+ and the rule has no way to tell the two apart. */
65
74
  /**
66
75
  * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
67
76
  * tokens/labels arrays. Compression is per-column SNAPPY.
@@ -247,7 +256,9 @@ export async function writeShards(perSplit, opts) {
247
256
  await writeFile(join(corpusDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
248
257
  return manifest;
249
258
  }
250
- /** Single-pass SHA-256 over the file at `path`. Cheap relative to Parquet write throughput. */
259
+ /**
260
+ * Single-pass SHA-256 over the file at `path`. Cheap relative to Parquet write throughput.
261
+ */
251
262
  async function hashFile(path) {
252
263
  const hash = createHash("sha256");
253
264
  const stream = createReadStream(path);
@@ -1 +1 @@
1
- {"version":3,"file":"parquet.js","sourceRoot":"","sources":["../../src/parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,aAAa,EAAgC,MAAM,4BAA4B,CAAA;AAIxF,sFAAsF;AACtF,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAA;AAEpC,mFAAmF;AACnF,MAAM,CAAC,MAAM,iBAAiB,GAAG,QAAiB,CAAA;AAwBlD,mEAAmE;AACnE,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,eAAe;CACN,CAAA;AAEV;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAwC;IACtE,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACrD,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxE,wFAAwF;IACxF,+FAA+F;IAC/F,gFAAgF;IAChF,WAAW,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC9E,SAAS,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC5E,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC3E,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACzD,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;IACxE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxD,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC3D,cAAc,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAChE,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACzD,YAAY,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC9E,aAAa,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;CAC/E,CAAA;AA6CD;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,GAAe;IAC3C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,GAAG,CAAA;IAEjD,IAAI,WAAW,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QACrF,MAAM,IAAI,KAAK,CACd,oEAAoE;YACnE,eAAe,WAAW,KAAK,SAAS,cAAc,SAAS,KAAK,SAAS,cAAc,SAAS,KAAK,SAAS,GAAG;YACrH,WAAW,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;YACtD,sEAAsE;YACtE,gFAAgF,CACjF,CAAA;IACF,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;QACxF,MAAM,IAAI,KAAK,CACd,sDAAsD;YACrD,UAAU,WAAW,CAAC,MAAM,SAAS,SAAS,CAAC,MAAM,SAAS,SAAS,CAAC,MAAM,GAAG;YACjF,WAAW,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,GAAG,CACrD,CAAA;IACF,CAAC;IAED,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW;QACX,SAAS;QACT,SAAS;QACT,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,IAAI;QAC1B,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc,EAAE,GAAG,CAAC,cAAc;QAClC,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI;QACvC,aAAa,EAAE,GAAG,CAAC,KAAK,EAAE,cAAc,IAAI,IAAI;KAChD,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,GAAe;IACnC,MAAM,GAAG,GAA4B;QACpC,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc,EAAE,GAAG,CAAC,cAAc;QAClC,OAAO,EAAE,GAAG,CAAC,OAAO;KACpB,CAAA;IAED,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;QACzB,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAA;IACxB,CAAC;IAED,IAAI,GAAG,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;QAC/B,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,CAAA;IACpC,CAAC;IAED,IAAI,GAAG,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;QAChC,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC,aAAa,CAAA;IACtC,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,QAAsB,EAAE,IAAwB;IACjF,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,IAAI,CAAC,aAAa,EAAE,CAAC,CAAA;IACvE,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE3C,MAAM,MAAM,GAAsB,EAAE,CAAA;IACpC,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,CAAC,IAAI;YAAE,SAAQ;QAEnB,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,MAAM,GAAqC,IAAI,CAAA;QACnD,IAAI,IAAI,GAAG,EAAE,CAAA;QACb,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,aAAa,GAAG,EAAE,CAAA;QACtB,IAAI,YAAY,GAAG,EAAE,CAAA;QAErB,MAAM,SAAS,GAAG,KAAK,IAAmB,EAAE;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;YACvC,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YAC1C,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,QAAQ,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAA;YAC5E,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAa,kBAAkB,EAAE,IAAI,EAAE;gBAC3E,YAAY,EAAE,cAAc;aAC5B,CAAC,CAAA;YACF,MAAM,CAAC,WAAW,CAAC,0BAA0B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;YAClE,MAAM,CAAC,WAAW,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAA;YAC5C,MAAM,CAAC,WAAW,CAAC,uBAAuB,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAA;YAC/D,SAAS,GAAG,CAAC,CAAA;YACb,aAAa,GAAG,EAAE,CAAA;YAClB,YAAY,GAAG,EAAE,CAAA;QAClB,CAAC,CAAA;QAED,MAAM,UAAU,GAAG,KAAK,IAAmB,EAAE;YAC5C,IAAI,CAAC,MAAM;gBAAE,OAAM;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YAEpB,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBACnB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAA;gBACjC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAA;gBACnC,MAAM,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,IAAI;oBACJ,MAAM,EAAE,SAAS;oBACjB,WAAW,EAAE,iBAAiB;oBAC9B,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,QAAQ,CAAC,IAAI;oBACpB,MAAM;oBACN,eAAe,EAAE,aAAa;oBAC9B,cAAc,EAAE,YAAY;iBAC5B,CAAC,CAAA;YACH,CAAC;YACD,MAAM,GAAG,IAAI,CAAA;QACd,CAAC,CAAA;QAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,MAAM,SAAS,EAAE,CAAA;YAClB,CAAC;YACD,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;YAC5B,MAAM,MAAO,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAA0B,CAAC,CAAA;YAEjE,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;gBACrB,aAAa,GAAG,GAAG,CAAC,SAAS,CAAA;YAC9B,CAAC;YACD,YAAY,GAAG,GAAG,CAAC,SAAS,CAAA;YAC5B,SAAS,EAAE,CAAA;YACX,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;YACf,SAAS,EAAE,CAAA;YAEX,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;gBAC/B,MAAM,UAAU,EAAE,CAAA;gBAClB,UAAU,EAAE,CAAA;YACb,CAAC;QACF,CAAC;QAED,MAAM,UAAU,EAAE,CAAA;IACnB,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAE5G,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,YAAY;QAC5B,cAAc,EAAE,cAAc;QAC9B,MAAM;QACN,MAAM;QACN,UAAU,EAAE,SAAS;KACrB,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAEnG,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,+FAA+F;AAC/F,KAAK,UAAU,QAAQ,CAAC,IAAY;IACnC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAA;IACjC,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;IAErC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,KAAe,CAAC,CAAA;IAC7B,CAAC;IAED,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC1B,CAAC"}
1
+ {"version":3,"file":"parquet.js","sourceRoot":"","sources":["../../src/parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,aAAa,EAAgC,MAAM,4BAA4B,CAAA;AAIxF;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAA;AAEpC;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,QAAiB,CAAA;AAwBlD;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,eAAe;CACN,CAAA;AAEV;;oDAEoD;AAEpD;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAwC;IACtE,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACrD,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxE,wFAAwF;IACxF,+FAA+F;IAC/F,gFAAgF;IAChF,WAAW,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC9E,SAAS,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC5E,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC3E,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACzD,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;IACxE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxD,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC3D,cAAc,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAChE,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACzD,YAAY,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC9E,aAAa,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;CAC/E,CAAA;AAqDD;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,GAAe;IAC3C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,GAAG,CAAA;IAEjD,IAAI,WAAW,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QACrF,MAAM,IAAI,KAAK,CACd,oEAAoE;YACnE,eAAe,WAAW,KAAK,SAAS,cAAc,SAAS,KAAK,SAAS,cAAc,SAAS,KAAK,SAAS,GAAG;YACrH,WAAW,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;YACtD,sEAAsE;YACtE,gFAAgF,CACjF,CAAA;IACF,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;QACxF,MAAM,IAAI,KAAK,CACd,sDAAsD;YACrD,UAAU,WAAW,CAAC,MAAM,SAAS,SAAS,CAAC,MAAM,SAAS,SAAS,CAAC,MAAM,GAAG;YACjF,WAAW,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,GAAG,CACrD,CAAA;IACF,CAAC;IAED,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW;QACX,SAAS;QACT,SAAS;QACT,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,IAAI;QAC1B,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc,EAAE,GAAG,CAAC,cAAc;QAClC,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI;QACvC,aAAa,EAAE,GAAG,CAAC,KAAK,EAAE,cAAc,IAAI,IAAI;KAChD,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,GAAe;IACnC,MAAM,GAAG,GAA4B;QACpC,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc,EAAE,GAAG,CAAC,cAAc;QAClC,OAAO,EAAE,GAAG,CAAC,OAAO;KACpB,CAAA;IAED,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;QACzB,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAA;IACxB,CAAC;IAED,IAAI,GAAG,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;QAC/B,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,CAAA;IACpC,CAAC;IAED,IAAI,GAAG,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;QAChC,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC,aAAa,CAAA;IACtC,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,QAAsB,EAAE,IAAwB;IACjF,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,IAAI,CAAC,aAAa,EAAE,CAAC,CAAA;IACvE,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE3C,MAAM,MAAM,GAAsB,EAAE,CAAA;IACpC,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,CAAC,IAAI;YAAE,SAAQ;QAEnB,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,MAAM,GAAqC,IAAI,CAAA;QACnD,IAAI,IAAI,GAAG,EAAE,CAAA;QACb,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,aAAa,GAAG,EAAE,CAAA;QACtB,IAAI,YAAY,GAAG,EAAE,CAAA;QAErB,MAAM,SAAS,GAAG,KAAK,IAAmB,EAAE;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;YACvC,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YAC1C,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,QAAQ,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAA;YAE5E,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAa,kBAAkB,EAAE,IAAI,EAAE;gBAC3E,YAAY,EAAE,cAAc;aAC5B,CAAC,CAAA;YAEF,MAAM,CAAC,WAAW,CAAC,0BAA0B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;YAClE,MAAM,CAAC,WAAW,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAA;YAC5C,MAAM,CAAC,WAAW,CAAC,uBAAuB,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAA;YAC/D,SAAS,GAAG,CAAC,CAAA;YACb,aAAa,GAAG,EAAE,CAAA;YAClB,YAAY,GAAG,EAAE,CAAA;QAClB,CAAC,CAAA;QAED,MAAM,UAAU,GAAG,KAAK,IAAmB,EAAE;YAC5C,IAAI,CAAC,MAAM;gBAAE,OAAM;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YAEpB,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBACnB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAA;gBACjC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAA;gBAEnC,MAAM,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,IAAI;oBACJ,MAAM,EAAE,SAAS;oBACjB,WAAW,EAAE,iBAAiB;oBAC9B,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,QAAQ,CAAC,IAAI;oBACpB,MAAM;oBACN,eAAe,EAAE,aAAa;oBAC9B,cAAc,EAAE,YAAY;iBAC5B,CAAC,CAAA;YACH,CAAC;YAED,MAAM,GAAG,IAAI,CAAA;QACd,CAAC,CAAA;QAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,MAAM,SAAS,EAAE,CAAA;YAClB,CAAC;YAED,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;YAC5B,MAAM,MAAO,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAA0B,CAAC,CAAA;YAEjE,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;gBACrB,aAAa,GAAG,GAAG,CAAC,SAAS,CAAA;YAC9B,CAAC;YAED,YAAY,GAAG,GAAG,CAAC,SAAS,CAAA;YAE5B,SAAS,EAAE,CAAA;YAEX,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;YAEf,SAAS,EAAE,CAAA;YAEX,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;gBAC/B,MAAM,UAAU,EAAE,CAAA;gBAElB,UAAU,EAAE,CAAA;YACb,CAAC;QACF,CAAC;QAED,MAAM,UAAU,EAAE,CAAA;IACnB,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAE5G,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,YAAY;QAC5B,cAAc,EAAE,cAAc;QAC9B,MAAM;QACN,MAAM;QACN,UAAU,EAAE,SAAS;KACrB,CAAA;IAED,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAEnG,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,QAAQ,CAAC,IAAY;IACnC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAA;IACjC,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;IAErC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,KAAe,CAAC,CAAA;IAC7B,CAAC;IAED,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC1B,CAAC"}
@@ -28,28 +28,50 @@
28
28
  */
29
29
  import { type AdapterRegistry } from "./adapter.ts";
30
30
  import type { AdapterOptions, CorpusAdapter } from "./types.ts";
31
- /** Snapshot of the runner's state, emitted on every progress tick. */
31
+ /**
32
+ * Snapshot of the runner's state, emitted on every progress tick.
33
+ */
32
34
  export interface RunnerProgress {
33
- /** Adapter being driven. */
35
+ /**
36
+ * Adapter being driven.
37
+ */
34
38
  adapterID: string;
35
- /** Total rows the adapter has yielded (before dedup). */
39
+ /**
40
+ * Total rows the adapter has yielded (before dedup).
41
+ */
36
42
  yielded: number;
37
- /** Rows actually written to JSONL (after dedup). */
43
+ /**
44
+ * Rows actually written to JSONL (after dedup).
45
+ */
38
46
  written: number;
39
- /** Bytes written to JSONL so far. */
47
+ /**
48
+ * Bytes written to JSONL so far.
49
+ */
40
50
  bytes: number;
41
- /** Wall-clock milliseconds since the run started. */
51
+ /**
52
+ * Wall-clock milliseconds since the run started.
53
+ */
42
54
  elapsed_ms: number;
43
55
  }
44
- /** Per-invocation options for `runAdapter`. */
56
+ /**
57
+ * Per-invocation options for `runAdapter`.
58
+ */
45
59
  export interface RunAdapterOptions {
46
- /** Adapter to drive. */
60
+ /**
61
+ * Adapter to drive.
62
+ */
47
63
  adapter: CorpusAdapter;
48
- /** Options handed to the adapter (input path, country filter, limit, signal). */
64
+ /**
65
+ * Options handed to the adapter (input path, country filter, limit, signal).
66
+ */
49
67
  adapterOptions: AdapterOptions;
50
- /** Root output directory; the runner creates `<outputDir>/<adapter.id>/` under it. */
68
+ /**
69
+ * Root output directory; the runner creates `<outputDir>/<adapter.id>/` under it.
70
+ */
51
71
  outputDir: string;
52
- /** Corpus version stamped onto every row. Locked together with the tokenizer version. */
72
+ /**
73
+ * Corpus version stamped onto every row. Locked together with the tokenizer version.
74
+ */
53
75
  corpusVersion: string;
54
76
  /**
55
77
  * Optional progress callback. Invoked every `progressEvery` rows yielded (default 1000) and once at the end of the
@@ -62,7 +84,9 @@ export interface RunAdapterOptions {
62
84
  */
63
85
  progressEvery?: number;
64
86
  }
65
- /** Return value of `runAdapter`: the same shape as `MANIFEST.json` on disk. */
87
+ /**
88
+ * Return value of `runAdapter`: the same shape as `MANIFEST.json` on disk.
89
+ */
66
90
  export interface AdapterRunManifest {
67
91
  adapter_id: string;
68
92
  corpus_version: string;
@@ -95,6 +119,8 @@ export declare function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunM
95
119
  export declare function runAllAdapters(registry: AdapterRegistry, common: Omit<RunAdapterOptions, "adapter"> & {
96
120
  adapterOptionsFor?: (a: CorpusAdapter) => AdapterOptions;
97
121
  }): Promise<AdapterRunManifest[]>;
98
- /** Convenience: ensure the parent directory of `filePath` exists. */
122
+ /**
123
+ * Convenience: ensure the parent directory of `filePath` exists.
124
+ */
99
125
  export declare function ensureParentDir(filePath: string): Promise<void>;
100
126
  //# sourceMappingURL=runner.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAMH,OAAO,EAAsC,KAAK,eAAe,EAAwB,MAAM,cAAc,CAAA;AAC7G,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAE,MAAM,YAAY,CAAA;AAE7E,sEAAsE;AACtE,MAAM,WAAW,cAAc;IAC9B,4BAA4B;IAC5B,SAAS,EAAE,MAAM,CAAA;IAEjB,yDAAyD;IACzD,OAAO,EAAE,MAAM,CAAA;IAEf,oDAAoD;IACpD,OAAO,EAAE,MAAM,CAAA;IAEf,qCAAqC;IACrC,KAAK,EAAE,MAAM,CAAA;IAEb,qDAAqD;IACrD,UAAU,EAAE,MAAM,CAAA;CAClB;AAED,+CAA+C;AAC/C,MAAM,WAAW,iBAAiB;IACjC,wBAAwB;IACxB,OAAO,EAAE,aAAa,CAAA;IAEtB,iFAAiF;IACjF,cAAc,EAAE,cAAc,CAAA;IAE9B,sFAAsF;IACtF,SAAS,EAAE,MAAM,CAAA;IAEjB,yFAAyF;IACzF,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,cAAc,KAAK,IAAI,CAAA;IAE/C;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,+EAA+E;AAC/E,MAAM,WAAW,kBAAkB;IAClC,UAAU,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,MAAM,CAAA;IACtB,eAAe,EAAE,MAAM,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAwGrF;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CACnC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,IAAI,CAAC,iBAAiB,EAAE,SAAS,CAAC,GAAG;IAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,EAAE,aAAa,KAAK,cAAc,CAAA;CAAE,GACvG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAe/B;AA4CD,qEAAqE;AACrE,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAErE"}
1
+ {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAMH,OAAO,EAAsC,KAAK,eAAe,EAAwB,MAAM,cAAc,CAAA;AAC7G,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAE,MAAM,YAAY,CAAA;AAE7E;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IAEb;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,OAAO,EAAE,aAAa,CAAA;IAEtB;;OAEG;IACH,cAAc,EAAE,cAAc,CAAA;IAE9B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,cAAc,KAAK,IAAI,CAAA;IAE/C;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,UAAU,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,MAAM,CAAA;IACtB,eAAe,EAAE,MAAM,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA2GrF;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CACnC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,IAAI,CAAC,iBAAiB,EAAE,SAAS,CAAC,GAAG;IAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,EAAE,aAAa,KAAK,cAAc,CAAA;CAAE,GACvG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAgB/B;AAgDD;;GAEG;AACH,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAErE"}
package/out/src/runner.js CHANGED
@@ -39,7 +39,7 @@ import { canonicalDedupKey, streamingSha256 } from "./adapter.js";
39
39
  */
40
40
  export async function runAdapter(opts) {
41
41
  const { adapter, adapterOptions, outputDir, corpusVersion } = opts;
42
- const progressEvery = opts.progressEvery ?? 1_000;
42
+ const progressEvery = opts.progressEvery ?? 1000;
43
43
  const adapterDir = join(outputDir, adapter.id);
44
44
  await mkdir(adapterDir, { recursive: true });
45
45
  const jsonlPath = join(adapterDir, "canonical.jsonl");
@@ -163,7 +163,9 @@ function assertEmittedRow(adapter, row) {
163
163
  throw new Error(`adapter ${adapter.id}: row.license is empty for source_id=${row.source_id}`);
164
164
  }
165
165
  }
166
- /** Promise-ify a single event emission. Used to await `drain` / `close` on the write stream. */
166
+ /**
167
+ * Promise-ify a single event emission. Used to await `drain` / `close` on the write stream.
168
+ */
167
169
  function once(emitter, event) {
168
170
  return new Promise((resolve, reject) => {
169
171
  const onEvent = () => {
@@ -178,7 +180,9 @@ function once(emitter, event) {
178
180
  emitter.once("error", onError);
179
181
  });
180
182
  }
181
- /** Convenience: ensure the parent directory of `filePath` exists. */
183
+ /**
184
+ * Convenience: ensure the parent directory of `filePath` exists.
185
+ */
182
186
  export async function ensureParentDir(filePath) {
183
187
  await mkdir(dirname(filePath), { recursive: true });
184
188
  }
@@ -1 +1 @@
1
- {"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,iBAAiB,EAAoB,MAAM,SAAS,CAAA;AAC7D,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEzC,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAA8C,MAAM,cAAc,CAAA;AAiE7G;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAuB;IACvD,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,aAAa,EAAE,GAAG,IAAI,CAAA;IAClE,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,KAAK,CAAA;IAEjD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;IAC9C,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE5C,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAA;IACrD,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;IAEtD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAA;IAC5B,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;IAE5B,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACjE,MAAM,MAAM,GAAoB,eAAe,EAAE,CAAA;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,MAAM,cAAc,GAAG,UAAU,CAAA;IACjC,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,MAAM,YAAY,GAAG,GAAS,EAAE;QAC/B,IAAI,CAAC,UAAU,EAAE,CAAC;YACjB,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,OAAO;YACP,OAAO;YACP,KAAK;YACL,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;SAClC,CAAC,CAAA;IACH,CAAC,CAAA;IAED,IAAI,CAAC;QACJ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YACtD,IAAI,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBACpC,MAAM,IAAI,YAAY,CAAC,+BAA+B,EAAE,YAAY,CAAC,CAAA;YACtE,CAAC;YAED,OAAO,EAAE,CAAA;YACT,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;YAE9B,MAAM,OAAO,GAAiB,EAAE,GAAG,GAAG,EAAE,cAAc,EAAE,aAAa,EAAE,CAAA;YACvE,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;YAEtC,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACnB,IAAI,OAAO,GAAG,aAAa,KAAK,CAAC,EAAE,CAAC;wBACnC,YAAY,EAAE,CAAA;oBACf,CAAC;oBACD,SAAQ;gBACT,CAAC;gBAED,IAAI,IAAI,CAAC,IAAI,IAAI,cAAc,EAAE,CAAC;oBACjC,cAAc,GAAG,IAAI,CAAA;oBACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,+BAA+B,cAAc,CAAC,cAAc,EAAE,wCAAwC,CACtG,CAAA;gBACF,CAAC;qBAAM,CAAC;oBACP,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBACd,CAAC;YACF,CAAC;YAED,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAA;YAC3C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACnB,KAAK,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YACxC,OAAO,EAAE,CAAA;YAET,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;YAC5B,CAAC;YAED,IAAI,OAAO,GAAG,aAAa,KAAK,CAAC,EAAE,CAAC;gBACnC,YAAY,EAAE,CAAA;YACf,CAAC;QACF,CAAC;IACF,CAAC;YAAS,CAAC;QACV,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC5B,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAA;IACzC,YAAY,EAAE,CAAA;IAEd,MAAM,QAAQ,GAAuB;QACpC,UAAU,EAAE,OAAO,CAAC,EAAE;QACtB,cAAc,EAAE,aAAa;QAC7B,eAAe,EAAE,OAAO,CAAC,cAAc;QACvC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,OAAO;QACP,OAAO;QACP,OAAO,EAAE,OAAO,GAAG,OAAO;QAC1B,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QACvB,UAAU,EAAE,SAAS;QACrB,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE;QACnC,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE;QAC/B,UAAU;KACV,CAAA;IAED,MAAM,SAAS,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAE/E,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,QAAyB,EACzB,MAAyG;IAEzG,MAAM,GAAG,GAAyB,EAAE,CAAA;IAEpC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACvC,MAAM,cAAc,GAAG,MAAM,CAAC,iBAAiB,EAAE,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,cAAc,CAAA;QACnF,GAAG,CAAC,IAAI,CACP,MAAM,UAAU,CAAC;YAChB,GAAG,MAAM;YACT,OAAO;YACP,cAAc;SACd,CAAC,CACF,CAAA;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,OAAsB,EAAE,GAAiB;IAClE,IAAI,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,2CAA2C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAC/G,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,0BAA0B,CAAC,CAAA;IACjE,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,oCAAoC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC1F,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,wCAAwC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC9F,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,wCAAwC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC9F,CAAC;AACF,CAAC;AAED,gGAAgG;AAChG,SAAS,IAAI,CAAC,OAAoB,EAAE,KAAwB;IAC3D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,MAAM,OAAO,GAAG,GAAS,EAAE;YAC1B,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAC7B,OAAO,EAAE,CAAA;QACV,CAAC,CAAA;QACD,MAAM,OAAO,GAAG,CAAC,GAAU,EAAQ,EAAE;YACpC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;YAC3B,MAAM,CAAC,GAAG,CAAC,CAAA;QACZ,CAAC,CAAA;QACD,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QAC5B,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;IAC/B,CAAC,CAAC,CAAA;AACH,CAAC;AAED,qEAAqE;AACrE,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB;IACrD,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;AACpD,CAAC"}
1
+ {"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,iBAAiB,EAAoB,MAAM,SAAS,CAAA;AAC7D,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEzC,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAA8C,MAAM,cAAc,CAAA;AAyF7G;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAuB;IACvD,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,aAAa,EAAE,GAAG,IAAI,CAAA;IAClE,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAA;IAEhD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;IAC9C,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE5C,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAA;IACrD,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;IAEtD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAA;IAC5B,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;IAE5B,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACjE,MAAM,MAAM,GAAoB,eAAe,EAAE,CAAA;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,MAAM,cAAc,GAAG,UAAU,CAAA;IACjC,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,MAAM,YAAY,GAAG,GAAS,EAAE;QAC/B,IAAI,CAAC,UAAU,EAAE,CAAC;YACjB,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,OAAO;YACP,OAAO;YACP,KAAK;YACL,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;SAClC,CAAC,CAAA;IACH,CAAC,CAAA;IAED,IAAI,CAAC;QACJ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YACtD,IAAI,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBACpC,MAAM,IAAI,YAAY,CAAC,+BAA+B,EAAE,YAAY,CAAC,CAAA;YACtE,CAAC;YAED,OAAO,EAAE,CAAA;YACT,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;YAE9B,MAAM,OAAO,GAAiB,EAAE,GAAG,GAAG,EAAE,cAAc,EAAE,aAAa,EAAE,CAAA;YACvE,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;YAEtC,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACnB,IAAI,OAAO,GAAG,aAAa,KAAK,CAAC,EAAE,CAAC;wBACnC,YAAY,EAAE,CAAA;oBACf,CAAC;oBAED,SAAQ;gBACT,CAAC;gBAED,IAAI,IAAI,CAAC,IAAI,IAAI,cAAc,EAAE,CAAC;oBACjC,cAAc,GAAG,IAAI,CAAA;oBAErB,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,+BAA+B,cAAc,CAAC,cAAc,EAAE,wCAAwC,CACtG,CAAA;gBACF,CAAC;qBAAM,CAAC;oBACP,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBACd,CAAC;YACF,CAAC;YAED,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAA;YAC3C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACnB,KAAK,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YAExC,OAAO,EAAE,CAAA;YAET,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;YAC5B,CAAC;YAED,IAAI,OAAO,GAAG,aAAa,KAAK,CAAC,EAAE,CAAC;gBACnC,YAAY,EAAE,CAAA;YACf,CAAC;QACF,CAAC;IACF,CAAC;YAAS,CAAC;QACV,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC5B,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAA;IACzC,YAAY,EAAE,CAAA;IAEd,MAAM,QAAQ,GAAuB;QACpC,UAAU,EAAE,OAAO,CAAC,EAAE;QACtB,cAAc,EAAE,aAAa;QAC7B,eAAe,EAAE,OAAO,CAAC,cAAc;QACvC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,OAAO;QACP,OAAO;QACP,OAAO,EAAE,OAAO,GAAG,OAAO;QAC1B,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QACvB,UAAU,EAAE,SAAS;QACrB,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE;QACnC,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE;QAC/B,UAAU;KACV,CAAA;IAED,MAAM,SAAS,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAE/E,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,QAAyB,EACzB,MAAyG;IAEzG,MAAM,GAAG,GAAyB,EAAE,CAAA;IAEpC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACvC,MAAM,cAAc,GAAG,MAAM,CAAC,iBAAiB,EAAE,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,cAAc,CAAA;QAEnF,GAAG,CAAC,IAAI,CACP,MAAM,UAAU,CAAC;YAChB,GAAG,MAAM;YACT,OAAO;YACP,cAAc;SACd,CAAC,CACF,CAAA;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,OAAsB,EAAE,GAAiB;IAClE,IAAI,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,2CAA2C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAC/G,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,0BAA0B,CAAC,CAAA;IACjE,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,oCAAoC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC1F,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,wCAAwC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC9F,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,wCAAwC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC9F,CAAC;AACF,CAAC;AAED;;GAEG;AACH,SAAS,IAAI,CAAC,OAAoB,EAAE,KAAwB;IAC3D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,MAAM,OAAO,GAAG,GAAS,EAAE;YAC1B,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAC7B,OAAO,EAAE,CAAA;QACV,CAAC,CAAA;QAED,MAAM,OAAO,GAAG,CAAC,GAAU,EAAQ,EAAE;YACpC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;YAC3B,MAAM,CAAC,GAAG,CAAC,CAAA;QACZ,CAAC,CAAA;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QAC5B,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;IAC/B,CAAC,CAAC,CAAA;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB;IACrD,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;AACpD,CAAC"}
@@ -12,5 +12,9 @@
12
12
  * scripts/build-anchor-absorption-shard.mjs.
13
13
  */
14
14
  import { type ShardRecipe } from "./scaffold.ts";
15
+ /**
16
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
17
+ * and `description` below for the surface form it generates.
18
+ */
15
19
  export declare const anchorAbsorptionRecipe: ShardRecipe;
16
20
  //# sourceMappingURL=anchor-absorption.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"anchor-absorption.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/anchor-absorption.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AASH,OAAO,EAAW,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAoBzD,eAAO,MAAM,sBAAsB,EAAE,WA4CpC,CAAA"}
1
+ {"version":3,"file":"anchor-absorption.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/anchor-absorption.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AASH,OAAO,EAAW,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAwBzD;;;GAGG;AACH,eAAO,MAAM,sBAAsB,EAAE,WAoDpC,CAAA"}
@@ -17,10 +17,14 @@ import { stableSourceID } from "../adapter.js";
17
17
  import { alignRow } from "../align.js";
18
18
  import { synthesizeAnchorAbsorptionRow } from "../synthesize-anchor-absorption.js";
19
19
  import { makeLcg } from "./scaffold.js";
20
- // The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through
21
- // the data-root helper (the lab default is `$MAILWOMAN_DATA_ROOT`), never re-hardcoded.
20
+ /**
21
+ * The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through the data-root
22
+ * helper (the lab default is `$MAILWOMAN_DATA_ROOT`), never re-hardcoded.
23
+ */
22
24
  const ANCHOR_LOOKUP = dataRootPath("anchor", "pilot-anchor-lookup.json");
23
- /** The real US ZIPs in the anchor lookup (entries whose value is a `[{ US: … }]` candidate list). */
25
+ /**
26
+ * The real US ZIPs in the anchor lookup (entries whose value is a `[{ US: … }]` candidate list).
27
+ */
24
28
  function loadRealUsZips(path) {
25
29
  const d = JSON.parse(readFileSync(path, "utf8"));
26
30
  const zips = [];
@@ -31,6 +35,10 @@ function loadRealUsZips(path) {
31
35
  }
32
36
  return zips;
33
37
  }
38
+ /**
39
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
40
+ * and `description` below for the surface form it generates.
41
+ */
34
42
  export const anchorAbsorptionRecipe = {
35
43
  name: "anchor-absorption",
36
44
  description: "Anchor-absorption counter-augmentation (#220/#723): 6-slice mix → synthesizeAnchorAbsorptionRow",
@@ -39,7 +47,7 @@ export const anchorAbsorptionRecipe = {
39
47
  // Emit PRNG: the legacy build-anchor-absorption-shard.mjs seeded an LCG (lcg(opts.seed)).
40
48
  const random = makeLcg(opts.seed);
41
49
  const source = opts.sourceName ?? "synth-anchor-absorption";
42
- const count = opts.count ?? 50000;
50
+ const count = opts.count ?? 50_000;
43
51
  const realZips = loadRealUsZips(ANCHOR_LOOKUP);
44
52
  console.error(`Loaded ${realZips.length} real US ZIPs from the anchor lookup (the leading-5-digit source).`);
45
53
  let written = 0;
@@ -47,7 +55,8 @@ export const anchorAbsorptionRecipe = {
47
55
  const byTemplate = {};
48
56
  for (let i = 0; i < count; i++) {
49
57
  const synth = synthesizeAnchorAbsorptionRow({ random, realZips });
50
- const country = synth.locale.split("-")[1]; // "en-US" -> "US", "de-DE" -> "DE"
58
+ const country = synth.locale.split("-")[1];
59
+ // "en-US" -> "US", "de-DE" -> "DE"
51
60
  const canonical = {
52
61
  raw: synth.raw,
53
62
  components: synth.components,
@@ -1 +1 @@
1
- {"version":3,"file":"anchor-absorption.js","sourceRoot":"","sources":["../../../src/shard-recipes/anchor-absorption.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,6BAA6B,EAAE,MAAM,oCAAoC,CAAA;AAClF,OAAO,EAAE,OAAO,EAAoB,MAAM,eAAe,CAAA;AAEzD,uGAAuG;AACvG,wFAAwF;AACxF,MAAM,aAAa,GAAG,YAAY,CAAC,QAAQ,EAAE,0BAA0B,CAAC,CAAA;AAExE,qGAAqG;AACrG,SAAS,cAAc,CAAC,IAAY;IACnC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAA4B,CAAA;IAC3E,MAAM,IAAI,GAAa,EAAE,CAAA;IAEzB,KAAK,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,IAAK,CAAC,CAAC,CAAC,CAAY,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YAC5G,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACd,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,MAAM,sBAAsB,GAAgB;IAClD,IAAI,EAAE,mBAAmB;IACzB,WAAW,EAAE,iGAAiG;IAC9G,IAAI,EAAE,UAAU;IAChB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,0FAA0F;QAC1F,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,yBAAyB,CAAA;QAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,MAAM,QAAQ,GAAG,cAAc,CAAC,aAAa,CAAC,CAAA;QAC9C,OAAO,CAAC,KAAK,CAAC,UAAU,QAAQ,CAAC,MAAM,oEAAoE,CAAC,CAAA;QAE5G,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,WAAW,GAAG,CAAC,CAAA;QACnB,MAAM,UAAU,GAA2B,EAAE,CAAA;QAE7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,6BAA6B,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;YACjE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA,CAAC,mCAAmC;YAC9E,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,OAAO;gBACP,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM;gBACN,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,EAAqD,CAAC;aAC5F,CAAA;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;YAErE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAChC,WAAW,EAAE,CAAA;gBACb,SAAQ;YACT,CAAC;YACD,KAAK,CACJ,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,mBAAmB,EAAE,cAAc,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,GAAG,IAAI,CAC5G,CAAA;YACD,OAAO,EAAE,CAAA;YACT,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACnE,CAAC;QACD,OAAO,CAAC,KAAK,CAAC,WAAW,OAAO,UAAU,WAAW,eAAe,CAAC,CAAA;QACrE,OAAO,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAA;QAExD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAA;IAClD,CAAC;CACD,CAAA"}
1
+ {"version":3,"file":"anchor-absorption.js","sourceRoot":"","sources":["../../../src/shard-recipes/anchor-absorption.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,6BAA6B,EAAE,MAAM,oCAAoC,CAAA;AAClF,OAAO,EAAE,OAAO,EAAoB,MAAM,eAAe,CAAA;AAEzD;;;GAGG;AACH,MAAM,aAAa,GAAG,YAAY,CAAC,QAAQ,EAAE,0BAA0B,CAAC,CAAA;AAExE;;GAEG;AACH,SAAS,cAAc,CAAC,IAAY;IACnC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAA4B,CAAA;IAC3E,MAAM,IAAI,GAAa,EAAE,CAAA;IAEzB,KAAK,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,IAAK,CAAC,CAAC,CAAC,CAAY,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YAC5G,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACd,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAgB;IAClD,IAAI,EAAE,mBAAmB;IACzB,WAAW,EAAE,iGAAiG;IAC9G,IAAI,EAAE,UAAU;IAChB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,0FAA0F;QAC1F,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,yBAAyB,CAAA;QAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,MAAM,CAAA;QAClC,MAAM,QAAQ,GAAG,cAAc,CAAC,aAAa,CAAC,CAAA;QAE9C,OAAO,CAAC,KAAK,CAAC,UAAU,QAAQ,CAAC,MAAM,oEAAoE,CAAC,CAAA;QAE5G,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,WAAW,GAAG,CAAC,CAAA;QACnB,MAAM,UAAU,GAA2B,EAAE,CAAA;QAE7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,6BAA6B,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;YACjE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;YAE1C,mCAAmC;YACnC,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,OAAO;gBACP,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM;gBACN,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,EAAqD,CAAC;aAC5F,CAAA;YAED,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;YAErE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAChC,WAAW,EAAE,CAAA;gBAEb,SAAQ;YACT,CAAC;YAED,KAAK,CACJ,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,mBAAmB,EAAE,cAAc,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,GAAG,IAAI,CAC5G,CAAA;YAED,OAAO,EAAE,CAAA;YACT,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACnE,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,WAAW,OAAO,UAAU,WAAW,eAAe,CAAC,CAAA;QACrE,OAAO,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAA;QAExD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAA;IAClD,CAAC;CACD,CAAA"}
@@ -12,5 +12,9 @@
12
12
  * `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
13
13
  */
14
14
  import { type ShardRecipe } from "./scaffold.ts";
15
+ /**
16
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
17
+ * and `description` below for the surface form it generates.
18
+ */
15
19
  export declare const boundaryStressRecipe: ShardRecipe;
16
20
  //# sourceMappingURL=boundary-stress.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"boundary-stress.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,OAAO,EAAkB,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AA8B/E,eAAO,MAAM,oBAAoB,EAAE,WAwClC,CAAA"}
1
+ {"version":3,"file":"boundary-stress.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,OAAO,EAAkB,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AAkC/E;;;GAGG;AACH,eAAO,MAAM,oBAAoB,EAAE,WA6ClC,CAAA"}
@@ -14,11 +14,13 @@
14
14
  import { alignRow } from "../align.js";
15
15
  import { synthesizeBoundaryStressRow } from "../synthesize-boundary-stress.js";
16
16
  import { makeMulberry32, shardSourceID } from "./scaffold.js";
17
- // Revised composition (v1.7.0, DeepSeek-tuned 2026-06-18): `bare-locality` ~11% (recover the 84% locality
18
- // drop on bare "City, STATE" rows WITHOUT becoming a locality-first majority), and
19
- // house-number-before:after = 7:3 (FR's dominant order is number-BEFORE; 30% after breaks the order-bias
20
- // shortcut without risking FR hn-before accuracy). The three original non-number shapes keep the bulk.
21
- // Weights sum to 1.0. Key order is load-bearing — it drives the cumulative thresholds below.
17
+ /**
18
+ * Revised composition (v1.7.0, DeepSeek-tuned 2026-06-18): `bare-locality` ~11% (recover the 84% locality drop on bare
19
+ * "City, STATE" rows WITHOUT becoming a locality-first majority), and house-number-before:after = 7:3 (FR's dominant
20
+ * order is number-BEFORE; 30% after breaks the order-bias shortcut without risking FR hn-before accuracy). The three
21
+ * original non-number shapes keep the bulk. Weights sum to 1.0. Key order is load-bearing — it drives the cumulative
22
+ * thresholds below.
23
+ */
22
24
  const WEIGHTS = {
23
25
  "street-eats-affix": 0.22,
24
26
  "comma-less-city-state": 0.22,
@@ -36,8 +38,12 @@ function pickTemplate(r) {
36
38
  for (const [t, c] of CUM)
37
39
  if (x <= c)
38
40
  return t;
39
- return CUM[CUM.length - 1][0];
41
+ return CUM.at(-1)[0];
40
42
  }
43
+ /**
44
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
45
+ * and `description` below for the surface form it generates.
46
+ */
41
47
  export const boundaryStressRecipe = {
42
48
  name: "boundary-stress",
43
49
  description: "Boundary-instability rows (#375): weighted template mix → synthesizeBoundaryStressRow → aligned BIO",
@@ -45,7 +51,7 @@ export const boundaryStressRecipe = {
45
51
  async run(opts, write) {
46
52
  // Emit PRNG: the legacy build-boundary-stress-shard.mjs seeded mulberry32(opts.seed).
47
53
  const random = makeMulberry32(opts.seed);
48
- const count = opts.count ?? 20000;
54
+ const count = opts.count ?? 20_000;
49
55
  let emitted = 0;
50
56
  let skipped = 0;
51
57
  for (let i = 0; i < count; i++) {
@@ -1 +1 @@
1
- {"version":3,"file":"boundary-stress.js","sourceRoot":"","sources":["../../../src/shard-recipes/boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAA+B,2BAA2B,EAAE,MAAM,kCAAkC,CAAA;AAC3G,OAAO,EAAE,cAAc,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE/E,0GAA0G;AAC1G,mFAAmF;AACnF,yGAAyG;AACzG,uGAAuG;AACvG,6FAA6F;AAC7F,MAAM,OAAO,GAA2C;IACvD,mBAAmB,EAAE,IAAI;IACzB,uBAAuB,EAAE,IAAI;IAC7B,WAAW,EAAE,IAAI;IACjB,eAAe,EAAE,IAAI;IACrB,4BAA4B,EAAE,KAAK;IACnC,2BAA2B,EAAE,KAAK;CAClC,CAAA;AACD,MAAM,GAAG,GAA4C,CAAC,GAAG,EAAE;IAC1D,IAAI,GAAG,GAAG,CAAC,CAAA;IAEX,OAAQ,MAAM,CAAC,OAAO,CAAC,OAAO,CAA6C,CAAC,GAAG,CAC9E,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAqC,CAC/D,CAAA;AACF,CAAC,CAAC,EAAE,CAAA;AACJ,SAAS,YAAY,CAAC,CAAe;IACpC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;IAEb,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG;QAAE,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,CAAA;IAE9C,OAAO,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AAC/B,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAgB;IAChD,IAAI,EAAE,iBAAiB;IACvB,WAAW,EAAE,qGAAqG;IAClH,IAAI,EAAE,UAAU;IAChB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,sFAAsF;QACtF,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,2BAA2B,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YACnG,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;YAChD,MAAM,SAAS,GAAG,aAAa,CAAC,uBAAuB,EAAE,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAC7F,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,OAAO;gBACP,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,MAAM,EAAE,uBAAuB;gBAC/B,SAAS;gBACT,cAAc,EAAE,OAAO;gBACvB,OAAO,EAAE,gFAAgF;aACzF,CAAA;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;YAErE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAChC,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,iGAAiG;YACjG,KAAK,CACJ,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,mBAAmB,GAAG,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAC/G,CAAA;YACD,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}
1
+ {"version":3,"file":"boundary-stress.js","sourceRoot":"","sources":["../../../src/shard-recipes/boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAA+B,2BAA2B,EAAE,MAAM,kCAAkC,CAAA;AAC3G,OAAO,EAAE,cAAc,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE/E;;;;;;GAMG;AACH,MAAM,OAAO,GAA2C;IACvD,mBAAmB,EAAE,IAAI;IACzB,uBAAuB,EAAE,IAAI;IAC7B,WAAW,EAAE,IAAI;IACjB,eAAe,EAAE,IAAI;IACrB,4BAA4B,EAAE,KAAK;IACnC,2BAA2B,EAAE,KAAK;CAClC,CAAA;AAED,MAAM,GAAG,GAA4C,CAAC,GAAG,EAAE;IAC1D,IAAI,GAAG,GAAG,CAAC,CAAA;IAEX,OAAQ,MAAM,CAAC,OAAO,CAAC,OAAO,CAA6C,CAAC,GAAG,CAC9E,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAqC,CAC/D,CAAA;AACF,CAAC,CAAC,EAAE,CAAA;AAEJ,SAAS,YAAY,CAAC,CAAe;IACpC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;IAEb,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG;QAAE,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,CAAA;IAE9C,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AACtB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAgB;IAChD,IAAI,EAAE,iBAAiB;IACvB,WAAW,EAAE,qGAAqG;IAClH,IAAI,EAAE,UAAU;IAChB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,sFAAsF;QACtF,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,MAAM,CAAA;QAClC,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,2BAA2B,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YACnG,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;YAChD,MAAM,SAAS,GAAG,aAAa,CAAC,uBAAuB,EAAE,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAE7F,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,OAAO;gBACP,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,MAAM,EAAE,uBAAuB;gBAC/B,SAAS;gBACT,cAAc,EAAE,OAAO;gBACvB,OAAO,EAAE,gFAAgF;aACzF,CAAA;YAED,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;YAErE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAChC,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,iGAAiG;YACjG,KAAK,CACJ,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,mBAAmB,GAAG,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAC/G,CAAA;YAED,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}
@@ -24,5 +24,9 @@
24
24
  * legacy script used.
25
25
  */
26
26
  import { type ShardRecipe } from "./scaffold.ts";
27
+ /**
28
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
29
+ * and `description` below for the surface form it generates.
30
+ */
27
31
  export declare const countryBalancedRecipe: ShardRecipe;
28
32
  //# sourceMappingURL=country-balanced.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"country-balanced.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/country-balanced.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAUH,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AA0VhE,eAAO,MAAM,qBAAqB,EAAE,WA0FnC,CAAA"}
1
+ {"version":3,"file":"country-balanced.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/country-balanced.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAUH,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AA6YhE;;;GAGG;AACH,eAAO,MAAM,qBAAqB,EAAE,WAoGnC,CAAA"}