@mailwoman/corpus 8.6.0 → 9.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (497) hide show
  1. package/README.md +2 -2
  2. package/data/PROVENANCE.md +214 -0
  3. package/data/sub-venue-lexicon.json +18269 -0
  4. package/out/src/adapters/ban/adapter.d.ts +5 -4
  5. package/out/src/adapters/ban/adapter.d.ts.map +1 -1
  6. package/out/src/adapters/ban/adapter.js +62 -69
  7. package/out/src/adapters/ban/adapter.js.map +1 -1
  8. package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
  9. package/out/src/adapters/ban/street-decompose.js +18 -25
  10. package/out/src/adapters/ban/street-decompose.js.map +1 -1
  11. package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -7
  12. package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
  13. package/out/src/adapters/fcc-bdc/adapter.js +4 -26
  14. package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
  15. package/out/src/adapters/geonames/adapter.d.ts +1 -1
  16. package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
  17. package/out/src/adapters/geonames/adapter.js +72 -78
  18. package/out/src/adapters/geonames/adapter.js.map +1 -1
  19. package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
  20. package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
  21. package/out/src/adapters/geonames-postal/adapter.js +46 -51
  22. package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
  23. package/out/src/adapters/gnaf/adapter.d.ts +1 -1
  24. package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
  25. package/out/src/adapters/gnaf/adapter.js +6 -9
  26. package/out/src/adapters/gnaf/adapter.js.map +1 -1
  27. package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
  28. package/out/src/adapters/gnaf/assemble.js +7 -12
  29. package/out/src/adapters/gnaf/assemble.js.map +1 -1
  30. package/out/src/adapters/index.d.ts +1 -1
  31. package/out/src/adapters/index.d.ts.map +1 -1
  32. package/out/src/adapters/index.js +1 -1
  33. package/out/src/adapters/index.js.map +1 -1
  34. package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
  35. package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
  36. package/out/src/adapters/openaddresses/adapter.js +5 -10
  37. package/out/src/adapters/openaddresses/adapter.js.map +1 -1
  38. package/out/src/adapters/overture/adapter.d.ts +1 -1
  39. package/out/src/adapters/overture/adapter.d.ts.map +1 -1
  40. package/out/src/adapters/overture/adapter.js +5 -9
  41. package/out/src/adapters/overture/adapter.js.map +1 -1
  42. package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
  43. package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
  44. package/out/src/adapters/state-hi-schools/adapter.js +57 -75
  45. package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
  46. package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
  47. package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
  48. package/out/src/adapters/state-ia-contractors/adapter.js +60 -78
  49. package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
  50. package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
  51. package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
  52. package/out/src/adapters/state-ny-notaries/adapter.js +69 -87
  53. package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
  54. package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
  55. package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
  56. package/out/src/adapters/state-tx-notaries/adapter.js +71 -89
  57. package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
  58. package/out/src/adapters/synth-po-box/adapter.d.ts +3 -3
  59. package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
  60. package/out/src/adapters/synth-po-box/adapter.js +10 -18
  61. package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
  62. package/out/src/adapters/tiger/adapter.d.ts +1 -1
  63. package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
  64. package/out/src/adapters/tiger/adapter.js +2 -2
  65. package/out/src/adapters/tiger/adapter.js.map +1 -1
  66. package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
  67. package/out/src/adapters/tiger/street-decompose.js +18 -27
  68. package/out/src/adapters/tiger/street-decompose.js.map +1 -1
  69. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +4 -4
  70. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
  71. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +64 -82
  72. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
  73. package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
  74. package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
  75. package/out/src/adapters/usgov-imls-pls/adapter.js +62 -84
  76. package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
  77. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
  78. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
  79. package/out/src/adapters/usgov-irs-bmf/adapter.js +59 -66
  80. package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
  81. package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
  82. package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
  83. package/out/src/adapters/usgov-nad/adapter.js +6 -9
  84. package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
  85. package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
  86. package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
  87. package/out/src/adapters/usgov-nppes/adapter.js +60 -78
  88. package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
  89. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
  90. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
  91. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +53 -73
  92. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
  93. package/out/src/{adapter.d.ts → adapters/utils/index.d.ts} +43 -2
  94. package/out/src/adapters/utils/index.d.ts.map +1 -0
  95. package/out/src/{adapter.js → adapters/utils/index.js} +47 -2
  96. package/out/src/adapters/utils/index.js.map +1 -0
  97. package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
  98. package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
  99. package/out/src/adapters/wof-admin-jp/adapter.js +24 -6
  100. package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
  101. package/out/src/adapters/wof-admin-json/adapter.d.ts +2 -2
  102. package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
  103. package/out/src/adapters/wof-admin-json/adapter.js +2 -2
  104. package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
  105. package/out/src/adapters/wof-postalcode-json/adapter.d.ts +2 -2
  106. package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
  107. package/out/src/adapters/wof-postalcode-json/adapter.js +2 -2
  108. package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
  109. package/out/src/build.d.ts +3 -4
  110. package/out/src/build.d.ts.map +1 -1
  111. package/out/src/build.js +6 -8
  112. package/out/src/build.js.map +1 -1
  113. package/out/src/index.d.ts +3 -17
  114. package/out/src/index.d.ts.map +1 -1
  115. package/out/src/index.js +3 -17
  116. package/out/src/index.js.map +1 -1
  117. package/out/src/name-prone-us-suffixes.d.ts +12 -0
  118. package/out/src/name-prone-us-suffixes.d.ts.map +1 -0
  119. package/out/src/name-prone-us-suffixes.js +12 -0
  120. package/out/src/name-prone-us-suffixes.js.map +1 -0
  121. package/out/src/parquet-wrapper/reader.d.ts +8 -0
  122. package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
  123. package/out/src/parquet-wrapper/reader.js +16 -0
  124. package/out/src/parquet-wrapper/reader.js.map +1 -1
  125. package/out/src/parquet-wrapper/schema.d.ts +4 -0
  126. package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
  127. package/out/src/parquet-wrapper/schema.js.map +1 -1
  128. package/out/src/runner.d.ts +2 -2
  129. package/out/src/runner.d.ts.map +1 -1
  130. package/out/src/runner.js +1 -1
  131. package/out/src/runner.js.map +1 -1
  132. package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
  133. package/out/src/shard-recipes/anchor-absorption.js +5 -4
  134. package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
  135. package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
  136. package/out/src/shard-recipes/boundary-stress.js +2 -2
  137. package/out/src/shard-recipes/boundary-stress.js.map +1 -1
  138. package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
  139. package/out/src/shard-recipes/country-balanced.js +75 -75
  140. package/out/src/shard-recipes/country-balanced.js.map +1 -1
  141. package/out/src/shard-recipes/fr-admin-split.js +2 -2
  142. package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
  143. package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
  144. package/out/src/shard-recipes/fr-fragment.js +8 -5
  145. package/out/src/shard-recipes/fr-fragment.js.map +1 -1
  146. package/out/src/shard-recipes/fr-lieudit.js +2 -2
  147. package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
  148. package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
  149. package/out/src/shard-recipes/fr-order.js +18 -72
  150. package/out/src/shard-recipes/fr-order.js.map +1 -1
  151. package/out/src/shard-recipes/german.d.ts.map +1 -1
  152. package/out/src/shard-recipes/german.js +17 -70
  153. package/out/src/shard-recipes/german.js.map +1 -1
  154. package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
  155. package/out/src/shard-recipes/house-venue.js +1 -1
  156. package/out/src/shard-recipes/house-venue.js.map +1 -1
  157. package/out/src/shard-recipes/index.d.ts.map +1 -1
  158. package/out/src/shard-recipes/index.js +4 -1
  159. package/out/src/shard-recipes/index.js.map +1 -1
  160. package/out/src/shard-recipes/intersection.d.ts +2 -2
  161. package/out/src/shard-recipes/intersection.d.ts.map +1 -1
  162. package/out/src/shard-recipes/intersection.js +28 -72
  163. package/out/src/shard-recipes/intersection.js.map +1 -1
  164. package/out/src/shard-recipes/locale.d.ts +5 -5
  165. package/out/src/shard-recipes/locale.d.ts.map +1 -1
  166. package/out/src/shard-recipes/locale.js +25 -35
  167. package/out/src/shard-recipes/locale.js.map +1 -1
  168. package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
  169. package/out/src/shard-recipes/no-fragment.js +8 -5
  170. package/out/src/shard-recipes/no-fragment.js.map +1 -1
  171. package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
  172. package/out/src/shard-recipes/no-street-led.js +8 -5
  173. package/out/src/shard-recipes/no-street-led.js.map +1 -1
  174. package/out/src/shard-recipes/no-street.d.ts.map +1 -1
  175. package/out/src/shard-recipes/no-street.js +1 -1
  176. package/out/src/shard-recipes/no-street.js.map +1 -1
  177. package/out/src/shard-recipes/po-box-cedex.d.ts +3 -3
  178. package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
  179. package/out/src/shard-recipes/po-box-cedex.js +101 -127
  180. package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
  181. package/out/src/shard-recipes/po-box.d.ts.map +1 -1
  182. package/out/src/shard-recipes/po-box.js +1 -1
  183. package/out/src/shard-recipes/po-box.js.map +1 -1
  184. package/out/src/shard-recipes/scaffold.d.ts +58 -9
  185. package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
  186. package/out/src/shard-recipes/scaffold.js +87 -29
  187. package/out/src/shard-recipes/scaffold.js.map +1 -1
  188. package/out/src/shard-recipes/street-affix.d.ts +54 -0
  189. package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
  190. package/out/src/shard-recipes/street-affix.js +253 -111
  191. package/out/src/shard-recipes/street-affix.js.map +1 -1
  192. package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
  193. package/out/src/shard-recipes/street-bare.js +3 -3
  194. package/out/src/shard-recipes/street-bare.js.map +1 -1
  195. package/out/src/shard-recipes/street.d.ts.map +1 -1
  196. package/out/src/shard-recipes/street.js +2 -2
  197. package/out/src/shard-recipes/street.js.map +1 -1
  198. package/out/src/shard-recipes/sub-venue-sources.d.ts +228 -0
  199. package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -0
  200. package/out/src/shard-recipes/sub-venue-sources.js +664 -0
  201. package/out/src/shard-recipes/sub-venue-sources.js.map +1 -0
  202. package/out/src/shard-recipes/sub-venue.d.ts +269 -0
  203. package/out/src/shard-recipes/sub-venue.d.ts.map +1 -0
  204. package/out/src/shard-recipes/sub-venue.js +709 -0
  205. package/out/src/shard-recipes/sub-venue.js.map +1 -0
  206. package/out/src/shard-recipes/unit.d.ts +1 -1
  207. package/out/src/shard-recipes/unit.d.ts.map +1 -1
  208. package/out/src/shard-recipes/unit.js +26 -74
  209. package/out/src/shard-recipes/unit.js.map +1 -1
  210. package/out/src/{synthesize-anchor-absorption.d.ts → synthesizers/anchor-absorption.d.ts} +1 -1
  211. package/out/src/synthesizers/anchor-absorption.d.ts.map +1 -0
  212. package/out/src/{synthesize-anchor-absorption.js → synthesizers/anchor-absorption.js} +1 -1
  213. package/out/src/synthesizers/anchor-absorption.js.map +1 -0
  214. package/out/src/{synthesize-boundary-stress.d.ts → synthesizers/boundary-stress.d.ts} +3 -3
  215. package/out/src/synthesizers/boundary-stress.d.ts.map +1 -0
  216. package/out/src/{synthesize-boundary-stress.js → synthesizers/boundary-stress.js} +2 -2
  217. package/out/src/synthesizers/boundary-stress.js.map +1 -0
  218. package/out/src/{synthesize-german.d.ts → synthesizers/german.d.ts} +2 -2
  219. package/out/src/synthesizers/german.d.ts.map +1 -0
  220. package/out/src/{synthesize-german.js → synthesizers/german.js} +2 -2
  221. package/out/src/synthesizers/german.js.map +1 -0
  222. package/out/src/{synthesize-house-venue.d.ts → synthesizers/house-venue.d.ts} +4 -4
  223. package/out/src/synthesizers/house-venue.d.ts.map +1 -0
  224. package/out/src/{synthesize-house-venue.js → synthesizers/house-venue.js} +4 -4
  225. package/out/src/synthesizers/house-venue.js.map +1 -0
  226. package/out/src/{synthesize-intersection.d.ts → synthesizers/intersection.d.ts} +2 -2
  227. package/out/src/synthesizers/intersection.d.ts.map +1 -0
  228. package/out/src/{synthesize-intersection.js → synthesizers/intersection.js} +1 -1
  229. package/out/src/synthesizers/intersection.js.map +1 -0
  230. package/out/src/{synthesize-no-street.d.ts → synthesizers/no-street.d.ts} +3 -3
  231. package/out/src/synthesizers/no-street.d.ts.map +1 -0
  232. package/out/src/{synthesize-no-street.js → synthesizers/no-street.js} +2 -2
  233. package/out/src/synthesizers/no-street.js.map +1 -0
  234. package/out/src/{synthesize-po-box.d.ts → synthesizers/po-box.d.ts} +2 -2
  235. package/out/src/synthesizers/po-box.d.ts.map +1 -0
  236. package/out/src/{synthesize-po-box.js → synthesizers/po-box.js} +1 -1
  237. package/out/src/synthesizers/po-box.js.map +1 -0
  238. package/out/src/{synthesize-street.d.ts → synthesizers/street.d.ts} +2 -2
  239. package/out/src/synthesizers/street.d.ts.map +1 -0
  240. package/out/src/{synthesize-street.js → synthesizers/street.js} +2 -2
  241. package/out/src/synthesizers/street.js.map +1 -0
  242. package/out/src/{synthesize.d.ts → synthesizers/utils.d.ts} +3 -3
  243. package/out/src/synthesizers/utils.d.ts.map +1 -0
  244. package/out/src/{synthesize.js → synthesizers/utils.js} +4 -15
  245. package/out/src/synthesizers/utils.js.map +1 -0
  246. package/out/src/tools/align-shard.d.ts.map +1 -1
  247. package/out/src/tools/align-shard.js +4 -8
  248. package/out/src/tools/align-shard.js.map +1 -1
  249. package/out/src/tools/audit.d.ts.map +1 -1
  250. package/out/src/tools/audit.js +4 -4
  251. package/out/src/tools/audit.js.map +1 -1
  252. package/out/src/tools/corpus-stats.d.ts +2 -2
  253. package/out/src/tools/corpus-stats.d.ts.map +1 -1
  254. package/out/src/tools/corpus-stats.js +18 -31
  255. package/out/src/tools/corpus-stats.js.map +1 -1
  256. package/out/src/tools/fetch/download.d.ts.map +1 -1
  257. package/out/src/tools/fetch/download.js +5 -6
  258. package/out/src/tools/fetch/download.js.map +1 -1
  259. package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
  260. package/out/src/tools/fetch/imls-pls.js +3 -15
  261. package/out/src/tools/fetch/imls-pls.js.map +1 -1
  262. package/out/src/tools/fetch/index.d.ts +14 -1
  263. package/out/src/tools/fetch/index.d.ts.map +1 -1
  264. package/out/src/tools/fetch/index.js +14 -1
  265. package/out/src/tools/fetch/index.js.map +1 -1
  266. package/out/src/tools/fetch/nad.d.ts +1 -1
  267. package/out/src/tools/fetch/nad.js +1 -1
  268. package/out/src/tools/fetch/nppes.d.ts.map +1 -1
  269. package/out/src/tools/fetch/nppes.js +8 -14
  270. package/out/src/tools/fetch/nppes.js.map +1 -1
  271. package/out/src/tools/fetch/openaddresses.d.ts +1 -1
  272. package/out/src/tools/fetch/openaddresses.js +1 -1
  273. package/out/src/tools/fetch/ourairports.d.ts +45 -0
  274. package/out/src/tools/fetch/ourairports.d.ts.map +1 -0
  275. package/out/src/tools/fetch/ourairports.js +123 -0
  276. package/out/src/tools/fetch/ourairports.js.map +1 -0
  277. package/out/src/tools/fetch/wikidata-subvenue.d.ts +171 -0
  278. package/out/src/tools/fetch/wikidata-subvenue.d.ts.map +1 -0
  279. package/out/src/tools/fetch/wikidata-subvenue.js +275 -0
  280. package/out/src/tools/fetch/wikidata-subvenue.js.map +1 -0
  281. package/out/src/tools/golden-expand.d.ts.map +1 -1
  282. package/out/src/tools/golden-expand.js +18 -16
  283. package/out/src/tools/golden-expand.js.map +1 -1
  284. package/out/src/tools/golden-promote.d.ts.map +1 -1
  285. package/out/src/tools/golden-promote.js +12 -6
  286. package/out/src/tools/golden-promote.js.map +1 -1
  287. package/out/src/tools/golden-relabel-street.d.ts +196 -0
  288. package/out/src/tools/golden-relabel-street.d.ts.map +1 -0
  289. package/out/src/tools/golden-relabel-street.js +440 -0
  290. package/out/src/tools/golden-relabel-street.js.map +1 -0
  291. package/out/src/tools/index.d.ts +4 -0
  292. package/out/src/tools/index.d.ts.map +1 -1
  293. package/out/src/tools/index.js +4 -0
  294. package/out/src/tools/index.js.map +1 -1
  295. package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
  296. package/out/src/tools/jsonl-to-parquet.js +3 -2
  297. package/out/src/tools/jsonl-to-parquet.js.map +1 -1
  298. package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
  299. package/out/src/tools/lint-shard-vocab.js +3 -2
  300. package/out/src/tools/lint-shard-vocab.js.map +1 -1
  301. package/out/src/tools/lint-shard.d.ts +3 -3
  302. package/out/src/tools/lint-shard.d.ts.map +1 -1
  303. package/out/src/tools/lint-shard.js +23 -32
  304. package/out/src/tools/lint-shard.js.map +1 -1
  305. package/out/src/tools/overlay-manifest.d.ts.map +1 -1
  306. package/out/src/tools/overlay-manifest.js +2 -1
  307. package/out/src/tools/overlay-manifest.js.map +1 -1
  308. package/out/src/tools/overture-subvenue.d.ts +112 -0
  309. package/out/src/tools/overture-subvenue.d.ts.map +1 -0
  310. package/out/src/tools/overture-subvenue.js +144 -0
  311. package/out/src/tools/overture-subvenue.js.map +1 -0
  312. package/out/src/tools/shard-kryptonite.d.ts +1 -1
  313. package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
  314. package/out/src/tools/shard-kryptonite.js +6 -6
  315. package/out/src/tools/shard-kryptonite.js.map +1 -1
  316. package/out/src/tools/shard-translit.d.ts +6 -3
  317. package/out/src/tools/shard-translit.d.ts.map +1 -1
  318. package/out/src/tools/shard-translit.js +9 -8
  319. package/out/src/tools/shard-translit.js.map +1 -1
  320. package/out/src/tools/sub-venue-lexicon.d.ts +507 -0
  321. package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -0
  322. package/out/src/tools/sub-venue-lexicon.js +817 -0
  323. package/out/src/tools/sub-venue-lexicon.js.map +1 -0
  324. package/out/src/tools/sub-venue-promotions.d.ts +94 -0
  325. package/out/src/tools/sub-venue-promotions.d.ts.map +1 -0
  326. package/out/src/tools/sub-venue-promotions.js +266 -0
  327. package/out/src/tools/sub-venue-promotions.js.map +1 -0
  328. package/out/src/{align.d.ts → utils/align.d.ts} +1 -1
  329. package/out/src/utils/align.d.ts.map +1 -0
  330. package/out/src/utils/align.js.map +1 -0
  331. package/out/src/{golden.d.ts → utils/golden.d.ts} +5 -0
  332. package/out/src/utils/golden.d.ts.map +1 -0
  333. package/out/src/{golden.js → utils/golden.js} +24 -10
  334. package/out/src/utils/golden.js.map +1 -0
  335. package/out/src/utils/index.d.ts +14 -0
  336. package/out/src/utils/index.d.ts.map +1 -0
  337. package/out/src/utils/index.js +14 -0
  338. package/out/src/utils/index.js.map +1 -0
  339. package/out/src/utils/license.d.ts.map +1 -0
  340. package/out/src/utils/license.js.map +1 -0
  341. package/out/src/{parquet.d.ts → utils/parquet.d.ts} +2 -2
  342. package/out/src/utils/parquet.d.ts.map +1 -0
  343. package/out/src/{parquet.js → utils/parquet.js} +3 -15
  344. package/out/src/utils/parquet.js.map +1 -0
  345. package/out/src/{split.d.ts → utils/split.d.ts} +8 -1
  346. package/out/src/utils/split.d.ts.map +1 -0
  347. package/out/src/{split.js → utils/split.js} +7 -0
  348. package/out/src/utils/split.js.map +1 -0
  349. package/out/src/utils/tokenize.d.ts.map +1 -0
  350. package/out/src/utils/tokenize.js.map +1 -0
  351. package/out/src/{wof-json.d.ts → utils/wof-json.d.ts} +2 -2
  352. package/out/src/utils/wof-json.d.ts.map +1 -0
  353. package/out/src/{wof-json.js → utils/wof-json.js} +5 -8
  354. package/out/src/utils/wof-json.js.map +1 -0
  355. package/package.json +66 -20
  356. package/src/adapters/ban/adapter.ts +58 -68
  357. package/src/adapters/ban/street-decompose.ts +17 -25
  358. package/src/adapters/fcc-bdc/adapter.ts +5 -36
  359. package/src/adapters/geonames/adapter.ts +67 -76
  360. package/src/adapters/geonames-postal/adapter.ts +45 -54
  361. package/src/adapters/gnaf/adapter.ts +7 -11
  362. package/src/adapters/gnaf/assemble.ts +7 -11
  363. package/src/adapters/index.ts +3 -2
  364. package/src/adapters/openaddresses/adapter.ts +6 -12
  365. package/src/adapters/overture/adapter.ts +6 -10
  366. package/src/adapters/state-hi-schools/adapter.ts +53 -77
  367. package/src/adapters/state-ia-contractors/adapter.ts +55 -80
  368. package/src/adapters/state-ny-notaries/adapter.ts +64 -89
  369. package/src/adapters/state-tx-notaries/adapter.ts +63 -88
  370. package/src/adapters/synth-po-box/adapter.ts +12 -23
  371. package/src/adapters/tiger/adapter.ts +4 -3
  372. package/src/adapters/tiger/street-decompose.ts +21 -31
  373. package/src/adapters/usgov-hrsa-fqhc/adapter.ts +64 -88
  374. package/src/adapters/usgov-imls-pls/adapter.ts +57 -86
  375. package/src/adapters/usgov-irs-bmf/adapter.ts +55 -66
  376. package/src/adapters/usgov-nad/adapter.ts +7 -11
  377. package/src/adapters/usgov-nppes/adapter.ts +54 -79
  378. package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +48 -75
  379. package/src/{adapter.ts → adapters/utils/index.ts} +59 -3
  380. package/src/adapters/wof-admin-jp/adapter.ts +35 -9
  381. package/src/adapters/wof-admin-json/adapter.ts +3 -4
  382. package/src/adapters/wof-postalcode-json/adapter.ts +3 -4
  383. package/src/build.ts +12 -11
  384. package/src/index.ts +3 -17
  385. package/src/name-prone-us-suffixes.ts +12 -0
  386. package/src/parquet-wrapper/reader.ts +19 -0
  387. package/src/parquet-wrapper/schema.ts +4 -0
  388. package/src/runner.ts +7 -2
  389. package/src/shard-recipes/anchor-absorption.ts +5 -4
  390. package/src/shard-recipes/boundary-stress.ts +6 -2
  391. package/src/shard-recipes/country-balanced.ts +77 -91
  392. package/src/shard-recipes/fr-admin-split.ts +3 -3
  393. package/src/shard-recipes/fr-fragment.ts +10 -7
  394. package/src/shard-recipes/fr-lieudit.ts +3 -3
  395. package/src/shard-recipes/fr-order.ts +18 -83
  396. package/src/shard-recipes/german.ts +17 -80
  397. package/src/shard-recipes/house-venue.ts +2 -1
  398. package/src/shard-recipes/index.ts +4 -1
  399. package/src/shard-recipes/intersection.ts +37 -74
  400. package/src/shard-recipes/locale.ts +31 -39
  401. package/src/shard-recipes/no-fragment.ts +10 -7
  402. package/src/shard-recipes/no-street-led.ts +10 -7
  403. package/src/shard-recipes/no-street.ts +2 -1
  404. package/src/shard-recipes/po-box-cedex.ts +127 -161
  405. package/src/shard-recipes/po-box.ts +6 -1
  406. package/src/shard-recipes/scaffold.ts +125 -27
  407. package/src/shard-recipes/street-affix.ts +335 -134
  408. package/src/shard-recipes/street-bare.ts +4 -3
  409. package/src/shard-recipes/street.ts +3 -2
  410. package/src/shard-recipes/sub-venue-sources.ts +895 -0
  411. package/src/shard-recipes/sub-venue.ts +982 -0
  412. package/src/shard-recipes/unit.ts +27 -85
  413. package/src/{synthesize-boundary-stress.ts → synthesizers/boundary-stress.ts} +2 -2
  414. package/src/{synthesize-german.ts → synthesizers/german.ts} +2 -2
  415. package/src/{synthesize-house-venue.ts → synthesizers/house-venue.ts} +4 -4
  416. package/src/{synthesize-intersection.ts → synthesizers/intersection.ts} +1 -1
  417. package/src/{synthesize-no-street.ts → synthesizers/no-street.ts} +2 -2
  418. package/src/{synthesize-po-box.ts → synthesizers/po-box.ts} +1 -1
  419. package/src/{synthesize-street.ts → synthesizers/street.ts} +2 -2
  420. package/src/{synthesize.ts → synthesizers/utils.ts} +4 -18
  421. package/src/tools/align-shard.ts +4 -7
  422. package/src/tools/audit.ts +6 -5
  423. package/src/tools/corpus-stats.ts +25 -33
  424. package/src/tools/fetch/download.ts +7 -5
  425. package/src/tools/fetch/imls-pls.ts +4 -18
  426. package/src/tools/fetch/index.ts +14 -1
  427. package/src/tools/fetch/nad.ts +1 -1
  428. package/src/tools/fetch/nppes.ts +9 -16
  429. package/src/tools/fetch/openaddresses.ts +1 -1
  430. package/src/tools/fetch/ourairports.ts +166 -0
  431. package/src/tools/fetch/wikidata-subvenue.ts +386 -0
  432. package/src/tools/golden-expand.ts +18 -13
  433. package/src/tools/golden-promote.ts +15 -6
  434. package/src/tools/golden-relabel-street.ts +685 -0
  435. package/src/tools/index.ts +4 -0
  436. package/src/tools/jsonl-to-parquet.ts +3 -2
  437. package/src/tools/lint-shard-vocab.ts +3 -2
  438. package/src/tools/lint-shard.ts +33 -36
  439. package/src/tools/overlay-manifest.ts +2 -1
  440. package/src/tools/overture-subvenue.ts +215 -0
  441. package/src/tools/shard-kryptonite.ts +8 -9
  442. package/src/tools/shard-translit.ts +22 -11
  443. package/src/tools/sub-venue-lexicon.ts +1250 -0
  444. package/src/tools/sub-venue-promotions.ts +330 -0
  445. package/src/{align.ts → utils/align.ts} +1 -1
  446. package/src/{golden.ts → utils/golden.ts} +25 -11
  447. package/src/utils/index.ts +14 -0
  448. package/src/{parquet.ts → utils/parquet.ts} +5 -19
  449. package/src/{split.ts → utils/split.ts} +8 -2
  450. package/src/{wof-json.ts → utils/wof-json.ts} +5 -8
  451. package/out/src/adapter.d.ts.map +0 -1
  452. package/out/src/adapter.js.map +0 -1
  453. package/out/src/align.d.ts.map +0 -1
  454. package/out/src/align.js.map +0 -1
  455. package/out/src/format.d.ts +0 -14
  456. package/out/src/format.d.ts.map +0 -1
  457. package/out/src/format.js +0 -14
  458. package/out/src/format.js.map +0 -1
  459. package/out/src/golden.d.ts.map +0 -1
  460. package/out/src/golden.js.map +0 -1
  461. package/out/src/license.d.ts.map +0 -1
  462. package/out/src/license.js.map +0 -1
  463. package/out/src/parquet.d.ts.map +0 -1
  464. package/out/src/parquet.js.map +0 -1
  465. package/out/src/split.d.ts.map +0 -1
  466. package/out/src/split.js.map +0 -1
  467. package/out/src/synthesize-anchor-absorption.d.ts.map +0 -1
  468. package/out/src/synthesize-anchor-absorption.js.map +0 -1
  469. package/out/src/synthesize-boundary-stress.d.ts.map +0 -1
  470. package/out/src/synthesize-boundary-stress.js.map +0 -1
  471. package/out/src/synthesize-german.d.ts.map +0 -1
  472. package/out/src/synthesize-german.js.map +0 -1
  473. package/out/src/synthesize-house-venue.d.ts.map +0 -1
  474. package/out/src/synthesize-house-venue.js.map +0 -1
  475. package/out/src/synthesize-intersection.d.ts.map +0 -1
  476. package/out/src/synthesize-intersection.js.map +0 -1
  477. package/out/src/synthesize-no-street.d.ts.map +0 -1
  478. package/out/src/synthesize-no-street.js.map +0 -1
  479. package/out/src/synthesize-po-box.d.ts.map +0 -1
  480. package/out/src/synthesize-po-box.js.map +0 -1
  481. package/out/src/synthesize-street.d.ts.map +0 -1
  482. package/out/src/synthesize-street.js.map +0 -1
  483. package/out/src/synthesize.d.ts.map +0 -1
  484. package/out/src/synthesize.js.map +0 -1
  485. package/out/src/tokenize.d.ts.map +0 -1
  486. package/out/src/tokenize.js.map +0 -1
  487. package/out/src/wof-json.d.ts.map +0 -1
  488. package/out/src/wof-json.js.map +0 -1
  489. package/src/format.ts +0 -14
  490. /package/out/src/{align.js → utils/align.js} +0 -0
  491. /package/out/src/{license.d.ts → utils/license.d.ts} +0 -0
  492. /package/out/src/{license.js → utils/license.js} +0 -0
  493. /package/out/src/{tokenize.d.ts → utils/tokenize.d.ts} +0 -0
  494. /package/out/src/{tokenize.js → utils/tokenize.js} +0 -0
  495. /package/src/{synthesize-anchor-absorption.ts → synthesizers/anchor-absorption.ts} +0 -0
  496. /package/src/{license.ts → utils/license.ts} +0 -0
  497. /package/src/{tokenize.ts → utils/tokenize.ts} +0 -0
@@ -23,7 +23,7 @@
23
23
  * - USPS Pub 28 §28C2.040 — Private Mailbox formatting
24
24
  * - USPS DMM 508 §4.1.4 / §4.5.4 — PO Box and street-addressed PO Box
25
25
  */
26
- import type { CanonicalRow } from "./types.ts";
26
+ import type { CanonicalRow } from "@mailwoman/corpus/types";
27
27
  export interface PoBoxBaseTuple {
28
28
  locality: string;
29
29
  region: string;
@@ -101,4 +101,4 @@ export declare function supportedLocales(): ReadonlyArray<string>;
101
101
  * locales as "missing region".
102
102
  */
103
103
  export declare const REGION_OPTIONAL_LOCALES: ReadonlySet<string>;
104
- //# sourceMappingURL=synthesize-po-box.d.ts.map
104
+ //# sourceMappingURL=po-box.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"po-box.d.ts","sourceRoot":"","sources":["../../../src/synthesizers/po-box.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAA;AAkB3D,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,cAAc;IAC9B,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;IAE9B,GAAG,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;CAC3B;AAED;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,EAAE,aAAa,CAAC,cAAc,CA2CjE,CAAA;AAID;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,MAAM,GAAG,MAAM,CAe/E;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAEzE;AAED,MAAM,WAAW,mBAAmB;IACnC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,QAAQ,GAAG,iBAAiB,GAAG,iBAAiB,CAAA;CAC1D;AAED,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB;;OAEG;IACH,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,MAAM,KAAK,MAAM,CAAA;IAC7C;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AAkBD;;;GAGG;AACH,wBAAgB,kBAAkB,CACjC,IAAI,EAAE,cAAc,GAAG;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,EAChE,IAAI,GAAE,kBAAuB,GAC3B,mBAAmB,GAAG,IAAI,CAyD5B;AA0BD;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,GAAE,kBAAuB,GAAG,mBAAmB,CAuB7F;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAsBvD;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,aAAa,CAAC,MAAM,CAAC,CAExD;AAED;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,EAAE,WAAW,CAAC,MAAM,CAAsB,CAAA"}
@@ -263,4 +263,4 @@ export function supportedLocales() {
263
263
  * locales as "missing region".
264
264
  */
265
265
  export const REGION_OPTIONAL_LOCALES = new Set(["en-NZ"]);
266
- //# sourceMappingURL=synthesize-po-box.js.map
266
+ //# sourceMappingURL=po-box.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"po-box.js","sourceRoot":"","sources":["../../../src/synthesizers/po-box.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAIH;;GAEG;AACH,MAAM,6BAA6B,GAAG,CAAC,CAAA;AAEvC;;GAEG;AACH,MAAM,8BAA8B,GAAG,CAAC,CAAA;AAsBxC;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAkC;IACrE;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,iBAAiB,EAAE,KAAK,CAAC;QACrF,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC;KACjB;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,iBAAiB,CAAC;QACzD,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC;KACjB;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,iBAAiB,CAAC;KAClD;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,EAAE,YAAY,CAAC;KAC3E;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,iBAAiB,EAAE,aAAa,EAAE,aAAa,CAAC;KAChF;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,KAAK,CAAC;KAC/C;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC;KACrD;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,qBAAqB,CAAC;KAC7D;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,IAAI,CAAC;KACvD;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,SAAS,EAAE,mBAAmB,EAAE,IAAI,CAAC;KAC/C;CACD,CAAA;AAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAyB,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAE5G;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAW,EAAE,MAAoB;IACtE,IAAI,MAAM,EAAE,GAAG,GAAG;QAAE,OAAO,GAAG,CAAA;IAE9B,MAAM,QAAQ,GAAiC;QAC9C,qDAAqD;QACrD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,6BAA6B,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3F,wCAAwC;QACxC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,8BAA8B,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5F,mDAAmD;QACnD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;KAC5B,CAAA;IAED,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAE,CAAA;IAE3D,OAAO,CAAC,CAAC,GAAG,CAAC,CAAA;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAc,EAAE,MAAc;IAChE,OAAO,GAAG,MAAM,IAAI,MAAM,EAAE,CAAA;AAC7B,CAAC;AAwBD,SAAS,iBAAiB,CAAC,MAAoB;IAC9C,sFAAsF;IACtF,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;IAElB,IAAI,CAAC,GAAG,GAAG;QAAE,OAAO,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;IAEzD,OAAO;IACP,IAAI,CAAC,GAAG,GAAG;QAAE,OAAO,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAA;IAE5D,UAAU;IACV,IAAI,CAAC,GAAG,IAAI;QAAE,OAAO,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAA;IAE/D,YAAY;IACZ,OAAO,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAA,CAAC,cAAc;AACrE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CACjC,IAAgE,EAChE,OAA2B,EAAE;IAE7B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAA;IACzC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,iBAAiB,CAAA;IACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAA;IAEnC,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAC5C,MAAM,GAAG,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAEzC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAErB,MAAM,MAAM,GAAG,qBAAqB,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAA;IAChE,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAE,CAAA;IACtE,MAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAEtD,mEAAmE;IACnE,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,MAAM,EAAE,GAAG,QAAQ,CAAA;IAE7D,IAAI,OAAO,EAAE,CAAC;QACb,MAAM,SAAS,GAAG,GAAG,CAAC,GAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,GAAI,CAAC,MAAM,CAAC,CAAE,CAAA;QACnE,MAAM,SAAS,GAAG,kBAAkB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;QACvD,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAO,CAAA;QACzF,MAAM,GAAG,GAAG,GAAG,UAAU,KAAK,SAAS,KAAK,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAA;QAE5F,OAAO;YACN,GAAG;YACH,UAAU,EAAE;gBACX,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/D,MAAM,EAAE,IAAI,CAAC,MAAO;gBACpB,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,OAAO,EAAE,IAAI,CAAC,OAAO;aACrB;YACD,MAAM;YACN,QAAQ,EAAE,iBAAiB;SAC3B,CAAA;IACF,CAAC;IAED,kGAAkG;IAClG,oGAAoG;IACpG,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;IAC5D,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAA;IAClH,MAAM,GAAG,GAAG,GAAG,WAAW,KAAK,IAAI,EAAE,CAAA;IAErC,OAAO;QACN,GAAG;QACH,UAAU,EAAE;YACX,MAAM,EAAE,WAAW;YACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7C,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,OAAO,EAAE,IAAI,CAAC,OAAO;SACrB;QACD,MAAM;QACN,QAAQ,EAAE,QAAQ;KAClB,CAAA;AACF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,SAAS,GAA0D;IACxE,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE;IAClC,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE;IAClC,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE;CACpC,CAAA;AAED,MAAM,YAAY,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAU,CAAA;AAEnD;;GAEG;AACH,MAAM,cAAc,GAAwE;IAC3F,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE;IACpF,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE;IACzF,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE;CACpF,CAAA;AAED;;GAEG;AACH,MAAM,UAAU,0BAA0B,CAAC,OAA2B,EAAE;IACvE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAA;IACzC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAE,CAAA;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAA;IACtD,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,CAAE,CAAA;IACrF,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAA;IAC1B,MAAM,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,CAAE,CAAA;IACpE,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,IAAI,MAAM,EAAE,GAAG,GAAG,CAAA;IACjD,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,QAAQ,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,EAAE,CAAA;IACpH,MAAM,GAAG,GAAG,GAAG,QAAQ,KAAK,EAAE,IAAI,MAAM,IAAI,MAAM,EAAE,CAAA;IAEpD,OAAO;QACN,GAAG;QACH,UAAU,EAAE;YACX,MAAM,EAAE,QAAQ;YAChB,QAAQ,EAAE,EAAE;YACZ,MAAM;YACN,QAAQ,EAAE,MAAM;YAChB,OAAO,EAAE,IAAI;SACb;QACD,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,iBAAiB;KAC3B,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe;IAC9C,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAEtC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,eAAe;QAAE,OAAO,OAAO,CAAA;IAEtE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAA;IAE/D,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,gBAAgB;QAAE,OAAO,OAAO,CAAA;IAErF,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,WAAW;QAAE,OAAO,OAAO,CAAA;IAElE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,aAAa;QAAE,OAAO,OAAO,CAAA;IAEpE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAA;IAE/D,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,OAAO;QAAE,OAAO,OAAO,CAAA;IAE9D,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAA;IAE/D,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,WAAW;QAAE,OAAO,OAAO,CAAA;IAElE,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB;IAC/B,OAAO,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;AACpD,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAwB,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAA"}
@@ -18,7 +18,7 @@
18
18
  * distribution matches what the model would see if/when TIGER shards are rebuilt with the new
19
19
  * adapter.
20
20
  */
21
- import type { CanonicalRow } from "./types.ts";
21
+ import type { CanonicalRow } from "@mailwoman/corpus/types";
22
22
  export interface StreetBaseTuple {
23
23
  locality: string;
24
24
  region: string;
@@ -51,4 +51,4 @@ export interface StreetSynthesisOpts {
51
51
  * distribution matches the canonical decomposition logic.
52
52
  */
53
53
  export declare function synthesizeStreetRow(base: StreetBaseTuple, opts?: StreetSynthesisOpts): SynthesizedStreetRow | null;
54
- //# sourceMappingURL=synthesize-street.d.ts.map
54
+ //# sourceMappingURL=street.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"street.d.ts","sourceRoot":"","sources":["../../../src/synthesizers/street.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAA;AAwI3D,MAAM,WAAW,eAAe;IAC/B,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,oBAAoB;IACpC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,mBAAmB;IACnC,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB;;OAEG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAA;IAC/B;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AAoBD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAClC,IAAI,EAAE,eAAe,EACrB,IAAI,GAAE,mBAAwB,GAC5B,oBAAoB,GAAG,IAAI,CA0D7B"}
@@ -18,7 +18,7 @@
18
18
  * distribution matches what the model would see if/when TIGER shards are rebuilt with the new
19
19
  * adapter.
20
20
  */
21
- import { decomposeStreet } from "./adapters/tiger/street-decompose.js";
21
+ import { decomposeStreet } from "@mailwoman/corpus/adapters/tiger/street-decompose";
22
22
  // Hand-curated US street name pool. Real frequency-weighted street names — sampled
23
23
  // from US Census TIGER 2024 top-1000 by occurrence count. Keep ~50 entries so the
24
24
  // synthesis distribution doesn't overfit to a tiny vocabulary.
@@ -217,4 +217,4 @@ export function synthesizeStreetRow(base, opts = {}) {
217
217
  }
218
218
  return { raw, components, locale: "en-US" };
219
219
  }
220
- //# sourceMappingURL=synthesize-street.js.map
220
+ //# sourceMappingURL=street.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"street.js","sourceRoot":"","sources":["../../../src/synthesizers/street.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,mDAAmD,CAAA;AAGnF,mFAAmF;AACnF,kFAAkF;AAClF,+DAA+D;AAC/D;;;;yCAIyC;AAEzC,MAAM,YAAY,GAAG;IACpB,MAAM;IACN,KAAK;IACL,OAAO;IACP,MAAM;IACN,OAAO;IACP,KAAK;IACL,YAAY;IACZ,SAAS;IACT,WAAW;IACX,MAAM;IACN,MAAM;IACN,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,MAAM;IACN,QAAQ;IACR,OAAO;IACP,UAAU;IACV,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,QAAQ;IACR,UAAU;IACV,WAAW;IACX,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,OAAO;IACP,SAAS;IACT,QAAQ;IACR,SAAS;IACT,UAAU;IACV,SAAS;IACT,WAAW;IACX,UAAU;IACV,QAAQ;IACR,UAAU;IACV,YAAY;IACZ,SAAS;IACT,UAAU;IACV,SAAS;IACT,OAAO;IACP,KAAK;IACL,QAAQ;IACR,OAAO;IACP,OAAO;CACE,CAAA;AAEV,MAAM,eAAe,GAAG;IACvB,IAAI;IACJ,QAAQ;IACR,KAAK;IACL,QAAQ;IACR,IAAI;IACJ,MAAM;IACN,MAAM;IACN,WAAW;IACX,IAAI;IACJ,OAAO;IACP,IAAI;IACJ,MAAM;IACN,KAAK;IACL,IAAI;IACJ,OAAO;IACP,IAAI;IACJ,OAAO;IACP,MAAM;IACN,SAAS;IACT,KAAK;IACL,SAAS;IACT,KAAK;IACL,QAAQ;IACR,KAAK;IACL,SAAS;CACA,CAAA;AAEV,MAAM,oBAAoB,GAAG;IAC5B,EAAE;IACF,EAAE;IACF,EAAE;IACF,EAAE;IACF,EAAE,EAAE,kCAAkC;IACtC,GAAG;IACH,GAAG;IACH,GAAG;IACH,GAAG;IACH,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,OAAO;IACP,MAAM;IACN,MAAM;CACG,CAAA;AAEV,MAAM,qBAAqB,GAAG;IAC7B,EAAE;IACF,EAAE;IACF,EAAE;IACF,EAAE;IACF,EAAE;IACF,EAAE;IACF,EAAE;IACF,EAAE,EAAE,4BAA4B;IAChC,GAAG;IACH,GAAG;IACH,GAAG;IACH,GAAG;IACH,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;CACK,CAAA;AA+BV,SAAS,IAAI,CAAI,GAAqB,EAAE,MAAoB;IAC3D,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAE,CAAA;AAC/C,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAoB;IAC9C,uEAAuE;IACvE,gCAAgC;IAChC,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;IAElB,IAAI,CAAC,GAAG,GAAG;QAAE,OAAO,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;IAEzD,IAAI,CAAC,GAAG,GAAG;QAAE,OAAO,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAA;IAE5D,IAAI,CAAC,GAAG,IAAI;QAAE,OAAO,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAA;IAE/D,OAAO,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAA;AACtD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAClC,IAAqB,EACrB,OAA4B,EAAE;IAE9B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAA;IACzC,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,IAAI,IAAI,CAAA;IAErD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IAEtC,MAAM,MAAM,GAAG,IAAI,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAA;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,CAAA;IAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAA;IAEpD,gFAAgF;IAChF,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAC9D,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAElC,sFAAsF;IACtF,MAAM,UAAU,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;IAE9C,iFAAiF;IACjF,gFAAgF;IAChF,qFAAqF;IACrF,wEAAwE;IACxE,+FAA+F;IAC/F,8CAA8C;IAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAA;IACnC,MAAM,IAAI,GAAG,QAAQ,GAAG,CAAC,IAAI,MAAM,EAAE,GAAG,QAAQ,CAAA;IAEhD,MAAM,UAAU,GAA+B,IAAI;QAClD,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC;YACA,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACvB,CAAA;IAEH,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,UAAU,CAAC,aAAa,GAAG,UAAU,CAAC,MAAM,CAAA;IAC7C,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAA;IACtC,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,UAAU,CAAC,aAAa,GAAG,UAAU,CAAC,MAAM,CAAA;IAC7C,CAAC;IAED,IAAI,GAAW,CAAA;IAEf,IAAI,MAAM,EAAE,GAAG,SAAS,EAAE,CAAC;QAC1B,MAAM,EAAE,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAA;QACpC,UAAU,CAAC,YAAY,GAAG,EAAE,CAAA;QAC5B,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,UAAU,KAAK,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAA;IAC9G,CAAC;SAAM,CAAC;QACP,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,KAAK,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAA;IAC7F,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,CAAA;AAC5C,CAAC"}
@@ -25,8 +25,8 @@
25
25
  * is kept OUT of the default set ({@link defaultAugmentationsForCountry}) until its on-model
26
26
  * effect is measured; see the note there.
27
27
  */
28
- import { type Tokenizer } from "./tokenize.ts";
29
- import type { CanonicalRow, LabeledRow, QuarantinedRow } from "./types.ts";
28
+ import type { CanonicalRow, LabeledRow, QuarantinedRow } from "@mailwoman/corpus/types";
29
+ import { type Tokenizer } from "../utils/tokenize.ts";
30
30
  /**
31
31
  * An augmentation transforms a single row. Return `null` if the augmentation doesn't apply (e.g. accent-strip on a row
32
32
  * that has no accents; particle-strip on a US row).
@@ -200,4 +200,4 @@ export type ComposeResult = {
200
200
  * - The address row fails alignment in isolation (the underlying failure reason is propagated).
201
201
  */
202
202
  export declare function composeAdversarialRow(venue: string, address: CanonicalRow, options: ComposeAdversarialOptions): ComposeResult;
203
- //# sourceMappingURL=synthesize.d.ts.map
203
+ //# sourceMappingURL=utils.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/synthesizers/utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAWH,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAGvF,OAAO,EAAuB,KAAK,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAE1E;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,YAAY,KAAK,YAAY,GAAG,IAAI,CAAA;AA4BrE;;GAEG;AACH,eAAO,MAAM,SAAS,EAAE,YAYvB,CAAA;AAED;;GAEG;AACH,eAAO,MAAM,SAAS,EAAE,YAYvB,CAAA;AAED;;GAEG;AACH,eAAO,MAAM,UAAU,EAAE,YAKxB,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,WAAW,EAAE,YAYzB,CAAA;AAED;;;GAGG;AACH,eAAO,MAAM,WAAW,EAAE,YAazB,CAAA;AA6DD;;;;;;GAMG;AACH,eAAO,MAAM,UAAU,EAAE,YAyDxB,CAAA;AAmED;;GAEG;AACH,eAAO,MAAM,WAAW,EAAE,YAiBzB,CAAA;AAED;;GAEG;AACH,eAAO,MAAM,eAAe,EAAE,YAe7B,CAAA;AAiBD;;GAEG;AACH,eAAO,MAAM,iBAAiB,EAAE,YAuB/B,CAAA;AAED;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,YA2BnC,CAAA;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,sBAAsB,EAAE,YAwBpC,CAAA;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,kBAAkB,EAAE,YAuBhC,CAAA;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,EAAE,YAwBtC,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,EAAE,YAuBlC,CAAA;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB,EAAE,YAW9B,CAAA;AAMD;;GAEG;AACH,eAAO,MAAM,aAAa,EAAE,YAc3B,CAAA;AAMD;;GAEG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAiBtD,CAAA;AAED;;GAEG;AACH,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,YAAY,EAAE,CA2BvF;AAED;;;GAGG;AACH,wBAAiB,aAAa,CAC7B,GAAG,EAAE,YAAY,EACjB,aAAa,GAAE,SAAS,YAAY,EAAgD,GAClF,SAAS,CAAC,YAAY,CAAC,CAQzB;AAqCD;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACzC;;;;;;;;;;OAUG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,UAAU,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,GAAG,EAAE,cAAc,CAAA;CAAE,CAAA;AAE/G;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,qBAAqB,CACpC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,yBAAyB,GAChC,aAAa,CAyFf"}
@@ -26,8 +26,9 @@
26
26
  * effect is measured; see the note there.
27
27
  */
28
28
  import { US_STREET_SUFFIX_PREFERRED_ABBR, US_UNIT_DESIGNATOR_PREFERRED_ABBR, matchCase, matchLeadingDesignator, matchTrailingSuffix, } from "@mailwoman/codex/us";
29
- import { alignRow, assertSpanInvariants } from "./align.js";
30
- import { whitespaceTokenizer } from "./tokenize.js";
29
+ import { mulberry32 } from "@mailwoman/core/utils";
30
+ import { alignRow, assertSpanInvariants } from "@mailwoman/corpus/utils";
31
+ import { whitespaceTokenizer } from "../utils/tokenize.js";
31
32
  /**
32
33
  * Helper: build the augmented row with synth marker + chained source_id.
33
34
  */
@@ -169,18 +170,6 @@ function hashString(s) {
169
170
  }
170
171
  return h >>> 0;
171
172
  }
172
- /**
173
- * Mulberry32 — a tiny seeded PRNG. Same seed → same stream → reproducible typos.
174
- */
175
- function mulberry32(seed) {
176
- let a = seed >>> 0;
177
- return () => {
178
- a = (a + 0x6d_2b_79_f5) | 0;
179
- let t = Math.imul(a ^ (a >>> 15), 1 | a);
180
- t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
181
- return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296;
182
- };
183
- }
184
173
  /**
185
174
  * Inject ONE realistic typo — an adjacent-QWERTY-key substitution OR an adjacent-character transposition — into a
186
175
  * single alpha name component (street/locality/region…), teaching the model to recover from real-world misspellings
@@ -715,4 +704,4 @@ export function composeAdversarialRow(venue, address, options) {
715
704
  assertSpanInvariants(spans, composed);
716
705
  return { kind: "labeled", row: composed };
717
706
  }
718
- //# sourceMappingURL=synthesize.js.map
707
+ //# sourceMappingURL=utils.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/synthesizers/utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EACN,+BAA+B,EAC/B,iCAAiC,EACjC,SAAS,EACT,sBAAsB,EACtB,mBAAmB,GACnB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAElD,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAsB,MAAM,yBAAyB,CAAA;AAE5F,OAAO,EAAE,mBAAmB,EAAkB,MAAM,sBAAsB,CAAA;AAU1E;;GAEG;AACH,SAAS,gBAAgB,CACxB,MAAoB,EACpB,MAAc,EACd,MAAc,EACd,aAA4B;IAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,cAAc,IAAI,MAAM,CAAC,SAAS,CAAA;IAE/D,OAAO;QACN,GAAG,MAAM;QACT,GAAG,EAAE,MAAM;QACX,UAAU,EAAE,aAAa;QACzB,SAAS,EAAE,GAAG,MAAM,CAAC,SAAS,IAAI,MAAM,EAAE;QAC1C,KAAK,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE;KACzC,CAAA;AACF,CAAC;AAED,8EAA8E;AAC9E,gCAAgC;AAChC,8EAA8E;AAE9E;;GAEG;AACH,MAAM,CAAC,MAAM,SAAS,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC9C,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE;QAAE,OAAO,IAAI,CAAA;IAClD,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAA;IACnC,MAAM,YAAY,GAAkB,EAAE,CAAA;IAEtC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC;YACP,YAAY,CAAC,CAAiB,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAA;QAClD,CAAC;IACF,CAAC;IAED,OAAO,gBAAgB,CAAC,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,YAAY,CAAC,CAAA;AAChE,CAAC,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,SAAS,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC9C,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE;QAAE,OAAO,IAAI,CAAA;IAClD,MAAM,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAA;IACrC,MAAM,cAAc,GAAkB,EAAE,CAAA;IAExC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC;YACP,cAAc,CAAC,CAAiB,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAA;QACpD,CAAC;IACF,CAAC;IAED,OAAO,gBAAgB,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;AACpE,CAAC,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IACvC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IAEzE,OAAO,gBAAgB,CAAC,GAAG,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;AAC3E,CAAC,CAAA;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,WAAW,GAAiB,CAAC,GAAG,EAAE,EAAE;IAChD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IACnC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IAC5C,MAAM,aAAa,GAAkB,EAAE,CAAA;IAEvC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC;YACP,aAAa,CAAC,CAAiB,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QAC3D,CAAC;IACF,CAAC;IAED,OAAO,gBAAgB,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACpE,CAAC,CAAA;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAiB,CAAC,GAAG,EAAE,EAAE;IAChD,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAEtC,IAAI,QAAQ,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,aAAa,GAAkB,EAAE,CAAA;IAEvC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC;YACP,aAAa,CAAC,CAAiB,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,CAAA;QACnD,CAAC;IACF,CAAC;IAED,OAAO,gBAAgB,CAAC,GAAG,EAAE,cAAc,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAA;AACtE,CAAC,CAAA;AAED,SAAS,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;AACpD,CAAC;AAED,8EAA8E;AAC9E,kGAAkG;AAClG,kGAAkG;AAClG,+DAA+D;AAE/D;;GAEG;AACH,MAAM,gBAAgB,GAA2B;IAChD,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,OAAO;IACV,CAAC,EAAE,KAAK;IACR,CAAC,EAAE,KAAK;IACR,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,IAAI;IACP,CAAC,EAAE,IAAI;IACP,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,KAAK;CACR,CAAA;AAED;;GAEG;AACH,MAAM,UAAU,GAAG,2BAA2B,CAAA;AAE9C;;GAEG;AACH,SAAS,UAAU,CAAC,CAAS;IAC5B,IAAI,CAAC,GAAG,IAAI,CAAA;IAEZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAC3C,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,CAAA;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,UAAU,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC/C,MAAM,GAAG,GAAG,UAAU,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,SAAS,OAAO,CAAC,CAAC,CAAA;IAE3D,iGAAiG;IACjG,mGAAmG;IACnG,kGAAkG;IAClG,qGAAqG;IACrG,MAAM,MAAM,GAAG,CAAC,MAAc,EAAU,EAAE;QACzC,IAAI,CAAC,GAAG,CAAC,CAAA;QAET,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9F,CAAC,EAAE,CAAA;QACJ,CAAC;QAED,OAAO,CAAC,CAAA;IACT,CAAC,CAAA;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,OAAO,CAAa,CAAA;IAExE,MAAM,QAAQ,GAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAmC,CAAC,MAAM,CACxF,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CACtG,CAAA;IAED,IAAI,CAAC,QAAQ,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACjC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAE,CAAA;IACnE,0GAA0G;IAC1G,MAAM,SAAS,GAAa,EAAE,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE;QACxC,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YAC9B,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAClB,CAAC;IAEF,IAAI,CAAC,SAAS,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IAClC,MAAM,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAE,CAAA;IAC1D,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;IACpB,IAAI,KAAa,CAAA;IAEjB,IAAI,GAAG,EAAE,GAAG,GAAG,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;QAE1B,IAAI,EAAE,KAAK,IAAI;YAAE,OAAO,IAAI,CAAA,CAAC,qCAAqC;QAClE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,GAAG,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAC3D,CAAC;SAAM,CAAC;QACP,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,EAAE,CAAA;QAC9B,MAAM,GAAG,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAA;QAEnC,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAA;QACrB,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAE,CAAA;QAChD,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1F,CAAC;IAED,IAAI,KAAK,KAAK,KAAK;QAAE,OAAO,IAAI,CAAA;IAChC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IAE5C,+DAA+D;IAC/D,OAAO,gBAAgB,CAAC,GAAG,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;AACzF,CAAC,CAAA;AAED,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E;;GAEG;AACH,MAAM,kBAAkB,GAA2B;IAClD,OAAO,EAAE,IAAI;IACb,MAAM,EAAE,IAAI;IACZ,OAAO,EAAE,IAAI;IACb,QAAQ,EAAE,IAAI;IACd,UAAU,EAAE,IAAI;IAChB,QAAQ,EAAE,IAAI;IACd,WAAW,EAAE,IAAI;IACjB,QAAQ,EAAE,IAAI;IACd,OAAO,EAAE,IAAI;IACb,OAAO,EAAE,IAAI;IACb,MAAM,EAAE,IAAI;IACZ,KAAK,EAAE,IAAI;IACX,QAAQ,EAAE,IAAI;IACd,OAAO,EAAE,IAAI;IACb,IAAI,EAAE,IAAI;IACV,MAAM,EAAE,IAAI;IACZ,QAAQ,EAAE,IAAI;IACd,SAAS,EAAE,IAAI;IACf,KAAK,EAAE,IAAI;IACX,QAAQ,EAAE,IAAI;IACd,aAAa,EAAE,IAAI;IACnB,QAAQ,EAAE,IAAI;IACd,SAAS,EAAE,IAAI;IACf,WAAW,EAAE,IAAI;IACjB,QAAQ,EAAE,IAAI;IACd,OAAO,EAAE,IAAI;IACb,QAAQ,EAAE,IAAI;IACd,MAAM,EAAE,IAAI;IACZ,eAAe,EAAE,IAAI;IACrB,YAAY,EAAE,IAAI;IAClB,YAAY,EAAE,IAAI;IAClB,UAAU,EAAE,IAAI;IAChB,gBAAgB,EAAE,IAAI;IACtB,cAAc,EAAE,IAAI;IACpB,IAAI,EAAE,IAAI;IACV,QAAQ,EAAE,IAAI;IACd,MAAM,EAAE,IAAI;IACZ,YAAY,EAAE,IAAI;IAClB,cAAc,EAAE,IAAI;IACpB,gBAAgB,EAAE,IAAI;IACtB,cAAc,EAAE,IAAI;IACpB,SAAS,EAAE,IAAI;IACf,KAAK,EAAE,IAAI;IACX,IAAI,EAAE,IAAI;IACV,OAAO,EAAE,IAAI;IACb,QAAQ,EAAE,IAAI;IACd,UAAU,EAAE,IAAI;IAChB,eAAe,EAAE,IAAI;IACrB,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,IAAI;IACb,sBAAsB,EAAE,IAAI;CAC5B,CAAA;AAED,MAAM,kBAAkB,GAA2B,MAAM,CAAC,WAAW,CACpE,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAC1D,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,WAAW,GAAiB,CAAC,GAAG,EAAE,EAAE;IAChD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IAEpC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAEvC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,wFAAwF;IACxF,wCAAwC;IACxC,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,MAAM,KAAK,EAAE,GAAG,CAAC,CAAA;IAE7C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAClC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,MAAM,KAAK,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;IAC3E,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;IAExE,OAAO,gBAAgB,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACpE,CAAC,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAiB,CAAC,GAAG,EAAE,EAAE;IACpD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IAEpC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAEvC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,MAAM,KAAK,EAAE,GAAG,CAAC,CAAA;IAE7C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAClC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,MAAM,KAAK,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;IAC3E,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;IAExE,OAAO,gBAAgB,CAAC,GAAG,EAAE,kBAAkB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACxE,CAAC,CAAA;AAED,MAAM,wBAAwB,GAA2B;IACxD,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,SAAS,EAAE,IAAI;IACf,SAAS,EAAE,IAAI;IACf,SAAS,EAAE,IAAI;IACf,SAAS,EAAE,IAAI;CACf,CAAA;AAED,MAAM,wBAAwB,GAA2B,MAAM,CAAC,WAAW,CAC1E,MAAM,CAAC,OAAO,CAAC,wBAAwB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAChE,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAiB,CAAC,GAAG,EAAE,EAAE;IACtD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,WAAW,GAAmB,CAAC,QAAQ,EAAE,eAAe,EAAE,eAAe,CAAC,CAAA;IAChF,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,MAAM,GAAG,GAAG,CAAC,GAAG,CAAA;IACpB,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IAE1D,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAA;QAE5B,IAAI,CAAC,CAAC;YAAE,SAAQ;QAChB,MAAM,QAAQ,GAAG,CAAC,CAAC,UAAU,CAAC,4BAA4B,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;QAEpG,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;YACpB,aAAa,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAA;YAChF,OAAO,GAAG,IAAI,CAAA;QACf,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzB,OAAO,gBAAgB,CAAC,GAAG,EAAE,oBAAoB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AAC1E,CAAC,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC1D,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,WAAW,GAAmB,CAAC,QAAQ,EAAE,eAAe,EAAE,eAAe,CAAC,CAAA;IAChF,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,MAAM,GAAG,GAAG,CAAC,GAAG,CAAA;IACpB,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IAE1D,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAA;QAE5B,IAAI,CAAC,CAAC;YAAE,SAAQ;QAEhB,MAAM,QAAQ,GAAG,CAAC,CAAC,UAAU,CAC5B,sEAAsE,EACtE,CAAC,CAAC,EAAE,EAAE,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,CACvC,CAAA;QAED,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;YACpB,aAAa,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAA;YAChF,OAAO,GAAG,IAAI,CAAA;QACf,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzB,OAAO,gBAAgB,CAAC,GAAG,EAAE,wBAAwB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AAC9E,CAAC,CAAA;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC3D,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IAEpC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,MAAM,KAAK,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAA;IAEzC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,MAAM,SAAS,GAAG,+BAA+B,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAClE,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;IAElD,IAAI,MAAM,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,MAAM,EAAE,CAAA;IAElF,IAAI,SAAS,KAAK,MAAM;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,CAAA;IAC7E,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;IAE7F,IAAI,MAAM,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAEnC,OAAO,gBAAgB,CAAC,GAAG,EAAE,6BAA6B,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACnF,CAAC,CAAA;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAiB,CAAC,GAAG,EAAE,EAAE;IACvD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IAEpC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,MAAM,KAAK,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAA;IAEzC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;IAExD,IAAI,MAAM,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,MAAM,EAAE,CAAA;IAElF,IAAI,SAAS,KAAK,MAAM;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,CAAA;IAC7E,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;IAE7F,IAAI,MAAM,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAEnC,OAAO,gBAAgB,CAAC,GAAG,EAAE,yBAAyB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AAC/E,CAAC,CAAA;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC7D,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,CAAA;IAEhC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;IAE1C,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,MAAM,SAAS,GAAG,iCAAiC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IACpE,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;IAElD,IAAI,MAAM,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAA;IAE9D,IAAI,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IAEjC,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;IACzE,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAA;IAEzF,IAAI,MAAM,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAEnC,OAAO,gBAAgB,CAAC,GAAG,EAAE,+BAA+B,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACrF,CAAC,CAAA;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAiB,CAAC,GAAG,EAAE,EAAE;IACzD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,CAAA;IAEhC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;IAE1C,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;IAExD,IAAI,MAAM,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAA;IAE9D,IAAI,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IAEjC,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;IACzE,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAA;IAEzF,IAAI,MAAM,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAEnC,OAAO,gBAAgB,CAAC,GAAG,EAAE,2BAA2B,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACjF,CAAC,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAiB,CAAC,GAAG,EAAE,EAAE;IACrD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAA;IAExC,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAA;IAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;IACxC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IAEhD,IAAI,MAAM,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAEnC,OAAO,gBAAgB,CAAC,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;AACrG,CAAC,CAAA;AAED,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAiB,CAAC,GAAG,EAAE,EAAE;IAClD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,UAAU,CAAC,sBAAsB,CAAA;IAEtD,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAA;IAC1B,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IAC1D,OAAO,aAAa,CAAC,sBAAsB,CAAA;IAC3C,+DAA+D;IAC/D,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,OAAO,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAE9D,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAClC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IAEtE,OAAO,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACtE,CAAC,CAAA;AAED,8EAA8E;AAC9E,8BAA8B;AAC9B,8EAA8E;AAE9E;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAiC;IAC1D,YAAY,EAAE,SAAS;IACvB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,UAAU;IACzB,cAAc,EAAE,WAAW;IAC3B,cAAc,EAAE,WAAW;IAC3B,cAAc,EAAE,WAAW;IAC3B,kBAAkB,EAAE,eAAe;IACnC,oBAAoB,EAAE,iBAAiB;IACvC,wBAAwB,EAAE,qBAAqB;IAC/C,6BAA6B,EAAE,sBAAsB;IACrD,yBAAyB,EAAE,kBAAkB;IAC7C,+BAA+B,EAAE,wBAAwB;IACzD,2BAA2B,EAAE,oBAAoB;IACjD,qBAAqB,EAAE,gBAAgB;IACvC,gBAAgB,EAAE,aAAa;IAC/B,aAAa,EAAE,UAAU;CACzB,CAAA;AAED;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAAC,OAAe;IAC7D,6FAA6F;IAC7F,kGAAkG;IAClG,gGAAgG;IAChG,iGAAiG;IACjG,4FAA4F;IAC5F,MAAM,SAAS,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAA;IAEjE,QAAQ,OAAO,EAAE,CAAC;QACjB,KAAK,IAAI;YACR,OAAO;gBACN,GAAG,SAAS;gBACZ,WAAW;gBACX,eAAe;gBACf,iBAAiB;gBACjB,qBAAqB;gBACrB,sBAAsB;gBACtB,kBAAkB;gBAClB,wBAAwB;gBACxB,oBAAoB;gBACpB,gBAAgB;aAChB,CAAA;QACF,KAAK,IAAI;YACR,OAAO,CAAC,GAAG,SAAS,EAAE,WAAW,EAAE,aAAa,CAAC,CAAA;QAClD;YACC,OAAO,SAAS,CAAA;IAClB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,SAAS,CAAC,CAAC,aAAa,CAC7B,GAAiB,EACjB,gBAAyC,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC;IAEpF,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAA;QAEpB,IAAI,GAAG,EAAE,CAAC;YACT,MAAM,GAAG,CAAA;QACV,CAAC;IACF,CAAC;AACF,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC7B,OAAO,CAAC,CAAC,UAAU,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAA;AACnD,CAAC;AAoED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,UAAU,qBAAqB,CACpC,KAAa,EACb,OAAqB,EACrB,OAAkC;IAElC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAA;IAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,mBAAmB,EAAE,CAAA;IAE5D,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IAEjC,IAAI,CAAC,YAAY,EAAE,CAAC;QACnB,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,CAAA;IAC7E,CAAC;IAED,6FAA6F;IAC7F,+EAA+E;IAC/E,IAAI,YAAY,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,YAAY,EAAE,CAAC;QACpD,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,EAAE,CAAA;IAC/E,CAAC;IAED,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;IAEvD,IAAI,cAAc,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QACvC,kFAAkF;QAClF,kFAAkF;QAClF,+BAA+B;QAC/B,OAAO;YACN,IAAI,EAAE,aAAa;YACnB,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,mBAAmB,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE;SAC7E,CAAA;IACF,CAAC;IAED,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA;IAEpD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QACzB,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,EAAE,CAAA;IACjF,CAAC;IAED,MAAM,WAAW,GAAe,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAA;IAE5F,MAAM,MAAM,GAAa,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAC1F,MAAM,MAAM,GAAe,CAAC,GAAG,WAAW,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAEzE,MAAM,WAAW,GAAG,GAAG,YAAY,GAAG,SAAS,GAAG,OAAO,CAAC,GAAG,EAAE,CAAA;IAE/D,MAAM,kBAAkB,GAAG;QAC1B,KAAK,EAAE,YAAY;QACnB,GAAG,OAAO,CAAC,UAAU;KACrB,CAAA;IAED,8FAA8F;IAC9F,gGAAgG;IAChG,4FAA4F;IAC5F,2FAA2F;IAC3F,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,cAAc,CAAC,GAAG,CAAA;IAEhG,IAAI,UAAU,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAClF,MAAM,IAAI,KAAK,CACd,iFAAiF;YAChF,WAAW,OAAO,CAAC,MAAM,eAAe,OAAO,CAAC,SAAS,kCAAkC,CAC5F,CAAA;IACF,CAAC;IAED,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAA;IAErD,MAAM,KAAK,GAAoB;QAC9B,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,CAAC,MAAM,EAAE;QACpD,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAE,GAAG,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAE,GAAG,MAAM,EAAE,CAAC,CAAC;KAClG,CAAA;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,EAAE,cAAc,IAAI,OAAO,CAAC,SAAS,CAAA;IACvE,MAAM,MAAM,GAAG,WAAW,OAAO,CAAC,OAAO,EAAE,CAAA;IAE3C,MAAM,QAAQ,GAAe;QAC5B,GAAG,EAAE,WAAW;QAChB,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,EAAE;QAC3C,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,KAAK,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE;QAC/C,MAAM;QACN,MAAM;QACN,WAAW,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QACtC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAClC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KAClC,CAAA;IAED,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;IAErC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAA;AAC1C,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"align-shard.d.ts","sourceRoot":"","sources":["../../../src/tools/align-shard.ts"],"names":[],"mappings":";AA+BA,MAAM,WAAW,iBAAiB;IACjC,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,aAAa,EAAE,MAAM,CAAA;CACrB;AAED,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAkChF"}
1
+ {"version":3,"file":"align-shard.d.ts","sourceRoot":"","sources":["../../../src/tools/align-shard.ts"],"names":[],"mappings":";AA+BA,MAAM,WAAW,iBAAiB;IACjC,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,aAAa,EAAE,MAAM,CAAA;CACrB;AAED,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CA+BhF"}
@@ -25,19 +25,15 @@ import { createWriteStream } from "node:fs";
25
25
  * --output /tmp/kryptonite-labeled.jsonl\
26
26
  * --corpus-version 0.5.0
27
27
  */
28
- import { alignRow } from "@mailwoman/corpus";
29
- import { TextSpliterator } from "spliterator";
28
+ import { alignRow } from "@mailwoman/corpus/utils";
29
+ import { JSONSpliterator } from "spliterator";
30
30
  export async function alignCanonicalShard(args) {
31
- // Read phase only — the write path stays on createWriteStream. TextSpliterator + JSON.parse keeps the
32
- // original tolerance: the `!line.trim()` guard skips blank lines and a trailing CR is valid JSON whitespace.
31
+ // Read phase only — the write path stays on createWriteStream.
33
32
  const outStream = createWriteStream(args.output, { encoding: "utf8" });
34
33
  let labeled = 0;
35
34
  let quarantined = 0;
36
35
  const quarantineReasons = {};
37
- for await (const line of TextSpliterator.fromAsync(args.input)) {
38
- if (!line.trim())
39
- continue;
40
- const canonical = JSON.parse(line);
36
+ for await (const canonical of JSONSpliterator.fromAsync(args.input)) {
41
37
  // Stamp the target corpus version so the emitted row's provenance matches the run it joins.
42
38
  canonical.corpus_version = args.corpusVersion;
43
39
  const result = alignRow(canonical);
@@ -1 +1 @@
1
- {"version":3,"file":"align-shard.js","sourceRoot":"","sources":["../../../src/tools/align-shard.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAA;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAQ7C,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAuB;IAChE,sGAAsG;IACtG,6GAA6G;IAC7G,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACtE,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,iBAAiB,GAA2B,EAAE,CAAA;IAEpD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAChE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAQ;QAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAmC,CAAA;QACpE,4FAA4F;QAC5F,SAAS,CAAC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAA;QAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;QAElC,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;YAElD,OAAO,EAAE,CAAA;QACV,CAAC;aAAM,CAAC;YACP,WAAW,EAAE,CAAA;YACb,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,SAAS,CAAA;YACxC,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QAC/B,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACnB,CAAC,CAAC,CAAA;IAEF,OAAO,CAAC,KAAK,CACZ,0BAA0B,OAAO,aAAa,WAAW,kBAAkB,IAAI,CAAC,MAAM,IAAI;QACzF,yBAAyB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAC7D,CAAA;AACF,CAAC"}
1
+ {"version":3,"file":"align-shard.js","sourceRoot":"","sources":["../../../src/tools/align-shard.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAA;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAQ7C,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAuB;IAChE,+DAA+D;IAC/D,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACtE,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,iBAAiB,GAA2B,EAAE,CAAA;IAEpD,IAAI,KAAK,EAAE,MAAM,SAAS,IAAI,eAAe,CAAC,SAAS,CAAiC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrG,4FAA4F;QAC5F,SAAS,CAAC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAA;QAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;QAElC,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;YAElD,OAAO,EAAE,CAAA;QACV,CAAC;aAAM,CAAC;YACP,WAAW,EAAE,CAAA;YACb,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,SAAS,CAAA;YACxC,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QAC/B,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACnB,CAAC,CAAC,CAAA;IAEF,OAAO,CAAC,KAAK,CACZ,0BAA0B,OAAO,aAAa,WAAW,kBAAkB,IAAI,CAAC,MAAM,IAAI;QACzF,yBAAyB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAC7D,CAAA;AACF,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"audit.d.ts","sourceRoot":"","sources":["../../../src/tools/audit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAgBH,MAAM,WAAW,SAAS;IACzB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACzB;AA4VD,wBAAgB,KAAK,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI,CAS3C"}
1
+ {"version":3,"file":"audit.d.ts","sourceRoot":"","sources":["../../../src/tools/audit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAmBH,MAAM,WAAW,SAAS;IACzB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACzB;AA0VD,wBAAgB,KAAK,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI,CAS3C"}
@@ -16,6 +16,8 @@
16
16
  */
17
17
  import { existsSync, readFileSync, readdirSync } from "node:fs";
18
18
  import { basename, join } from "node:path";
19
+ import { parseJSONStrict } from "@mailwoman/core/objects";
20
+ import { TextSpliterator } from "spliterator";
19
21
  /**
20
22
  * Share of a shard one source may hold before the mix is flagged as dominated by it.
21
23
  */
@@ -33,12 +35,10 @@ const MAX_TOP_TO_RUNNER_UP_RATIO = 1.5;
33
35
  function parseConfig(configPath) {
34
36
  if (!existsSync(configPath))
35
37
  return null;
36
- const text = readFileSync(configPath, "utf8");
37
- const lines = text.split("\n");
38
38
  const weights = {};
39
39
  let inBlock = false;
40
40
  let blockIndent = -1;
41
- for (const raw of lines) {
41
+ for (const raw of TextSpliterator.from(readFileSync(configPath))) {
42
42
  const sourceWeightsMatch = raw.match(/^([\t ]*)source_weights:\s*$/);
43
43
  if (sourceWeightsMatch) {
44
44
  inBlock = true;
@@ -160,7 +160,7 @@ function manifestScan(corpusDir, knownPrefixes) {
160
160
  const manifestPath = join(corpusDir, "MANIFEST.json");
161
161
  if (!existsSync(manifestPath))
162
162
  return null;
163
- const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
163
+ const manifest = parseJSONStrict(readFileSync(manifestPath, "utf8"));
164
164
  if (!Array.isArray(manifest.shards))
165
165
  return null;
166
166
  const bySplit = {};
@@ -1 +1 @@
1
- {"version":3,"file":"audit.js","sourceRoot":"","sources":["../../../src/tools/audit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAC/D,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAE1C;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;;GAGG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAA;AAgCtC;;;;GAIG;AACH,SAAS,WAAW,CAAC,UAAkB;IACtC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,IAAI,CAAA;IACxC,MAAM,IAAI,GAAG,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC9B,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,WAAW,GAAG,CAAC,CAAC,CAAA;IAEpB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,kBAAkB,GAAG,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;QAEpE,IAAI,kBAAkB,EAAE,CAAC;YACxB,OAAO,GAAG,IAAI,CAAA;YACd,WAAW,GAAG,kBAAkB,CAAC,CAAC,CAAE,CAAC,MAAM,CAAA;YAE3C,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,iCAAiC;QACjC,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAQ;QACtC,6FAA6F;QAC7F,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QAE9C,IAAI,MAAM,IAAI,WAAW,EAAE,CAAC;YAC3B,OAAO,GAAG,KAAK,CAAA;YAEf,SAAQ;QACT,CAAC;QAED,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;QAElD,IAAI,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAA;QAC1C,CAAC;IACF,CAAC;IAED,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,CAAA;AAClC,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,SAAiB,EAAE,WAAmB;IACzD,MAAM,KAAK,GAAe,EAAE,OAAO,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAA;IAExE,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;QAEvC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,SAAQ;QAEnC,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC;aACjC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;aACrC,QAAQ,EAAE,CAAA;QAEZ,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAA;QAChC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAA;QACvE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,WAAW,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;QACnF,MAAM,QAAQ,GAA2B,EAAE,CAAA;QAE3C,yFAAyF;QACzF,2FAA2F;QAC3F,mFAAmF;QACnF,0CAA0C;QAC1C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,uBAAuB,CAAC,CAAC,CAAC,CAAA;YAC3C,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACnD,CAAC;QAED,wCAAwC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QAExD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAE,GAAG,KAAK,CAAC,CAAA;QAC/C,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAA;QAC/B,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAA;IAClC,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAgB;IAChD,8FAA8F;IAC9F,8FAA8F;IAC9F,wEAAwE;IACxE,MAAM,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAEjE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IAExC,OAAO,WAAW,CAAA;AACnB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,qBAAqB,GAA0B;IACpD,WAAW;IACX,gBAAgB;IAChB,KAAK;IACL,OAAO;IACP,WAAW;IACX,aAAa;IACb,iBAAiB;IACjB,gBAAgB;IAChB,sBAAsB;IACtB,mBAAmB;IACnB,mBAAmB;IACnB,eAAe;IACf,4DAA4D;IAC5D,qBAAqB;IACrB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;CACxB,CAAA;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,QAAgB,EAAE,aAAgC;IACvE,oEAAoE;IACpE,MAAM,MAAM,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IAEzE,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC7B,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,QAAQ,KAAK,MAAM;YAAE,OAAO,MAAM,CAAA;IAC5E,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,YAAY,CAAC,SAAiB,EAAE,aAAgC;IACxE,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IAErD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAA;IAE1C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAE7D,CAAA;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAA;IAChD,MAAM,OAAO,GAA2C,EAAE,CAAA;IAE1D,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;QACzB,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,eAAe,IAAI,EAAE,EAAE,aAAa,CAAC,CAAA;QACpF,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QACrB,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;IACrD,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE7G,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,CAAA;AAC1D,CAAC;AAWD,SAAS,cAAc,CAAC,KAA6B,EAAE,OAA+B;IACrF,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IACnE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAC5E,MAAM,IAAI,GAAe,EAAE,CAAA;IAC3B,+FAA+F;IAC/F,kCAAkC;IAClC,MAAM,aAAa,GAA4B,EAAE,CAAA;IAEjD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;QAC3B,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAC5D,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,iBAAiB,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IAEtE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAA;QAClC,MAAM,SAAS,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;QAEvG,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,GAAG;YACX,MAAM;YACN,QAAQ,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM;YACN,kBAAkB,EAAE,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG;SACnE,CAAC,CAAA;IACH,CAAC;IAED,wFAAwF;IACxF,8FAA8F;IAC9F,4FAA4F;IAC5F,2EAA2E;IAC3E,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,kBAAkB,KAAK,QAAQ,CAE1E,CAAA;IAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,GAAG,CAAC,CAAC,kBAAkB,CAAC,CAAA;IAEnE,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;QACvB,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,kBAAkB,IAAI,CAAC,CAAA;QAEhD,IACC,GAAG,CAAC,kBAAkB,GAAG,qBAAqB;YAC9C,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,kBAAkB,GAAG,IAAI,GAAG,0BAA0B,CAAC,EACvE,CAAC;YACF,GAAG,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IAExC,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,SAAS,SAAS,CAAC,CAAe;IACjC,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,GAAG,CAAA;IAEzB,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;AAClC,CAAC;AAED,SAAS,WAAW,CAAC,SAAiB,EAAE,UAA8B,EAAE,KAAiB,EAAE,IAAgB;IAC1G,OAAO,CAAC,GAAG,CAAC,oBAAoB,SAAS,EAAE,CAAC,CAAA;IAE5C,IAAI,UAAU,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,CAAC,GAAG,CACV,kBAAkB,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,UAAU,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5H,CAAA;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAEf,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEzC,IAAI,UAAU,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAElE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAA;QAC3C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEf,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAA;QAC1E,MAAM,MAAM,GAAG,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,CAAC,KAAe,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAExF,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAA;QAC5B,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAErD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CACV,MAAM,CAAC;gBACN,GAAG,CAAC,MAAM;gBACV,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;gBAClB,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC;gBACvB,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;gBAC5D,SAAS,CAAC,GAAG,CAAC,kBAAkB,CAAC;aACjC,CAAC,CACF,CAAA;QACF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEf,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAA;QAEpE,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,SAAS,CAAC,gBAAgB,CAAA;YACzC,MAAM,SAAS,GAAG,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;YAEhE,OAAO,CAAC,KAAK,CACZ,oBAAoB,SAAS,CAAC,MAAM,iBAAiB,SAAS,CAAC,SAAS,CAAC,kBAAkB,CAAC,GAAG;gBAC9F,qBAAqB,SAAS,uBAAuB;gBACrD,2EAA2E;gBAC3E,4DAA4D,CAC7D,CAAA;QACF,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,uFAAuF,CAAC,CAAA;QACrG,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAE3E,IAAI,cAAc,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YACzC,OAAO,CAAC,KAAK,CACZ,oEAAoE;gBACnE,4BAA4B,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC7E,CAAA;QACF,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAA;QAExF,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,CACZ,6DAA6D;gBAC5D,oBAAoB,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpE,CAAA;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,IAAe;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACpE,8FAA8F;IAC9F,0EAA0E;IAC1E,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,qBAAqB,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACtG,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,gBAAgB,IAAI,GAAG,CAAC,CAAA;IAChH,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IAC/C,MAAM,IAAI,GAAG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAA;IACpE,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,CAAA;AAC1D,CAAC"}
1
+ {"version":3,"file":"audit.js","sourceRoot":"","sources":["../../../src/tools/audit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAC/D,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAE1C,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;;GAGG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAA;AAgCtC;;;;GAIG;AACH,SAAS,WAAW,CAAC,UAAkB;IACtC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,IAAI,CAAA;IACxC,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,WAAW,GAAG,CAAC,CAAC,CAAA;IAEpB,KAAK,MAAM,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAClE,MAAM,kBAAkB,GAAG,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;QAEpE,IAAI,kBAAkB,EAAE,CAAC;YACxB,OAAO,GAAG,IAAI,CAAA;YACd,WAAW,GAAG,kBAAkB,CAAC,CAAC,CAAE,CAAC,MAAM,CAAA;YAE3C,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,iCAAiC;QACjC,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAQ;QACtC,6FAA6F;QAC7F,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QAE9C,IAAI,MAAM,IAAI,WAAW,EAAE,CAAC;YAC3B,OAAO,GAAG,KAAK,CAAA;YAEf,SAAQ;QACT,CAAC;QAED,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;QAElD,IAAI,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAA;QAC1C,CAAC;IACF,CAAC;IAED,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,CAAA;AAClC,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,SAAiB,EAAE,WAAmB;IACzD,MAAM,KAAK,GAAe,EAAE,OAAO,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAA;IAExE,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;QAEvC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,SAAQ;QAEnC,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC;aACjC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;aACrC,QAAQ,EAAE,CAAA;QAEZ,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAA;QAChC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAA;QACvE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,WAAW,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;QACnF,MAAM,QAAQ,GAA2B,EAAE,CAAA;QAE3C,yFAAyF;QACzF,2FAA2F;QAC3F,mFAAmF;QACnF,0CAA0C;QAC1C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,uBAAuB,CAAC,CAAC,CAAC,CAAA;YAC3C,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACnD,CAAC;QAED,wCAAwC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QAExD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAE,GAAG,KAAK,CAAC,CAAA;QAC/C,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAA;QAC/B,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAA;IAClC,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAgB;IAChD,8FAA8F;IAC9F,8FAA8F;IAC9F,wEAAwE;IACxE,MAAM,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAEjE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IAExC,OAAO,WAAW,CAAA;AACnB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,qBAAqB,GAA0B;IACpD,WAAW;IACX,gBAAgB;IAChB,KAAK;IACL,OAAO;IACP,WAAW;IACX,aAAa;IACb,iBAAiB;IACjB,gBAAgB;IAChB,sBAAsB;IACtB,mBAAmB;IACnB,mBAAmB;IACnB,eAAe;IACf,4DAA4D;IAC5D,qBAAqB;IACrB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;CACxB,CAAA;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,QAAgB,EAAE,aAAgC;IACvE,oEAAoE;IACpE,MAAM,MAAM,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IAEzE,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC7B,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,QAAQ,KAAK,MAAM;YAAE,OAAO,MAAM,CAAA;IAC5E,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,YAAY,CAAC,SAAiB,EAAE,aAAgC;IACxE,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IAErD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAA;IAE1C,MAAM,QAAQ,GAAG,eAAe,CAE7B,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IAEtC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAA;IAChD,MAAM,OAAO,GAA2C,EAAE,CAAA;IAE1D,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;QACzB,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,eAAe,IAAI,EAAE,EAAE,aAAa,CAAC,CAAA;QACpF,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QACrB,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;IACrD,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE7G,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,CAAA;AAC1D,CAAC;AAWD,SAAS,cAAc,CAAC,KAA6B,EAAE,OAA+B;IACrF,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IACnE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAC5E,MAAM,IAAI,GAAe,EAAE,CAAA;IAC3B,+FAA+F;IAC/F,kCAAkC;IAClC,MAAM,aAAa,GAA4B,EAAE,CAAA;IAEjD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;QAC3B,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAC5D,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,iBAAiB,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IAEtE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAA;QAClC,MAAM,SAAS,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;QAEvG,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,GAAG;YACX,MAAM;YACN,QAAQ,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM;YACN,kBAAkB,EAAE,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG;SACnE,CAAC,CAAA;IACH,CAAC;IAED,wFAAwF;IACxF,8FAA8F;IAC9F,4FAA4F;IAC5F,2EAA2E;IAC3E,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,kBAAkB,KAAK,QAAQ,CAE1E,CAAA;IAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,GAAG,CAAC,CAAC,kBAAkB,CAAC,CAAA;IAEnE,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;QACvB,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,kBAAkB,IAAI,CAAC,CAAA;QAEhD,IACC,GAAG,CAAC,kBAAkB,GAAG,qBAAqB;YAC9C,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,kBAAkB,GAAG,IAAI,GAAG,0BAA0B,CAAC,EACvE,CAAC;YACF,GAAG,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IAExC,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,SAAS,SAAS,CAAC,CAAe;IACjC,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,GAAG,CAAA;IAEzB,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;AAClC,CAAC;AAED,SAAS,WAAW,CAAC,SAAiB,EAAE,UAA8B,EAAE,KAAiB,EAAE,IAAgB;IAC1G,OAAO,CAAC,GAAG,CAAC,oBAAoB,SAAS,EAAE,CAAC,CAAA;IAE5C,IAAI,UAAU,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,CAAC,GAAG,CACV,kBAAkB,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,UAAU,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5H,CAAA;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAEf,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEzC,IAAI,UAAU,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAElE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAA;QAC3C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEf,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAA;QAC1E,MAAM,MAAM,GAAG,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,CAAC,KAAe,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAExF,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAA;QAC5B,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAErD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CACV,MAAM,CAAC;gBACN,GAAG,CAAC,MAAM;gBACV,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;gBAClB,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC;gBACvB,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;gBAC5D,SAAS,CAAC,GAAG,CAAC,kBAAkB,CAAC;aACjC,CAAC,CACF,CAAA;QACF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEf,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAA;QAEpE,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,SAAS,CAAC,gBAAgB,CAAA;YACzC,MAAM,SAAS,GAAG,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;YAEhE,OAAO,CAAC,KAAK,CACZ,oBAAoB,SAAS,CAAC,MAAM,iBAAiB,SAAS,CAAC,SAAS,CAAC,kBAAkB,CAAC,GAAG;gBAC9F,qBAAqB,SAAS,uBAAuB;gBACrD,2EAA2E;gBAC3E,4DAA4D,CAC7D,CAAA;QACF,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,uFAAuF,CAAC,CAAA;QACrG,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAE3E,IAAI,cAAc,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YACzC,OAAO,CAAC,KAAK,CACZ,oEAAoE;gBACnE,4BAA4B,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC7E,CAAA;QACF,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAA;QAExF,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,CACZ,6DAA6D;gBAC5D,oBAAoB,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpE,CAAA;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,IAAe;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACpE,8FAA8F;IAC9F,0EAA0E;IAC1E,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,qBAAqB,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACtG,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,gBAAgB,IAAI,GAAG,CAAC,CAAA;IAChH,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IAC/C,MAAM,IAAI,GAAG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAA;IACpE,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,CAAA;AAC1D,CAAC"}
@@ -32,7 +32,7 @@
32
32
  *
33
33
  * For a quick local-corpus baseline (limited but useful for linter testing): node
34
34
  * scripts/build-corpus-stats.ts\
35
- * --shards /mnt/playpen/mailwoman-data/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
35
+ * --shards $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
36
36
  * --output /tmp/corpus-stats-local.json
37
37
  */
38
38
  export interface CorpusStatsOptions {
@@ -40,5 +40,5 @@ export interface CorpusStatsOptions {
40
40
  outputPath: string;
41
41
  limitPerShard?: number;
42
42
  }
43
- export declare function buildCorpusStats(args: CorpusStatsOptions): void;
43
+ export declare function buildCorpusStats(args: CorpusStatsOptions): Promise<void>;
44
44
  //# sourceMappingURL=corpus-stats.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"corpus-stats.d.ts","sourceRoot":"","sources":["../../../src/tools/corpus-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AASH,MAAM,WAAW,kBAAkB;IAClC,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AA+CD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI,CA+F/D"}
1
+ {"version":3,"file":"corpus-stats.d.ts","sourceRoot":"","sources":["../../../src/tools/corpus-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAUH,MAAM,WAAW,kBAAkB;IAClC,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAuCD,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CA8F9E"}
@@ -32,12 +32,12 @@
32
32
  *
33
33
  * For a quick local-corpus baseline (limited but useful for linter testing): node
34
34
  * scripts/build-corpus-stats.ts\
35
- * --shards /mnt/playpen/mailwoman-data/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
35
+ * --shards $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
36
36
  * --output /tmp/corpus-stats-local.json
37
37
  */
38
- import { execSync } from "node:child_process";
39
38
  import { readdirSync, statSync, writeFileSync } from "node:fs";
40
39
  import { join } from "node:path";
40
+ import { ParquetReader } from "../parquet-wrapper/index.js";
41
41
  const SEP = "";
42
42
  const MIN_BIGRAM_COUNT = 2;
43
43
  function discoverShards(shardsArg) {
@@ -53,33 +53,21 @@ function discoverShards(shardsArg) {
53
53
  return [shardsArg];
54
54
  }
55
55
  /**
56
- * Use a Python subprocess to read parquet (pyarrow is heavier than parquet-wasm but already on the path here, and we
57
- * have nothing in the JS ecosystem that reads parquet cleanly at this scale). Emits one JSON object per line: `{tokens:
58
- * [...], labels: [...]}`.
56
+ * Stream a shard's `tokens`/`labels` columns.
57
+ *
58
+ * `limit` stops the iteration rather than filtering afterwards, so a capped run reads only the row groups it needs.
59
59
  */
60
- function streamShardRows(shardPath, limit) {
61
- // Pipe the python script via stdin instead of `-c` to preserve newlines verbatim
62
- // (JSON-encoding the script for -c collapses real newlines into literal `\n`).
63
- const py = `
64
- import pyarrow.parquet as pq
65
- import json, sys
66
- t = pq.read_table(${JSON.stringify(shardPath)}, columns=['tokens', 'labels'])
67
- tokens_col = t['tokens'].to_pylist()
68
- labels_col = t['labels'].to_pylist()
69
- n = min(len(tokens_col), ${limit ?? "len(tokens_col)"})
70
- for i in range(n):
71
- sys.stdout.write(json.dumps({"tokens": tokens_col[i], "labels": labels_col[i]}) + "\\n")
72
- `;
73
- const buf = execSync(`python3`, { input: py, maxBuffer: 1024 * 1024 * 1024 });
74
- const rows = [];
75
- for (const line of buf.toString("utf8").split("\n")) {
76
- if (!line)
77
- continue;
78
- rows.push(JSON.parse(line));
60
+ async function* streamShardRows(shardPath, limit) {
61
+ await using reader = await ParquetReader.openFile(shardPath);
62
+ let emitted = 0;
63
+ for await (const row of reader.project("tokens", "labels")) {
64
+ if (limit !== undefined && emitted >= limit)
65
+ break;
66
+ yield row;
67
+ emitted++;
79
68
  }
80
- return rows;
81
69
  }
82
- export function buildCorpusStats(args) {
70
+ export async function buildCorpusStats(args) {
83
71
  const shardPaths = discoverShards(args.shardsArg);
84
72
  console.error(`Discovered ${shardPaths.length} parquet shard(s)`);
85
73
  const tokenStats = new Map();
@@ -87,10 +75,9 @@ export function buildCorpusStats(args) {
87
75
  let totalRows = 0;
88
76
  for (const path of shardPaths) {
89
77
  console.error(`Reading ${path}...`);
90
- const rows = streamShardRows(path, args.limitPerShard);
91
- totalRows += rows.length;
92
- for (const row of rows) {
93
- const { tokens, labels } = row;
78
+ const before = totalRows;
79
+ for await (const { tokens, labels } of streamShardRows(path, args.limitPerShard)) {
80
+ totalRows++;
94
81
  if (tokens.length !== labels.length)
95
82
  continue;
96
83
  // skip malformed
@@ -115,7 +102,7 @@ export function buildCorpusStats(args) {
115
102
  }
116
103
  }
117
104
  }
118
- console.error(` ${rows.length} rows; running totals: ${tokenStats.size} unique tokens, ${bigramStats.size} unique bigrams`);
105
+ console.error(` ${totalRows - before} rows; running totals: ${tokenStats.size} unique tokens, ${bigramStats.size} unique bigrams`);
119
106
  }
120
107
  // Prune bigrams below MIN_BIGRAM_COUNT to keep the output file size sane. Token stats
121
108
  // stay complete — they're cheap and we need accuracy at the long tail for label-vacuum
@@ -1 +1 @@
1
- {"version":3,"file":"corpus-stats.js","sourceRoot":"","sources":["../../../src/tools/corpus-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAC7C,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,MAAM,GAAG,GAAG,GAAG,CAAA;AACf,MAAM,gBAAgB,GAAG,CAAC,CAAA;AAQ1B,SAAS,cAAc,CAAC,SAAiB;IACxC,MAAM,IAAI,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;IAEhC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QACxB,OAAO,WAAW,CAAC,SAAS,CAAC;aAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;aACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAA;IACjC,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,CAAC,SAAS,CAAC,CAAA;IAEvE,gFAAgF;IAChF,OAAO,CAAC,SAAS,CAAC,CAAA;AACnB,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CAAC,SAAiB,EAAE,KAAc;IACzD,iFAAiF;IACjF,+EAA+E;IAC/E,MAAM,EAAE,GAAG;;;oBAGQ,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;;;2BAGlB,KAAK,IAAI,iBAAiB;;;CAGpD,CAAA;IAEA,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAA;IAC7E,MAAM,IAAI,GAAkD,EAAE,CAAA;IAE9D,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC,IAAI;YAAE,SAAQ;QACnB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAwB;IACxD,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAEjD,OAAO,CAAC,KAAK,CAAC,cAAc,UAAU,CAAC,MAAM,mBAAmB,CAAC,CAAA;IAEjE,MAAM,UAAU,GAAG,IAAI,GAAG,EAA+B,CAAA;IACzD,MAAM,WAAW,GAAG,IAAI,GAAG,EAA+B,CAAA;IAC1D,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,OAAO,CAAC,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,CAAA;QAEnC,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QACtD,SAAS,IAAI,IAAI,CAAC,MAAM,CAAA;QAExB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,GAAG,CAAA;YAE9B,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;gBAAE,SAAQ;YAE7C,iBAAiB;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;gBACrB,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;gBACrB,IAAI,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;gBAEjC,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACf,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAA;oBACpB,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;gBAC7B,CAAC;gBAED,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAE7C,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC3B,MAAM,SAAS,GAAG,EAAE,GAAG,GAAG,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;oBAC3C,MAAM,WAAW,GAAG,EAAE,GAAG,GAAG,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;oBAC7C,IAAI,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;oBAErC,IAAI,CAAC,IAAI,EAAE,CAAC;wBACX,IAAI,GAAG,IAAI,GAAG,EAAE,CAAA;wBAChB,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;oBACjC,CAAC;oBAED,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBACxD,CAAC;YACF,CAAC;QACF,CAAC;QAED,OAAO,CAAC,KAAK,CACZ,KAAK,IAAI,CAAC,MAAM,0BAA0B,UAAU,CAAC,IAAI,mBAAmB,WAAW,CAAC,IAAI,iBAAiB,CAC7G,CAAA;IACF,CAAC;IAED,sFAAsF;IACtF,uFAAuF;IACvF,aAAa;IACb,IAAI,aAAa,GAAG,CAAC,CAAA;IAErB,KAAK,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,WAAW,EAAE,CAAC;QACzC,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YACnC,KAAK,IAAI,CAAC,CAAA;QACX,CAAC;QAED,IAAI,KAAK,GAAG,gBAAgB,EAAE,CAAC;YAC9B,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;YAErB,aAAa,EAAE,CAAA;QAChB,CAAC;IACF,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,UAAU,aAAa,uBAAuB,WAAW,CAAC,IAAI,SAAS,CAAC,CAAA;IAEtF,MAAM,GAAG,GAAG;QACX,SAAS,EAAE,SAAS;QACpB,WAAW,EAAE,UAAU;QACvB,MAAM,EAAE,EAA4C;QACpD,OAAO,EAAE,EAA4C;KACrD,CAAA;IAED,KAAK,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,UAAU,EAAE,CAAC;QACzC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC9C,CAAC;IAED,KAAK,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,WAAW,EAAE,CAAC;QACzC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC9C,CAAC;IAED,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAA;IACnD,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAEhF,OAAO,CAAC,KAAK,CACZ,SAAS,IAAI,CAAC,UAAU,KAAK,MAAM,UAAU,SAAS,UAAU,UAAU,CAAC,IAAI,YAAY,WAAW,CAAC,IAAI,UAAU,CACrH,CAAA;AACF,CAAC"}
1
+ {"version":3,"file":"corpus-stats.js","sourceRoot":"","sources":["../../../src/tools/corpus-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAE3D,MAAM,GAAG,GAAG,GAAG,CAAA;AACf,MAAM,gBAAgB,GAAG,CAAC,CAAA;AAQ1B,SAAS,cAAc,CAAC,SAAiB;IACxC,MAAM,IAAI,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;IAEhC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QACxB,OAAO,WAAW,CAAC,SAAS,CAAC;aAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;aACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAA;IACjC,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,CAAC,SAAS,CAAC,CAAA;IAEvE,gFAAgF;IAChF,OAAO,CAAC,SAAS,CAAC,CAAA;AACnB,CAAC;AAED;;;;GAIG;AACH,KAAK,SAAS,CAAC,CAAC,eAAe,CAC9B,SAAiB,EACjB,KAAc;IAEd,YAAY,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAyC,SAAS,CAAC,CAAA;IAEpG,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QAC5D,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,KAAK;YAAE,MAAK;QAElD,MAAM,GAAG,CAAA;QAET,OAAO,EAAE,CAAA;IACV,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAwB;IAC9D,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAEjD,OAAO,CAAC,KAAK,CAAC,cAAc,UAAU,CAAC,MAAM,mBAAmB,CAAC,CAAA;IAEjE,MAAM,UAAU,GAAG,IAAI,GAAG,EAA+B,CAAA;IACzD,MAAM,WAAW,GAAG,IAAI,GAAG,EAA+B,CAAA;IAC1D,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,OAAO,CAAC,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,CAAA;QAEnC,MAAM,MAAM,GAAG,SAAS,CAAA;QAExB,IAAI,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YAClF,SAAS,EAAE,CAAA;YAEX,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;gBAAE,SAAQ;YAE7C,iBAAiB;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;gBACrB,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;gBACrB,IAAI,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;gBAEjC,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACf,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAA;oBACpB,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;gBAC7B,CAAC;gBAED,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAE7C,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC3B,MAAM,SAAS,GAAG,EAAE,GAAG,GAAG,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;oBAC3C,MAAM,WAAW,GAAG,EAAE,GAAG,GAAG,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;oBAC7C,IAAI,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;oBAErC,IAAI,CAAC,IAAI,EAAE,CAAC;wBACX,IAAI,GAAG,IAAI,GAAG,EAAE,CAAA;wBAChB,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;oBACjC,CAAC;oBAED,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBACxD,CAAC;YACF,CAAC;QACF,CAAC;QAED,OAAO,CAAC,KAAK,CACZ,KAAK,SAAS,GAAG,MAAM,0BAA0B,UAAU,CAAC,IAAI,mBAAmB,WAAW,CAAC,IAAI,iBAAiB,CACpH,CAAA;IACF,CAAC;IAED,sFAAsF;IACtF,uFAAuF;IACvF,aAAa;IACb,IAAI,aAAa,GAAG,CAAC,CAAA;IAErB,KAAK,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,WAAW,EAAE,CAAC;QACzC,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YACnC,KAAK,IAAI,CAAC,CAAA;QACX,CAAC;QAED,IAAI,KAAK,GAAG,gBAAgB,EAAE,CAAC;YAC9B,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;YAErB,aAAa,EAAE,CAAA;QAChB,CAAC;IACF,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,UAAU,aAAa,uBAAuB,WAAW,CAAC,IAAI,SAAS,CAAC,CAAA;IAEtF,MAAM,GAAG,GAAG;QACX,SAAS,EAAE,SAAS;QACpB,WAAW,EAAE,UAAU;QACvB,MAAM,EAAE,EAA4C;QACpD,OAAO,EAAE,EAA4C;KACrD,CAAA;IAED,KAAK,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,UAAU,EAAE,CAAC;QACzC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC9C,CAAC;IAED,KAAK,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,WAAW,EAAE,CAAC;QACzC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC9C,CAAC;IAED,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAA;IACnD,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAEhF,OAAO,CAAC,KAAK,CACZ,SAAS,IAAI,CAAC,UAAU,KAAK,MAAM,UAAU,SAAS,UAAU,UAAU,CAAC,IAAI,YAAY,WAAW,CAAC,IAAI,UAAU,CACrH,CAAA;AACF,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAqBH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,EAAE,CAAA;CACrB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED,MAAM,WAAW,eAAe;IAC/B,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA0CzF;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAQrE;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAS7G;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElF"}
1
+ {"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAuBH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,EAAE,CAAA;CACrB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED,MAAM,WAAW,eAAe;IAC/B,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA0CzF;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAQrE;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAS7G;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElF"}
@@ -10,6 +10,7 @@
10
10
  import { existsSync } from "node:fs";
11
11
  import { readFile, writeFile } from "node:fs/promises";
12
12
  import { setTimeout as sleep } from "node:timers/promises";
13
+ import { tryParsingJSON } from "@mailwoman/core/objects";
13
14
  /**
14
15
  * Rate limited — retryable, the server is asking us to back off.
15
16
  */
@@ -74,12 +75,10 @@ export async function downloadToFile(options) {
74
75
  export async function readManifest(path) {
75
76
  if (!existsSync(path))
76
77
  return null;
77
- try {
78
- return JSON.parse(await readFile(path, "utf8"));
79
- }
80
- catch {
81
- return null;
82
- }
78
+ // A read failure (e.g. the file vanished after the existsSync probe) maps to null like corrupt
79
+ // JSON does; tryParsingJSON returns null for the non-string sentinel.
80
+ const text = await readFile(path, "utf8").catch(() => null);
81
+ return tryParsingJSON(text);
83
82
  }
84
83
  /**
85
84
  * Load manifest entries into a map so untouched keys survive a partial re-fetch.
@@ -1 +1 @@
1
- {"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE1D;;GAEG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAsBjC;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC/C,OAAO,MAAM,KAAK,sBAAsB,IAAI,CAAC,MAAM,IAAI,qBAAqB,IAAI,MAAM,IAAI,qBAAqB,CAAC,CAAA;AACjH,CAAC;AAqBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAwB;IAC5D,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,EAAE,OAAO,GAAG,CAAC,EAAE,YAAY,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IACrG,IAAI,SAAkB,CAAA;IAEtB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,EAAE,CAAC,SAAS,OAAO,IAAI,OAAO,UAAU,YAAY,QAAQ,GAAG,EAAE,CAAC,CAAA;YACxE,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,CAAC;QAED,IAAI,GAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;QAC5E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,iEAAiE;YACjE,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,MAAM,GAAG,EAAE,CAAC,CAAA;YAExE,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,MAAM,KAAK,CAAA;YAC/C,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAA;YACnD,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,CAAA;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,8DAA8D;YAC9D,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;AAC5E,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAI,IAAY;IACjD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAElC,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAM,CAAA;IACrD,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAI,IAAY,EAAE,GAAyB;IACnF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAA;IACpC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAM,IAAI,CAAC,CAAA;IAE5C,KAAK,MAAM,KAAK,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,QAAiB;IAClE,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;AAChE,CAAC"}
1
+ {"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE1D,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAExD;;GAEG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAsBjC;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC/C,OAAO,MAAM,KAAK,sBAAsB,IAAI,CAAC,MAAM,IAAI,qBAAqB,IAAI,MAAM,IAAI,qBAAqB,CAAC,CAAA;AACjH,CAAC;AAqBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAwB;IAC5D,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,EAAE,OAAO,GAAG,CAAC,EAAE,YAAY,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IACrG,IAAI,SAAkB,CAAA;IAEtB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,EAAE,CAAC,SAAS,OAAO,IAAI,OAAO,UAAU,YAAY,QAAQ,GAAG,EAAE,CAAC,CAAA;YACxE,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,CAAC;QAED,IAAI,GAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;QAC5E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,iEAAiE;YACjE,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,MAAM,GAAG,EAAE,CAAC,CAAA;YAExE,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,MAAM,KAAK,CAAA;YAC/C,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAA;YACnD,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,CAAA;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,8DAA8D;YAC9D,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;AAC5E,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAI,IAAY;IACjD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAElC,+FAA+F;IAC/F,sEAAsE;IACtE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;IAE3D,OAAO,cAAc,CAAI,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAI,IAAY,EAAE,GAAyB;IACnF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAA;IACpC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAM,IAAI,CAAC,CAAA;IAE5C,KAAK,MAAM,KAAK,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,QAAiB;IAClE,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;AAChE,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"imls-pls.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/imls-pls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAaH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAiBnE,MAAM,MAAM,mBAAmB,GAAG,gBAAgB,CAAA;AAsBlD,wBAAsB,YAAY,CACjC,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAgGvB"}
1
+ {"version":3,"file":"imls-pls.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/imls-pls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAYH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAenE,MAAM,MAAM,mBAAmB,GAAG,gBAAgB,CAAA;AAUlD,wBAAsB,YAAY,CACjC,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAiGvB"}
@@ -21,11 +21,10 @@
21
21
  */
22
22
  /* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
23
23
  procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
24
- import { execFile } from "node:child_process";
25
24
  import { existsSync, mkdirSync, statSync } from "node:fs";
26
25
  import { rm } from "node:fs/promises";
27
26
  import { basename, join } from "node:path";
28
- import { promisify } from "node:util";
27
+ import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip";
29
28
  import { sha256File } from "@mailwoman/core/utils";
30
29
  import { downloadToFile, readManifest, writeManifest } from "./download.js";
31
30
  /**
@@ -33,22 +32,11 @@ import { downloadToFile, readManifest, writeManifest } from "./download.js";
33
32
  * than data.
34
33
  */
35
34
  const BYTES_PER_KIB = 1024;
36
- const execFileAsync = promisify(execFile);
37
35
  /**
38
36
  * The PLS FY 2023 bulk CSV ZIP (most recent as of 2026-05). If IMLS publishes a newer year, update this URL.
39
37
  */
40
38
  const ZIP_URL = "https://www.imls.gov/sites/default/files/2025-08/pls_fy2023_csv.zip";
41
39
  const SLUG = "usgov-imls-pls";
42
- /**
43
- * Return the filenames listed inside a ZIP (the trailing column of each `unzip -l` row).
44
- */
45
- async function listZipEntries(zipPath) {
46
- const listing = await execFileAsync("unzip", ["-l", zipPath]);
47
- return listing.stdout
48
- .split("\n")
49
- .map((line) => line.trim().split(/\s+/).pop() ?? "")
50
- .filter((name) => name.length > 0);
51
- }
52
40
  export async function fetchIMLSPLS(options, report) {
53
41
  const destDir = join(options.outRoot, SLUG);
54
42
  mkdirSync(destDir, { recursive: true });
@@ -82,7 +70,7 @@ export async function fetchIMLSPLS(options, report) {
82
70
  // Outlet files match: pls_fy*outlet*.csv (case-insensitive)
83
71
  // Administrative-entity files match: pls_fy*ae*.csv — we skip those.
84
72
  report?.(" Inspecting ZIP contents ...");
85
- const entries = await listZipEntries(zipDest);
73
+ const entries = (await listZipEntries(zipDest)).map((entry) => entry.name);
86
74
  let csvName = entries.find((name) => /pls_fy.*outlet.*\.csv/i.test(name));
87
75
  // Fallback: if IMLS renames the file, grab any CSV that is NOT the ae file.
88
76
  if (!csvName) {
@@ -97,8 +85,8 @@ export async function fetchIMLSPLS(options, report) {
97
85
  return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
98
86
  }
99
87
  report?.(` Extracting outlet CSV: ${csvName}`);
100
- await execFileAsync("unzip", ["-o", "-j", zipDest, csvName, "-d", destDir]);
101
88
  const csvDest = join(destDir, basename(csvName));
89
+ await extractZipEntry(zipDest, csvName, csvDest);
102
90
  const csvSize = statSync(csvDest).size;
103
91
  const csvSha = await sha256File(csvDest);
104
92
  // MARK: Remove ZIP (small, but keep destDir clean)