data-analyst-mcp 0.0.0-stage → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (640) hide show
  1. package/README.md +262 -3
  2. package/dist/adapters/_acf_tidy.d.ts +18 -0
  3. package/dist/adapters/_acf_tidy.js +432 -0
  4. package/dist/adapters/_acf_tidy.js.map +1 -0
  5. package/dist/adapters/_actuaries_read.d.ts +38 -0
  6. package/dist/adapters/_actuaries_read.js +728 -0
  7. package/dist/adapters/_actuaries_read.js.map +1 -0
  8. package/dist/adapters/_census_micro.d.ts +70 -0
  9. package/dist/adapters/_census_micro.js +622 -0
  10. package/dist/adapters/_census_micro.js.map +1 -0
  11. package/dist/adapters/_census_util.d.ts +40 -0
  12. package/dist/adapters/_census_util.js +275 -0
  13. package/dist/adapters/_census_util.js.map +1 -0
  14. package/dist/adapters/_dol_states.d.ts +6 -0
  15. package/dist/adapters/_dol_states.js +58 -0
  16. package/dist/adapters/_dol_states.js.map +1 -0
  17. package/dist/adapters/_govinfo_entities.d.ts +4 -0
  18. package/dist/adapters/_govinfo_entities.js +6 -0
  19. package/dist/adapters/_govinfo_entities.js.map +1 -0
  20. package/dist/adapters/_govinfo_json.d.ts +13 -0
  21. package/dist/adapters/_govinfo_json.js +107 -0
  22. package/dist/adapters/_govinfo_json.js.map +1 -0
  23. package/dist/adapters/_govinfo_py.d.ts +22 -0
  24. package/dist/adapters/_govinfo_py.js +272 -0
  25. package/dist/adapters/_govinfo_py.js.map +1 -0
  26. package/dist/adapters/_govinfo_xml.d.ts +22 -0
  27. package/dist/adapters/_govinfo_xml.js +233 -0
  28. package/dist/adapters/_govinfo_xml.js.map +1 -0
  29. package/dist/adapters/_hud_excel.d.ts +2 -0
  30. package/dist/adapters/_hud_excel.js +166 -0
  31. package/dist/adapters/_hud_excel.js.map +1 -0
  32. package/dist/adapters/_omb_grid.d.ts +17 -0
  33. package/dist/adapters/_omb_grid.js +132 -0
  34. package/dist/adapters/_omb_grid.js.map +1 -0
  35. package/dist/adapters/_omb_util.d.ts +40 -0
  36. package/dist/adapters/_omb_util.js +192 -0
  37. package/dist/adapters/_omb_util.js.map +1 -0
  38. package/dist/adapters/_usaspending_state.d.ts +6 -0
  39. package/dist/adapters/_usaspending_state.js +54 -0
  40. package/dist/adapters/_usaspending_state.js.map +1 -0
  41. package/dist/adapters/_usda-fna_book.d.ts +59 -0
  42. package/dist/adapters/_usda-fna_book.js +869 -0
  43. package/dist/adapters/_usda-fna_book.js.map +1 -0
  44. package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
  45. package/dist/adapters/_usda-fna_pdf.js +361 -0
  46. package/dist/adapters/_usda-fna_pdf.js.map +1 -0
  47. package/dist/adapters/_usda-outlook_read.d.ts +70 -0
  48. package/dist/adapters/_usda-outlook_read.js +362 -0
  49. package/dist/adapters/_usda-outlook_read.js.map +1 -0
  50. package/dist/adapters/acf.d.ts +3 -0
  51. package/dist/adapters/acf.js +187 -0
  52. package/dist/adapters/acf.js.map +1 -0
  53. package/dist/adapters/actuaries.d.ts +3 -0
  54. package/dist/adapters/actuaries.js +293 -0
  55. package/dist/adapters/actuaries.js.map +1 -0
  56. package/dist/adapters/bts.d.ts +3 -0
  57. package/dist/adapters/bts.js +26 -0
  58. package/dist/adapters/bts.js.map +1 -0
  59. package/dist/adapters/census-geocoder.d.ts +10 -0
  60. package/dist/adapters/census-geocoder.js +47 -0
  61. package/dist/adapters/census-geocoder.js.map +1 -0
  62. package/dist/adapters/census.d.ts +17 -0
  63. package/dist/adapters/census.js +983 -0
  64. package/dist/adapters/census.js.map +1 -0
  65. package/dist/adapters/cfpb.d.ts +7 -0
  66. package/dist/adapters/cfpb.js +348 -0
  67. package/dist/adapters/cfpb.js.map +1 -0
  68. package/dist/adapters/clinical-trials.d.ts +3 -0
  69. package/dist/adapters/clinical-trials.js +135 -0
  70. package/dist/adapters/clinical-trials.js.map +1 -0
  71. package/dist/adapters/cms.d.ts +3 -0
  72. package/dist/adapters/cms.js +292 -0
  73. package/dist/adapters/cms.js.map +1 -0
  74. package/dist/adapters/college-scorecard.d.ts +3 -0
  75. package/dist/adapters/college-scorecard.js +122 -0
  76. package/dist/adapters/college-scorecard.js.map +1 -0
  77. package/dist/adapters/congress.d.ts +33 -0
  78. package/dist/adapters/congress.js +589 -0
  79. package/dist/adapters/congress.js.map +1 -0
  80. package/dist/adapters/dhs-ohss.d.ts +13 -0
  81. package/dist/adapters/dhs-ohss.js +598 -0
  82. package/dist/adapters/dhs-ohss.js.map +1 -0
  83. package/dist/adapters/doj-news.d.ts +7 -0
  84. package/dist/adapters/doj-news.js +286 -0
  85. package/dist/adapters/doj-news.js.map +1 -0
  86. package/dist/adapters/dol.d.ts +4 -0
  87. package/dist/adapters/dol.js +634 -0
  88. package/dist/adapters/dol.js.map +1 -0
  89. package/dist/adapters/eia.d.ts +15 -0
  90. package/dist/adapters/eia.js +456 -0
  91. package/dist/adapters/eia.js.map +1 -0
  92. package/dist/adapters/epa-aqs.d.ts +3 -0
  93. package/dist/adapters/epa-aqs.js +357 -0
  94. package/dist/adapters/epa-aqs.js.map +1 -0
  95. package/dist/adapters/epa.d.ts +3 -0
  96. package/dist/adapters/epa.js +298 -0
  97. package/dist/adapters/epa.js.map +1 -0
  98. package/dist/adapters/fbi.d.ts +3 -0
  99. package/dist/adapters/fbi.js +192 -0
  100. package/dist/adapters/fbi.js.map +1 -0
  101. package/dist/adapters/fda.d.ts +5 -0
  102. package/dist/adapters/fda.js +184 -0
  103. package/dist/adapters/fda.js.map +1 -0
  104. package/dist/adapters/fdic.d.ts +4 -0
  105. package/dist/adapters/fdic.js +95 -0
  106. package/dist/adapters/fdic.js.map +1 -0
  107. package/dist/adapters/fec.d.ts +3 -0
  108. package/dist/adapters/fec.js +142 -0
  109. package/dist/adapters/fec.js.map +1 -0
  110. package/dist/adapters/fed-board.d.ts +12 -0
  111. package/dist/adapters/fed-board.js +581 -0
  112. package/dist/adapters/fed-board.js.map +1 -0
  113. package/dist/adapters/federal-register.d.ts +11 -0
  114. package/dist/adapters/federal-register.js +43 -0
  115. package/dist/adapters/federal-register.js.map +1 -0
  116. package/dist/adapters/fhfa.d.ts +3 -0
  117. package/dist/adapters/fhfa.js +253 -0
  118. package/dist/adapters/fhfa.js.map +1 -0
  119. package/dist/adapters/fsa.d.ts +9 -0
  120. package/dist/adapters/fsa.js +335 -0
  121. package/dist/adapters/fsa.js.map +1 -0
  122. package/dist/adapters/govinfo.d.ts +30 -0
  123. package/dist/adapters/govinfo.js +636 -0
  124. package/dist/adapters/govinfo.js.map +1 -0
  125. package/dist/adapters/gsa-calc.d.ts +10 -0
  126. package/dist/adapters/gsa-calc.js +318 -0
  127. package/dist/adapters/gsa-calc.js.map +1 -0
  128. package/dist/adapters/hud.d.ts +3 -0
  129. package/dist/adapters/hud.js +200 -0
  130. package/dist/adapters/hud.js.map +1 -0
  131. package/dist/adapters/index.d.ts +3 -0
  132. package/dist/adapters/index.js +25 -0
  133. package/dist/adapters/index.js.map +1 -0
  134. package/dist/adapters/irs-soi.d.ts +6 -0
  135. package/dist/adapters/irs-soi.js +576 -0
  136. package/dist/adapters/irs-soi.js.map +1 -0
  137. package/dist/adapters/naep.d.ts +15 -0
  138. package/dist/adapters/naep.js +341 -0
  139. package/dist/adapters/naep.js.map +1 -0
  140. package/dist/adapters/nhtsa.d.ts +10 -0
  141. package/dist/adapters/nhtsa.js +239 -0
  142. package/dist/adapters/nhtsa.js.map +1 -0
  143. package/dist/adapters/nih.d.ts +5 -0
  144. package/dist/adapters/nih.js +196 -0
  145. package/dist/adapters/nih.js.map +1 -0
  146. package/dist/adapters/noaa.d.ts +3 -0
  147. package/dist/adapters/noaa.js +148 -0
  148. package/dist/adapters/noaa.js.map +1 -0
  149. package/dist/adapters/nrel.d.ts +3 -0
  150. package/dist/adapters/nrel.js +186 -0
  151. package/dist/adapters/nrel.js.map +1 -0
  152. package/dist/adapters/nws.d.ts +3 -0
  153. package/dist/adapters/nws.js +119 -0
  154. package/dist/adapters/nws.js.map +1 -0
  155. package/dist/adapters/omb.d.ts +15 -0
  156. package/dist/adapters/omb.js +517 -0
  157. package/dist/adapters/omb.js.map +1 -0
  158. package/dist/adapters/open-payments.d.ts +5 -0
  159. package/dist/adapters/open-payments.js +149 -0
  160. package/dist/adapters/open-payments.js.map +1 -0
  161. package/dist/adapters/pbgc.d.ts +13 -0
  162. package/dist/adapters/pbgc.js +283 -0
  163. package/dist/adapters/pbgc.js.map +1 -0
  164. package/dist/adapters/philfed.d.ts +12 -0
  165. package/dist/adapters/philfed.js +59 -0
  166. package/dist/adapters/philfed.js.map +1 -0
  167. package/dist/adapters/regulations.d.ts +6 -0
  168. package/dist/adapters/regulations.js +58 -0
  169. package/dist/adapters/regulations.js.map +1 -0
  170. package/dist/adapters/sec.d.ts +13 -0
  171. package/dist/adapters/sec.js +504 -0
  172. package/dist/adapters/sec.js.map +1 -0
  173. package/dist/adapters/senate-lobbying.d.ts +3 -0
  174. package/dist/adapters/senate-lobbying.js +181 -0
  175. package/dist/adapters/senate-lobbying.js.map +1 -0
  176. package/dist/adapters/snapqc.d.ts +32 -0
  177. package/dist/adapters/snapqc.js +701 -0
  178. package/dist/adapters/snapqc.js.map +1 -0
  179. package/dist/adapters/treasury-yields.d.ts +3 -0
  180. package/dist/adapters/treasury-yields.js +170 -0
  181. package/dist/adapters/treasury-yields.js.map +1 -0
  182. package/dist/adapters/usaspending.d.ts +3 -0
  183. package/dist/adapters/usaspending.js +340 -0
  184. package/dist/adapters/usaspending.js.map +1 -0
  185. package/dist/adapters/usda-fna.d.ts +7 -0
  186. package/dist/adapters/usda-fna.js +707 -0
  187. package/dist/adapters/usda-fna.js.map +1 -0
  188. package/dist/adapters/usda-fooddata.d.ts +3 -0
  189. package/dist/adapters/usda-fooddata.js +101 -0
  190. package/dist/adapters/usda-fooddata.js.map +1 -0
  191. package/dist/adapters/usda-nass.d.ts +3 -0
  192. package/dist/adapters/usda-nass.js +108 -0
  193. package/dist/adapters/usda-nass.js.map +1 -0
  194. package/dist/adapters/usda-outlook.d.ts +12 -0
  195. package/dist/adapters/usda-outlook.js +696 -0
  196. package/dist/adapters/usda-outlook.js.map +1 -0
  197. package/dist/adapters/usgs.d.ts +9 -0
  198. package/dist/adapters/usgs.js +265 -0
  199. package/dist/adapters/usgs.js.map +1 -0
  200. package/dist/adapters/usitc.d.ts +18 -0
  201. package/dist/adapters/usitc.js +170 -0
  202. package/dist/adapters/usitc.js.map +1 -0
  203. package/dist/adapters/uspto.d.ts +3 -0
  204. package/dist/adapters/uspto.js +47 -0
  205. package/dist/adapters/uspto.js.map +1 -0
  206. package/dist/adapters/world-bank.d.ts +3 -0
  207. package/dist/adapters/world-bank.js +163 -0
  208. package/dist/adapters/world-bank.js.map +1 -0
  209. package/dist/analysis/cite.d.ts +31 -0
  210. package/dist/analysis/cite.js +229 -0
  211. package/dist/analysis/cite.js.map +1 -0
  212. package/dist/analysis/combine.d.ts +27 -0
  213. package/dist/analysis/combine.js +198 -0
  214. package/dist/analysis/combine.js.map +1 -0
  215. package/dist/analysis/compute.d.ts +62 -0
  216. package/dist/analysis/compute.js +583 -0
  217. package/dist/analysis/compute.js.map +1 -0
  218. package/dist/analysis/expr.d.ts +69 -0
  219. package/dist/analysis/expr.js +780 -0
  220. package/dist/analysis/expr.js.map +1 -0
  221. package/dist/analysis/factfmt.d.ts +16 -0
  222. package/dist/analysis/factfmt.js +187 -0
  223. package/dist/analysis/factfmt.js.map +1 -0
  224. package/dist/analysis/facts.d.ts +63 -0
  225. package/dist/analysis/facts.js +355 -0
  226. package/dist/analysis/facts.js.map +1 -0
  227. package/dist/analysis/handoff.d.ts +58 -0
  228. package/dist/analysis/handoff.js +208 -0
  229. package/dist/analysis/handoff.js.map +1 -0
  230. package/dist/analysis/pyfmt.d.ts +16 -0
  231. package/dist/analysis/pyfmt.js +156 -0
  232. package/dist/analysis/pyfmt.js.map +1 -0
  233. package/dist/analysis/rerun.d.ts +13 -0
  234. package/dist/analysis/rerun.js +125 -0
  235. package/dist/analysis/rerun.js.map +1 -0
  236. package/dist/analysis/table.d.ts +18 -0
  237. package/dist/analysis/table.js +109 -0
  238. package/dist/analysis/table.js.map +1 -0
  239. package/dist/analysis/tools.d.ts +67 -0
  240. package/dist/analysis/tools.js +217 -0
  241. package/dist/analysis/tools.js.map +1 -0
  242. package/dist/analysis/write_table.d.ts +13 -0
  243. package/dist/analysis/write_table.js +59 -0
  244. package/dist/analysis/write_table.js.map +1 -0
  245. package/dist/bin/cli.d.ts +10 -0
  246. package/dist/bin/cli.js +177 -0
  247. package/dist/bin/cli.js.map +1 -0
  248. package/dist/core/cache.d.ts +14 -0
  249. package/dist/core/cache.js +90 -0
  250. package/dist/core/cache.js.map +1 -0
  251. package/dist/core/context.d.ts +12 -0
  252. package/dist/core/context.js +15 -0
  253. package/dist/core/context.js.map +1 -0
  254. package/dist/core/env.d.ts +14 -0
  255. package/dist/core/env.js +52 -0
  256. package/dist/core/env.js.map +1 -0
  257. package/dist/core/generic-rest.d.ts +61 -0
  258. package/dist/core/generic-rest.js +639 -0
  259. package/dist/core/generic-rest.js.map +1 -0
  260. package/dist/core/http.d.ts +120 -0
  261. package/dist/core/http.js +514 -0
  262. package/dist/core/http.js.map +1 -0
  263. package/dist/core/keys.d.ts +43 -0
  264. package/dist/core/keys.js +174 -0
  265. package/dist/core/keys.js.map +1 -0
  266. package/dist/core/manual.d.ts +41 -0
  267. package/dist/core/manual.js +218 -0
  268. package/dist/core/manual.js.map +1 -0
  269. package/dist/core/package.d.ts +58 -0
  270. package/dist/core/package.js +228 -0
  271. package/dist/core/package.js.map +1 -0
  272. package/dist/core/packs.d.ts +139 -0
  273. package/dist/core/packs.js +173 -0
  274. package/dist/core/packs.js.map +1 -0
  275. package/dist/core/policy.d.ts +69 -0
  276. package/dist/core/policy.js +189 -0
  277. package/dist/core/policy.js.map +1 -0
  278. package/dist/core/py.d.ts +40 -0
  279. package/dist/core/py.js +171 -0
  280. package/dist/core/py.js.map +1 -0
  281. package/dist/core/routing.d.ts +10 -0
  282. package/dist/core/routing.js +39 -0
  283. package/dist/core/routing.js.map +1 -0
  284. package/dist/core/tabular.d.ts +21 -0
  285. package/dist/core/tabular.js +229 -0
  286. package/dist/core/tabular.js.map +1 -0
  287. package/dist/deep/_bea_api.d.ts +102 -0
  288. package/dist/deep/_bea_api.js +457 -0
  289. package/dist/deep/_bea_api.js.map +1 -0
  290. package/dist/deep/_bea_fetch.d.ts +151 -0
  291. package/dist/deep/_bea_fetch.js +1274 -0
  292. package/dist/deep/_bea_fetch.js.map +1 -0
  293. package/dist/deep/_bea_util.d.ts +14 -0
  294. package/dist/deep/_bea_util.js +156 -0
  295. package/dist/deep/_bea_util.js.map +1 -0
  296. package/dist/deep/_bls_api.d.ts +84 -0
  297. package/dist/deep/_bls_api.js +231 -0
  298. package/dist/deep/_bls_api.js.map +1 -0
  299. package/dist/deep/_bls_ids.d.ts +43 -0
  300. package/dist/deep/_bls_ids.js +188 -0
  301. package/dist/deep/_bls_ids.js.map +1 -0
  302. package/dist/deep/_bls_lib.d.ts +61 -0
  303. package/dist/deep/_bls_lib.js +442 -0
  304. package/dist/deep/_bls_lib.js.map +1 -0
  305. package/dist/deep/_fred_api.d.ts +148 -0
  306. package/dist/deep/_fred_api.js +545 -0
  307. package/dist/deep/_fred_api.js.map +1 -0
  308. package/dist/deep/_fred_pack.d.ts +162 -0
  309. package/dist/deep/_fred_pack.js +1027 -0
  310. package/dist/deep/_fred_pack.js.map +1 -0
  311. package/dist/deep/_fred_py.d.ts +40 -0
  312. package/dist/deep/_fred_py.js +199 -0
  313. package/dist/deep/_fred_py.js.map +1 -0
  314. package/dist/deep/_ipums_bundle.d.ts +29 -0
  315. package/dist/deep/_ipums_bundle.js +194 -0
  316. package/dist/deep/_ipums_bundle.js.map +1 -0
  317. package/dist/deep/_ipums_ddi.d.ts +23 -0
  318. package/dist/deep/_ipums_ddi.js +162 -0
  319. package/dist/deep/_ipums_ddi.js.map +1 -0
  320. package/dist/deep/_opm_io.d.ts +26 -0
  321. package/dist/deep/_opm_io.js +383 -0
  322. package/dist/deep/_opm_io.js.map +1 -0
  323. package/dist/deep/_opm_parquet.d.ts +31 -0
  324. package/dist/deep/_opm_parquet.js +744 -0
  325. package/dist/deep/_opm_parquet.js.map +1 -0
  326. package/dist/deep/_opm_py.d.ts +6 -0
  327. package/dist/deep/_opm_py.js +74 -0
  328. package/dist/deep/_opm_py.js.map +1 -0
  329. package/dist/deep/_treasury_api.d.ts +48 -0
  330. package/dist/deep/_treasury_api.js +221 -0
  331. package/dist/deep/_treasury_api.js.map +1 -0
  332. package/dist/deep/_treasury_context.d.ts +34 -0
  333. package/dist/deep/_treasury_context.js +142 -0
  334. package/dist/deep/_treasury_context.js.map +1 -0
  335. package/dist/deep/_treasury_lib.d.ts +109 -0
  336. package/dist/deep/_treasury_lib.js +686 -0
  337. package/dist/deep/_treasury_lib.js.map +1 -0
  338. package/dist/deep/bea.d.ts +3 -0
  339. package/dist/deep/bea.js +248 -0
  340. package/dist/deep/bea.js.map +1 -0
  341. package/dist/deep/bls.d.ts +3 -0
  342. package/dist/deep/bls.js +220 -0
  343. package/dist/deep/bls.js.map +1 -0
  344. package/dist/deep/fred.d.ts +3 -0
  345. package/dist/deep/fred.js +325 -0
  346. package/dist/deep/fred.js.map +1 -0
  347. package/dist/deep/ipums.d.ts +59 -0
  348. package/dist/deep/ipums.js +744 -0
  349. package/dist/deep/ipums.js.map +1 -0
  350. package/dist/deep/opm.d.ts +56 -0
  351. package/dist/deep/opm.js +1033 -0
  352. package/dist/deep/opm.js.map +1 -0
  353. package/dist/deep/treasury.d.ts +3 -0
  354. package/dist/deep/treasury.js +218 -0
  355. package/dist/deep/treasury.js.map +1 -0
  356. package/dist/envelope.d.ts +47 -0
  357. package/dist/envelope.js +88 -0
  358. package/dist/envelope.js.map +1 -0
  359. package/dist/index.d.ts +277 -0
  360. package/dist/index.js +106 -0
  361. package/dist/index.js.map +1 -0
  362. package/dist/remote/app.d.ts +21 -0
  363. package/dist/remote/app.js +218 -0
  364. package/dist/remote/app.js.map +1 -0
  365. package/dist/remote/oauth.d.ts +14 -0
  366. package/dist/remote/oauth.js +90 -0
  367. package/dist/remote/oauth.js.map +1 -0
  368. package/dist/remote/pages.d.ts +8 -0
  369. package/dist/remote/pages.js +56 -0
  370. package/dist/remote/pages.js.map +1 -0
  371. package/dist/remote/sign.d.ts +4 -0
  372. package/dist/remote/sign.js +40 -0
  373. package/dist/remote/sign.js.map +1 -0
  374. package/dist/remote/store.d.ts +31 -0
  375. package/dist/remote/store.js +129 -0
  376. package/dist/remote/store.js.map +1 -0
  377. package/dist/report/audit_answer.d.ts +94 -0
  378. package/dist/report/audit_answer.js +1349 -0
  379. package/dist/report/audit_answer.js.map +1 -0
  380. package/dist/report/audit_report.d.ts +23 -0
  381. package/dist/report/audit_report.js +637 -0
  382. package/dist/report/audit_report.js.map +1 -0
  383. package/dist/report/build_report.d.ts +27 -0
  384. package/dist/report/build_report.js +1168 -0
  385. package/dist/report/build_report.js.map +1 -0
  386. package/dist/report/method_audit.d.ts +47 -0
  387. package/dist/report/method_audit.js +710 -0
  388. package/dist/report/method_audit.js.map +1 -0
  389. package/dist/report/policyset.d.ts +50 -0
  390. package/dist/report/policyset.js +421 -0
  391. package/dist/report/policyset.js.map +1 -0
  392. package/dist/report/py.d.ts +58 -0
  393. package/dist/report/py.js +311 -0
  394. package/dist/report/py.js.map +1 -0
  395. package/dist/report/scan_secrets.d.ts +21 -0
  396. package/dist/report/scan_secrets.js +234 -0
  397. package/dist/report/scan_secrets.js.map +1 -0
  398. package/dist/report/style_check.d.ts +29 -0
  399. package/dist/report/style_check.js +173 -0
  400. package/dist/report/style_check.js.map +1 -0
  401. package/dist/report/tools.d.ts +87 -0
  402. package/dist/report/tools.js +167 -0
  403. package/dist/report/tools.js.map +1 -0
  404. package/dist/server.d.ts +5 -0
  405. package/dist/server.js +246 -0
  406. package/dist/server.js.map +1 -0
  407. package/dist/session.d.ts +47 -0
  408. package/dist/session.js +107 -0
  409. package/dist/session.js.map +1 -0
  410. package/dist/toolkit.d.ts +21 -0
  411. package/dist/toolkit.js +2 -0
  412. package/dist/toolkit.js.map +1 -0
  413. package/dist/tools.d.ts +299 -0
  414. package/dist/tools.js +491 -0
  415. package/dist/tools.js.map +1 -0
  416. package/guide/WORKFLOW.md +134 -0
  417. package/guide/assets/disclaimer.txt +1 -0
  418. package/guide/assets/download_data.ps1 +162 -0
  419. package/guide/assets/download_data.sh +143 -0
  420. package/guide/assets/report_template.html +597 -0
  421. package/guide/references/report_patterns.md +199 -0
  422. package/guide/references/routing.md +171 -0
  423. package/guide/references/style_guide.md +398 -0
  424. package/package.json +56 -6
  425. package/packs/acf/endpoints.json +114 -0
  426. package/packs/acf/policy.json +104 -0
  427. package/packs/acf/source.md +138 -0
  428. package/packs/actuaries/endpoints.json +82 -0
  429. package/packs/actuaries/policy.json +109 -0
  430. package/packs/actuaries/source.md +137 -0
  431. package/packs/bea/assets/download_bea.ps1 +182 -0
  432. package/packs/bea/assets/download_bea.sh +144 -0
  433. package/packs/bea/docs/bea_api.md +461 -0
  434. package/packs/bea/docs/datasets.md +157 -0
  435. package/packs/bea/docs/report_patterns.md +112 -0
  436. package/packs/bea/policy.json +170 -0
  437. package/packs/bea/source.md +152 -0
  438. package/packs/bls/assets/download_bls.ps1 +152 -0
  439. package/packs/bls/assets/download_bls.sh +167 -0
  440. package/packs/bls/docs/analysis_patterns.md +108 -0
  441. package/packs/bls/docs/bls_api.md +73 -0
  442. package/packs/bls/docs/series_ids.md +149 -0
  443. package/packs/bls/policy.json +238 -0
  444. package/packs/bls/series_ids.json +1375 -0
  445. package/packs/bls/source.md +156 -0
  446. package/packs/bts/endpoints.json +347 -0
  447. package/packs/bts/policy.json +94 -0
  448. package/packs/bts/source.md +97 -0
  449. package/packs/cdc/endpoints.json +197 -0
  450. package/packs/cdc/policy.json +83 -0
  451. package/packs/cdc/source.md +74 -0
  452. package/packs/census/endpoints.json +99 -0
  453. package/packs/census/policy.json +238 -0
  454. package/packs/census/source.md +304 -0
  455. package/packs/census-geocoder/endpoints.json +70 -0
  456. package/packs/census-geocoder/policy.json +80 -0
  457. package/packs/census-geocoder/source.md +53 -0
  458. package/packs/cfpb/endpoints.json +327 -0
  459. package/packs/cfpb/policy.json +119 -0
  460. package/packs/cfpb/source.md +96 -0
  461. package/packs/clinical-trials/endpoints.json +167 -0
  462. package/packs/clinical-trials/policy.json +84 -0
  463. package/packs/clinical-trials/source.md +82 -0
  464. package/packs/cms/endpoints.json +69 -0
  465. package/packs/cms/policy.json +96 -0
  466. package/packs/cms/source.md +112 -0
  467. package/packs/college-scorecard/endpoints.json +253 -0
  468. package/packs/college-scorecard/policy.json +106 -0
  469. package/packs/college-scorecard/source.md +92 -0
  470. package/packs/congress/endpoints.json +1701 -0
  471. package/packs/congress/policy.json +117 -0
  472. package/packs/congress/source.md +190 -0
  473. package/packs/dhs-ohss/endpoints.json +66 -0
  474. package/packs/dhs-ohss/policy.json +107 -0
  475. package/packs/dhs-ohss/source.md +120 -0
  476. package/packs/doj-news/endpoints.json +59 -0
  477. package/packs/doj-news/policy.json +96 -0
  478. package/packs/doj-news/source.md +99 -0
  479. package/packs/dol/endpoints.json +818 -0
  480. package/packs/dol/policy.json +135 -0
  481. package/packs/dol/source.md +181 -0
  482. package/packs/eia/endpoints.json +718 -0
  483. package/packs/eia/policy.json +112 -0
  484. package/packs/eia/source.md +112 -0
  485. package/packs/epa/endpoints.json +292 -0
  486. package/packs/epa/policy.json +99 -0
  487. package/packs/epa/source.md +99 -0
  488. package/packs/epa-aqs/endpoints.json +174 -0
  489. package/packs/epa-aqs/policy.json +96 -0
  490. package/packs/epa-aqs/source.md +97 -0
  491. package/packs/fbi/endpoints.json +1842 -0
  492. package/packs/fbi/policy.json +118 -0
  493. package/packs/fbi/source.md +108 -0
  494. package/packs/fda/endpoints.json +238 -0
  495. package/packs/fda/policy.json +121 -0
  496. package/packs/fda/source.md +117 -0
  497. package/packs/fdic/endpoints.json +417 -0
  498. package/packs/fdic/policy.json +83 -0
  499. package/packs/fdic/source.md +92 -0
  500. package/packs/fec/endpoints.json +708 -0
  501. package/packs/fec/policy.json +93 -0
  502. package/packs/fec/source.md +90 -0
  503. package/packs/fed-board/endpoints.json +89 -0
  504. package/packs/fed-board/policy.json +136 -0
  505. package/packs/fed-board/source.md +118 -0
  506. package/packs/federal-register/endpoints.json +143 -0
  507. package/packs/federal-register/policy.json +86 -0
  508. package/packs/federal-register/source.md +98 -0
  509. package/packs/fema/endpoints.json +646 -0
  510. package/packs/fema/policy.json +103 -0
  511. package/packs/fema/source.md +103 -0
  512. package/packs/fhfa/endpoints.json +105 -0
  513. package/packs/fhfa/policy.json +109 -0
  514. package/packs/fhfa/source.md +108 -0
  515. package/packs/fred/assets/download_fred.ps1 +122 -0
  516. package/packs/fred/assets/download_fred.sh +123 -0
  517. package/packs/fred/docs/analysis_patterns.md +124 -0
  518. package/packs/fred/docs/fred_api.md +94 -0
  519. package/packs/fred/docs/series.md +188 -0
  520. package/packs/fred/policy.json +253 -0
  521. package/packs/fred/source.md +142 -0
  522. package/packs/fsa/endpoints.json +150 -0
  523. package/packs/fsa/policy.json +99 -0
  524. package/packs/fsa/source.md +102 -0
  525. package/packs/govinfo/endpoints.json +520 -0
  526. package/packs/govinfo/policy.json +116 -0
  527. package/packs/govinfo/source.md +241 -0
  528. package/packs/gsa-calc/endpoints.json +93 -0
  529. package/packs/gsa-calc/policy.json +97 -0
  530. package/packs/gsa-calc/source.md +108 -0
  531. package/packs/hud/endpoints.json +147 -0
  532. package/packs/hud/policy.json +84 -0
  533. package/packs/hud/source.md +97 -0
  534. package/packs/ipums/assets/download_ipums.ps1 +215 -0
  535. package/packs/ipums/assets/download_ipums.sh +192 -0
  536. package/packs/ipums/docs/ipums_api.md +67 -0
  537. package/packs/ipums/docs/report_patterns.md +117 -0
  538. package/packs/ipums/docs/variables.md +284 -0
  539. package/packs/ipums/lib/fetch_text.json +112 -0
  540. package/packs/ipums/policy.json +215 -0
  541. package/packs/ipums/source.md +169 -0
  542. package/packs/irs-soi/endpoints.json +114 -0
  543. package/packs/irs-soi/policy.json +107 -0
  544. package/packs/irs-soi/source.md +124 -0
  545. package/packs/medicaid/endpoints.json +412 -0
  546. package/packs/medicaid/policy.json +108 -0
  547. package/packs/medicaid/source.md +136 -0
  548. package/packs/naep/endpoints.json +217 -0
  549. package/packs/naep/policy.json +100 -0
  550. package/packs/naep/source.md +120 -0
  551. package/packs/nhtsa/endpoints.json +132 -0
  552. package/packs/nhtsa/policy.json +93 -0
  553. package/packs/nhtsa/source.md +109 -0
  554. package/packs/nih/endpoints.json +453 -0
  555. package/packs/nih/policy.json +86 -0
  556. package/packs/nih/source.md +108 -0
  557. package/packs/noaa/endpoints.json +432 -0
  558. package/packs/noaa/policy.json +95 -0
  559. package/packs/noaa/source.md +88 -0
  560. package/packs/nrel/endpoints.json +328 -0
  561. package/packs/nrel/policy.json +96 -0
  562. package/packs/nrel/source.md +91 -0
  563. package/packs/nws/endpoints.json +272 -0
  564. package/packs/nws/policy.json +82 -0
  565. package/packs/nws/source.md +81 -0
  566. package/packs/nyfed/endpoints.json +80 -0
  567. package/packs/nyfed/policy.json +92 -0
  568. package/packs/nyfed/source.md +89 -0
  569. package/packs/omb/endpoints.json +460 -0
  570. package/packs/omb/policy.json +106 -0
  571. package/packs/omb/source.md +130 -0
  572. package/packs/open-payments/endpoints.json +298 -0
  573. package/packs/open-payments/policy.json +101 -0
  574. package/packs/open-payments/source.md +99 -0
  575. package/packs/opm/assets/download_opm.ps1 +125 -0
  576. package/packs/opm/assets/download_opm.sh +85 -0
  577. package/packs/opm/docs/fields.md +184 -0
  578. package/packs/opm/docs/opm_api.md +71 -0
  579. package/packs/opm/docs/report_patterns.md +83 -0
  580. package/packs/opm/lib/data_dictionary.csv +73 -0
  581. package/packs/opm/policy.json +142 -0
  582. package/packs/opm/source.md +137 -0
  583. package/packs/pbgc/endpoints.json +483 -0
  584. package/packs/pbgc/policy.json +94 -0
  585. package/packs/pbgc/source.md +95 -0
  586. package/packs/philfed/endpoints.json +85 -0
  587. package/packs/philfed/policy.json +94 -0
  588. package/packs/philfed/source.md +125 -0
  589. package/packs/port.json +287 -0
  590. package/packs/regulations/endpoints.json +288 -0
  591. package/packs/regulations/policy.json +86 -0
  592. package/packs/regulations/source.md +84 -0
  593. package/packs/sec/endpoints.json +237 -0
  594. package/packs/sec/policy.json +86 -0
  595. package/packs/sec/source.md +101 -0
  596. package/packs/senate-lobbying/endpoints.json +554 -0
  597. package/packs/senate-lobbying/policy.json +99 -0
  598. package/packs/senate-lobbying/source.md +100 -0
  599. package/packs/snapqc/endpoints.json +1426 -0
  600. package/packs/snapqc/policy.json +92 -0
  601. package/packs/snapqc/source.md +96 -0
  602. package/packs/treasury/catalog.json +2267 -0
  603. package/packs/treasury/docs/api_guide.md +245 -0
  604. package/packs/treasury/docs/datasets.md +56 -0
  605. package/packs/treasury/docs/report_patterns.md +126 -0
  606. package/packs/treasury/policy.json +159 -0
  607. package/packs/treasury/source.md +165 -0
  608. package/packs/treasury-yields/endpoints.json +226 -0
  609. package/packs/treasury-yields/policy.json +86 -0
  610. package/packs/treasury-yields/source.md +80 -0
  611. package/packs/usaspending/endpoints.json +1316 -0
  612. package/packs/usaspending/policy.json +109 -0
  613. package/packs/usaspending/source.md +103 -0
  614. package/packs/usda-fna/endpoints.json +427 -0
  615. package/packs/usda-fna/policy.json +133 -0
  616. package/packs/usda-fna/source.md +217 -0
  617. package/packs/usda-fooddata/endpoints.json +181 -0
  618. package/packs/usda-fooddata/policy.json +94 -0
  619. package/packs/usda-fooddata/source.md +105 -0
  620. package/packs/usda-nass/endpoints.json +324 -0
  621. package/packs/usda-nass/policy.json +95 -0
  622. package/packs/usda-nass/source.md +85 -0
  623. package/packs/usda-outlook/endpoints.json +86 -0
  624. package/packs/usda-outlook/policy.json +109 -0
  625. package/packs/usda-outlook/source.md +144 -0
  626. package/packs/usgs/endpoints.json +313 -0
  627. package/packs/usgs/policy.json +95 -0
  628. package/packs/usgs/source.md +96 -0
  629. package/packs/usitc/endpoints.json +74 -0
  630. package/packs/usitc/policy.json +111 -0
  631. package/packs/usitc/source.md +90 -0
  632. package/packs/uspto/endpoints.json +141 -0
  633. package/packs/uspto/policy.json +95 -0
  634. package/packs/uspto/source.md +101 -0
  635. package/packs/va/endpoints.json +414 -0
  636. package/packs/va/policy.json +101 -0
  637. package/packs/va/source.md +129 -0
  638. package/packs/world-bank/endpoints.json +122 -0
  639. package/packs/world-bank/policy.json +101 -0
  640. package/packs/world-bank/source.md +93 -0
@@ -0,0 +1,744 @@
1
+ /**
2
+ * Parquet for the opm pack, with Node built-ins only (no Parquet package is installed): a reader for flat files
3
+ * (the FWD API serves every column as an optional UTF-8 string) and a small writer for the filtered tables.
4
+ *
5
+ * Reader: Thrift compact footer, data pages v1 and v2, dictionary pages; encodings PLAIN, PLAIN_DICTIONARY,
6
+ * RLE_DICTIONARY, RLE (booleans), DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY,
7
+ * BYTE_STREAM_SPLIT; codecs UNCOMPRESSED, SNAPPY, GZIP, BROTLI, ZSTD, LZ4_RAW. Values come back as strings, the
8
+ * way pandas `astype("string")` shows them (ints as digits, floats as Python repr, booleans True/False), and
9
+ * nulls as null.
10
+ * Writer: one data page v1 per column and row group, PLAIN values, RLE definition levels, ZSTD compression.
11
+ */
12
+ import fs from "node:fs";
13
+ import zlib from "node:zlib";
14
+ import { pyFloatRepr } from "../core/py.js";
15
+ // ---------------------------------------------------------------------- Thrift compact protocol
16
+ class TReader {
17
+ b;
18
+ pos;
19
+ constructor(b, pos = 0) {
20
+ this.b = b;
21
+ this.pos = pos;
22
+ }
23
+ byte() {
24
+ if (this.pos >= this.b.length)
25
+ throw new Error("truncated Parquet metadata");
26
+ return this.b[this.pos++];
27
+ }
28
+ varint() {
29
+ let n = 0, mul = 1;
30
+ for (;;) {
31
+ const x = this.byte();
32
+ n += (x & 0x7f) * mul;
33
+ if (!(x & 0x80))
34
+ return n;
35
+ mul *= 128;
36
+ }
37
+ }
38
+ varbig() {
39
+ let n = 0n, shift = 0n;
40
+ for (;;) {
41
+ const x = this.byte();
42
+ n |= BigInt(x & 0x7f) << shift;
43
+ if (!(x & 0x80))
44
+ return n;
45
+ shift += 7n;
46
+ }
47
+ }
48
+ zz32() {
49
+ const n = this.varint();
50
+ return n % 2 ? -(n + 1) / 2 : n / 2;
51
+ }
52
+ zz64() {
53
+ const n = this.varbig();
54
+ return Number((n >> 1n) ^ -(n & 1n));
55
+ }
56
+ binary() {
57
+ const n = this.varint();
58
+ const out = this.b.subarray(this.pos, this.pos + n);
59
+ this.pos += n;
60
+ return out;
61
+ }
62
+ value(t) {
63
+ switch (t) {
64
+ case 1: return true;
65
+ case 2: return false;
66
+ case 3: return (this.byte() << 24) >> 24;
67
+ case 4:
68
+ case 5: return this.zz32();
69
+ case 6: return this.zz64();
70
+ case 7: {
71
+ const v = this.b.readDoubleLE(this.pos);
72
+ this.pos += 8;
73
+ return v;
74
+ }
75
+ case 8: return this.binary();
76
+ case 9:
77
+ case 10: return this.list();
78
+ case 11: return this.map();
79
+ case 12: return this.struct();
80
+ default: throw new Error(`Parquet metadata: unknown Thrift type ${t}`);
81
+ }
82
+ }
83
+ list() {
84
+ const h = this.byte();
85
+ let n = h >> 4;
86
+ const et = h & 15;
87
+ if (n === 15)
88
+ n = this.varint();
89
+ const out = new Array(n);
90
+ for (let i = 0; i < n; i++)
91
+ out[i] = et === 1 || et === 2 ? this.byte() === 1 : this.value(et);
92
+ return out;
93
+ }
94
+ map() {
95
+ const n = this.varint();
96
+ if (!n)
97
+ return [];
98
+ const kv = this.byte();
99
+ const out = [];
100
+ for (let i = 0; i < n; i++)
101
+ out.push([this.value(kv >> 4), this.value(kv & 15)]);
102
+ return out;
103
+ }
104
+ struct() {
105
+ const out = {};
106
+ let last = 0;
107
+ for (;;) {
108
+ const h = this.byte();
109
+ if (h === 0)
110
+ return out;
111
+ const t = h & 15, d = h >> 4;
112
+ const id = d ? last + d : this.zz32();
113
+ last = id;
114
+ out[id] = this.value(t);
115
+ }
116
+ }
117
+ }
118
+ /** A Thrift compact struct writer (fields in increasing id order). */
119
+ class TWriter {
120
+ out = [];
121
+ last = [0];
122
+ raw(...b) {
123
+ for (const x of b)
124
+ this.out.push(x);
125
+ }
126
+ varint(n) {
127
+ while (n >= 0x80) {
128
+ this.out.push((n % 128) | 0x80);
129
+ n = Math.floor(n / 128);
130
+ }
131
+ this.out.push(n);
132
+ }
133
+ zz(n) {
134
+ this.varint(n >= 0 ? n * 2 : -n * 2 - 1);
135
+ }
136
+ field(id, type) {
137
+ const top = this.last.length - 1;
138
+ const d = id - this.last[top];
139
+ if (d > 0 && d <= 15)
140
+ this.raw((d << 4) | type);
141
+ else {
142
+ this.raw(type);
143
+ this.zz(id);
144
+ }
145
+ this.last[top] = id;
146
+ }
147
+ i32(id, v) {
148
+ this.field(id, 5);
149
+ this.zz(v);
150
+ return this;
151
+ }
152
+ i64(id, v) {
153
+ this.field(id, 6);
154
+ this.zz(v);
155
+ return this;
156
+ }
157
+ bin(id, v) {
158
+ const b = typeof v === "string" ? Buffer.from(v, "utf8") : v;
159
+ this.field(id, 8);
160
+ this.varint(b.length);
161
+ for (const x of b)
162
+ this.out.push(x);
163
+ return this;
164
+ }
165
+ struct(id, body) {
166
+ this.field(id, 12);
167
+ this.last.push(0);
168
+ body(this);
169
+ this.raw(0);
170
+ this.last.pop();
171
+ return this;
172
+ }
173
+ listHead(n, et) {
174
+ if (n < 15)
175
+ this.raw((n << 4) | et);
176
+ else {
177
+ this.raw(0xf0 | et);
178
+ this.varint(n);
179
+ }
180
+ }
181
+ listI32(id, vals) {
182
+ this.field(id, 9);
183
+ this.listHead(vals.length, 5);
184
+ for (const v of vals)
185
+ this.zz(v);
186
+ return this;
187
+ }
188
+ listBin(id, vals) {
189
+ this.field(id, 9);
190
+ this.listHead(vals.length, 8);
191
+ for (const v of vals) {
192
+ const b = Buffer.from(v, "utf8");
193
+ this.varint(b.length);
194
+ for (const x of b)
195
+ this.out.push(x);
196
+ }
197
+ return this;
198
+ }
199
+ listStruct(id, items) {
200
+ this.field(id, 9);
201
+ this.listHead(items.length, 12);
202
+ for (const body of items) {
203
+ this.last.push(0);
204
+ body(this);
205
+ this.raw(0);
206
+ this.last.pop();
207
+ }
208
+ return this;
209
+ }
210
+ end() {
211
+ this.raw(0);
212
+ return Buffer.from(this.out);
213
+ }
214
+ }
215
+ // ---------------------------------------------------------------------- codecs
216
+ function snappy(src) {
217
+ let pos = 0, len = 0, mul = 1;
218
+ for (;;) {
219
+ const b = src[pos++];
220
+ len += (b & 0x7f) * mul;
221
+ if (!(b & 0x80))
222
+ break;
223
+ mul *= 128;
224
+ }
225
+ const out = Buffer.allocUnsafe(len);
226
+ let o = 0;
227
+ while (pos < src.length) {
228
+ const tag = src[pos++];
229
+ const t = tag & 3;
230
+ if (t === 0) {
231
+ let l = tag >> 2;
232
+ if (l >= 60) {
233
+ const nb = l - 59;
234
+ l = 0;
235
+ for (let i = 0; i < nb; i++)
236
+ l += src[pos + i] * 2 ** (8 * i);
237
+ pos += nb;
238
+ }
239
+ l += 1;
240
+ src.copy(out, o, pos, pos + l);
241
+ pos += l;
242
+ o += l;
243
+ continue;
244
+ }
245
+ let l, off;
246
+ if (t === 1) {
247
+ l = ((tag >> 2) & 7) + 4;
248
+ off = ((tag >> 5) << 8) | src[pos++];
249
+ }
250
+ else if (t === 2) {
251
+ l = (tag >> 2) + 1;
252
+ off = src.readUInt16LE(pos);
253
+ pos += 2;
254
+ }
255
+ else {
256
+ l = (tag >> 2) + 1;
257
+ off = src.readUInt32LE(pos);
258
+ pos += 4;
259
+ }
260
+ if (!off || off > o)
261
+ throw new Error("corrupt Snappy data");
262
+ for (let i = 0; i < l; i++)
263
+ out[o + i] = out[o - off + i];
264
+ o += l;
265
+ }
266
+ return out.subarray(0, o);
267
+ }
268
+ function lz4Raw(src, size) {
269
+ const out = Buffer.allocUnsafe(size);
270
+ let pos = 0, o = 0;
271
+ while (pos < src.length) {
272
+ const token = src[pos++];
273
+ let lit = token >> 4;
274
+ if (lit === 15) {
275
+ let b;
276
+ do {
277
+ b = src[pos++];
278
+ lit += b;
279
+ } while (b === 255);
280
+ }
281
+ src.copy(out, o, pos, pos + lit);
282
+ pos += lit;
283
+ o += lit;
284
+ if (pos >= src.length)
285
+ break;
286
+ const off = src[pos] | (src[pos + 1] << 8);
287
+ pos += 2;
288
+ let ml = token & 15;
289
+ if (ml === 15) {
290
+ let b;
291
+ do {
292
+ b = src[pos++];
293
+ ml += b;
294
+ } while (b === 255);
295
+ }
296
+ ml += 4;
297
+ for (let i = 0; i < ml; i++)
298
+ out[o + i] = out[o - off + i];
299
+ o += ml;
300
+ }
301
+ return out.subarray(0, o);
302
+ }
303
+ const CODECS = ["UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "BROTLI", "LZ4", "ZSTD", "LZ4_RAW"];
304
+ function decompress(codec, data, size) {
305
+ switch (codec) {
306
+ case 0: return data;
307
+ case 1: return snappy(data);
308
+ case 2: return zlib.gunzipSync(data);
309
+ case 4: return zlib.brotliDecompressSync(data);
310
+ case 6: return zlib.zstdDecompressSync(data);
311
+ case 7: return lz4Raw(data, size);
312
+ default: throw new Error(`Parquet codec ${CODECS[codec] ?? codec} is not supported here`);
313
+ }
314
+ }
315
+ // ---------------------------------------------------------------------- level and value decoders
316
+ function uvarint(b, st) {
317
+ let n = 0, mul = 1;
318
+ for (;;) {
319
+ const x = b[st.pos++];
320
+ n += (x & 0x7f) * mul;
321
+ if (!(x & 0x80))
322
+ return n;
323
+ mul *= 128;
324
+ }
325
+ }
326
+ /** Read `count` values of `width` bits, LSB first, starting at byte `pos`. */
327
+ function unpack(b, pos, width, count, out, at) {
328
+ if (width === 0) {
329
+ for (let i = 0; i < count; i++)
330
+ out[at + i] = 0;
331
+ return;
332
+ }
333
+ let bit = pos * 8;
334
+ for (let i = 0; i < count; i++) {
335
+ let v = 0;
336
+ for (let k = 0; k < width;) {
337
+ const byte = b[bit >> 3] ?? 0;
338
+ const off = bit & 7;
339
+ const take = Math.min(8 - off, width - k);
340
+ v += ((byte >> off) & ((1 << take) - 1)) * 2 ** k;
341
+ k += take;
342
+ bit += take;
343
+ }
344
+ out[at + i] = v;
345
+ }
346
+ }
347
+ /** The RLE / bit-packing hybrid encoding (definition levels, dictionary indexes, booleans). */
348
+ function rleHybrid(b, pos, end, width, count) {
349
+ const out = new Int32Array(count);
350
+ const st = { pos };
351
+ const bw = (width + 7) >> 3;
352
+ let i = 0;
353
+ while (i < count && st.pos < end) {
354
+ const h = uvarint(b, st);
355
+ if (h & 1) {
356
+ const n = (h >>> 1) * 8;
357
+ const tmp = new Array(n);
358
+ unpack(b, st.pos, width, n, tmp, 0);
359
+ st.pos += (h >>> 1) * width;
360
+ for (let k = 0; k < n && i < count; k++)
361
+ out[i++] = tmp[k];
362
+ }
363
+ else {
364
+ const run = h >>> 1;
365
+ let v = 0;
366
+ for (let k = 0; k < bw; k++)
367
+ v += b[st.pos + k] * 2 ** (8 * k);
368
+ st.pos += bw;
369
+ for (let k = 0; k < run && i < count; k++)
370
+ out[i++] = v;
371
+ }
372
+ }
373
+ return out;
374
+ }
375
+ /** DELTA_BINARY_PACKED: the values and the position after them. */
376
+ function deltaBinary(b, pos) {
377
+ const st = { pos };
378
+ const block = uvarint(b, st), minis = uvarint(b, st), total = uvarint(b, st);
379
+ const zz = (n) => (n % 2 ? -(n + 1) / 2 : n / 2);
380
+ const out = [];
381
+ if (!total)
382
+ return [out, st.pos];
383
+ let v = zz(uvarint(b, st));
384
+ out.push(v);
385
+ const per = block / minis;
386
+ while (out.length < total) {
387
+ const minDelta = zz(uvarint(b, st));
388
+ const widths = b.subarray(st.pos, st.pos + minis);
389
+ st.pos += minis;
390
+ for (let m = 0; m < minis && out.length < total; m++) {
391
+ const w = widths[m];
392
+ const tmp = new Array(per);
393
+ unpack(b, st.pos, w, per, tmp, 0);
394
+ st.pos += (per * w) / 8;
395
+ for (let k = 0; k < per && out.length < total; k++) {
396
+ v += minDelta + tmp[k];
397
+ out.push(v);
398
+ }
399
+ }
400
+ }
401
+ return [out, st.pos];
402
+ }
403
+ function plainValues(col, b, pos, count) {
404
+ const out = new Array(count);
405
+ switch (col.type) {
406
+ case 0: // BOOLEAN, bit-packed
407
+ for (let i = 0; i < count; i++)
408
+ out[i] = ((b[pos + (i >> 3)] >> (i & 7)) & 1) === 1;
409
+ return [out, pos + Math.ceil(count / 8)];
410
+ case 1:
411
+ for (let i = 0; i < count; i++)
412
+ out[i] = b.readInt32LE(pos + 4 * i);
413
+ return [out, pos + 4 * count];
414
+ case 2:
415
+ for (let i = 0; i < count; i++)
416
+ out[i] = b.readBigInt64LE(pos + 8 * i);
417
+ return [out, pos + 8 * count];
418
+ case 4:
419
+ for (let i = 0; i < count; i++)
420
+ out[i] = b.readFloatLE(pos + 4 * i);
421
+ return [out, pos + 4 * count];
422
+ case 5:
423
+ for (let i = 0; i < count; i++)
424
+ out[i] = b.readDoubleLE(pos + 8 * i);
425
+ return [out, pos + 8 * count];
426
+ case 6:
427
+ for (let i = 0; i < count; i++) {
428
+ const n = b.readUInt32LE(pos);
429
+ out[i] = b.subarray(pos + 4, pos + 4 + n);
430
+ pos += 4 + n;
431
+ }
432
+ return [out, pos];
433
+ case 7: {
434
+ const n = col.typeLength || 0;
435
+ for (let i = 0; i < count; i++)
436
+ out[i] = b.subarray(pos + n * i, pos + n * (i + 1));
437
+ return [out, pos + n * count];
438
+ }
439
+ default:
440
+ throw new Error(`Parquet column ${col.name}: physical type ${col.type} is not supported here`);
441
+ }
442
+ }
443
+ function byteStreamSplit(col, b, pos, count) {
444
+ const w = col.type === 1 || col.type === 4 ? 4 : col.type === 7 ? col.typeLength || 0 : 8;
445
+ const raw = Buffer.allocUnsafe(w * count);
446
+ for (let k = 0; k < w; k++)
447
+ for (let i = 0; i < count; i++)
448
+ raw[i * w + k] = b[pos + k * count + i];
449
+ return plainValues(col, raw, 0, count)[0];
450
+ }
451
+ function decodeValues(col, enc, b, pos, end, count, dict) {
452
+ switch (enc) {
453
+ case 0: return plainValues(col, b, pos, count)[0];
454
+ case 2:
455
+ case 8: {
456
+ if (!dict)
457
+ throw new Error(`Parquet column ${col.name}: dictionary page missing`);
458
+ const width = b[pos];
459
+ const idx = rleHybrid(b, pos + 1, end, width, count);
460
+ const out = new Array(count);
461
+ for (let i = 0; i < count; i++)
462
+ out[i] = dict[idx[i]];
463
+ return out;
464
+ }
465
+ case 3: { // RLE booleans: 4-byte length, then the hybrid encoding at width 1
466
+ const idx = rleHybrid(b, pos + 4, end, 1, count);
467
+ return Array.from(idx, (x) => x === 1);
468
+ }
469
+ case 5: {
470
+ const [vals] = deltaBinary(b, pos);
471
+ return col.type === 2 ? vals.map((v) => BigInt(v)) : vals;
472
+ }
473
+ case 6: {
474
+ const [lens, p] = deltaBinary(b, pos);
475
+ const out = new Array(count);
476
+ let q = p;
477
+ for (let i = 0; i < count; i++) {
478
+ out[i] = b.subarray(q, q + lens[i]);
479
+ q += lens[i];
480
+ }
481
+ return out;
482
+ }
483
+ case 7: {
484
+ const [pre, p1] = deltaBinary(b, pos);
485
+ const [lens, p2] = deltaBinary(b, p1);
486
+ const out = new Array(count);
487
+ let q = p2;
488
+ let prev = Buffer.alloc(0);
489
+ for (let i = 0; i < count; i++) {
490
+ const v = Buffer.concat([prev.subarray(0, pre[i]), b.subarray(q, q + lens[i])]);
491
+ q += lens[i];
492
+ out[i] = v;
493
+ prev = v;
494
+ }
495
+ return out;
496
+ }
497
+ case 9: return byteStreamSplit(col, b, pos, count);
498
+ default: throw new Error(`Parquet column ${col.name}: encoding ${enc} is not supported here`);
499
+ }
500
+ }
501
+ /** A raw value as pandas astype("string") shows it. */
502
+ function show(col, v) {
503
+ if (Buffer.isBuffer(v))
504
+ return v.toString("utf8");
505
+ if (typeof v === "boolean")
506
+ return v ? "True" : "False";
507
+ if (typeof v === "bigint")
508
+ return v.toString();
509
+ if (typeof v === "number")
510
+ return col.type === 4 || col.type === 5 ? pyFloatRepr(v) : String(v);
511
+ return String(v);
512
+ }
513
+ // ---------------------------------------------------------------------- reader
514
+ export class ParquetFile {
515
+ path;
516
+ buf;
517
+ meta;
518
+ cols = [];
519
+ numRows;
520
+ constructor(path) {
521
+ this.path = path;
522
+ this.buf = fs.readFileSync(path);
523
+ const n = this.buf.length;
524
+ if (n < 12 || this.buf.toString("latin1", 0, 4) !== "PAR1" || this.buf.toString("latin1", n - 4) !== "PAR1") {
525
+ throw new Error(`${path} is not a Parquet file`);
526
+ }
527
+ const len = this.buf.readUInt32LE(n - 8);
528
+ this.meta = new TReader(this.buf, n - 8 - len).struct();
529
+ const schema = this.meta[2] || [];
530
+ for (const el of schema.slice(1)) {
531
+ if (el[5])
532
+ throw new Error(`${path}: nested Parquet columns are not supported here`);
533
+ if (el[3] === 2)
534
+ throw new Error(`${path}: repeated Parquet columns are not supported here`);
535
+ this.cols.push({ name: el[4].toString("utf8"), type: el[1], converted: el[6], logical: el[10], maxDef: el[3] === 1 ? 1 : 0,
536
+ typeLength: el[2] });
537
+ }
538
+ this.numRows = Number(this.meta[3] || 0);
539
+ }
540
+ columns() {
541
+ return this.cols.map((c) => c.name);
542
+ }
543
+ /** One table per row group: {column: values}, only the named columns (all when omitted). */
544
+ *rowGroups(names) {
545
+ const want = new Set(names ?? this.columns());
546
+ for (const rg of (this.meta[4] || [])) {
547
+ const rows = Number(rg[3]);
548
+ const data = {};
549
+ for (const cc of rg[1]) {
550
+ const md = cc[3];
551
+ const name = md[3].map((x) => x.toString("utf8")).join(".");
552
+ if (!want.has(name))
553
+ continue;
554
+ const col = this.cols.find((c) => c.name === name);
555
+ data[name] = this.readChunk(col, md, rows);
556
+ }
557
+ yield { rows, data };
558
+ }
559
+ }
560
+ readChunk(col, md, rows) {
561
+ const codec = md[4];
562
+ const dpo = Number(md[9]);
563
+ const dict0 = md[11] !== undefined ? Number(md[11]) : 0;
564
+ let pos = dict0 > 0 && dict0 < dpo ? dict0 : dpo;
565
+ const end = pos + Number(md[7]);
566
+ const total = Number(md[5]);
567
+ const out = new Array(rows);
568
+ let dict = null;
569
+ let got = 0;
570
+ while (got < total && pos < end) {
571
+ const r = new TReader(this.buf, pos);
572
+ const h = r.struct();
573
+ pos = r.pos;
574
+ const csize = h[3], usize = h[2];
575
+ const page = this.buf.subarray(pos, pos + csize);
576
+ pos += csize;
577
+ if (h[1] === 2) { // dictionary page
578
+ const dh = h[7];
579
+ const b = decompress(codec, page, usize);
580
+ dict = plainValues(col, b, 0, dh[1])[0].map((v) => show(col, v));
581
+ continue;
582
+ }
583
+ let defs = null;
584
+ let b, vpos, n, enc;
585
+ if (h[1] === 0) {
586
+ const dh = h[5];
587
+ n = dh[1];
588
+ enc = dh[2];
589
+ b = decompress(codec, page, usize);
590
+ vpos = 0;
591
+ if (col.maxDef > 0) {
592
+ const l = b.readUInt32LE(0);
593
+ defs = rleHybrid(b, 4, 4 + l, 1, n);
594
+ vpos = 4 + l;
595
+ }
596
+ }
597
+ else if (h[1] === 3) {
598
+ const dh = h[8];
599
+ n = dh[1];
600
+ enc = dh[4];
601
+ const dl = dh[5] || 0, rl = dh[6] || 0;
602
+ if (col.maxDef > 0)
603
+ defs = rleHybrid(page, rl, rl + dl, 1, n);
604
+ const rest = page.subarray(rl + dl);
605
+ b = dh[7] === false || codec === 0 ? rest : decompress(codec, rest, usize - rl - dl);
606
+ vpos = 0;
607
+ }
608
+ else
609
+ continue;
610
+ let present = n;
611
+ if (defs) {
612
+ present = 0;
613
+ for (let i = 0; i < n; i++)
614
+ if (defs[i] === col.maxDef)
615
+ present++;
616
+ }
617
+ const vals = decodeValues(col, enc, b, vpos, b.length, present, enc === 2 || enc === 8 ? dict : null);
618
+ const isDict = enc === 2 || enc === 8;
619
+ let k = 0;
620
+ for (let i = 0; i < n; i++) {
621
+ if (defs && defs[i] !== col.maxDef)
622
+ out[got + i] = null;
623
+ else {
624
+ const v = vals[k++];
625
+ out[got + i] = isDict ? v : show(col, v);
626
+ }
627
+ }
628
+ got += n;
629
+ }
630
+ for (let i = got; i < rows; i++)
631
+ out[i] = null;
632
+ // pandas turns an integer column with nulls into floats ("12.0"): so does this, per row group
633
+ if ((col.type === 1 || col.type === 2) && out.some((v) => v === null)) {
634
+ for (let i = 0; i < rows; i++)
635
+ if (out[i] !== null)
636
+ out[i] = pyFloatRepr(Number(out[i]));
637
+ }
638
+ return out;
639
+ }
640
+ }
641
+ function rleLevels(defs) {
642
+ const parts = [];
643
+ const push = (n) => {
644
+ while (n >= 0x80) {
645
+ parts.push((n % 128) | 0x80);
646
+ n = Math.floor(n / 128);
647
+ }
648
+ parts.push(n);
649
+ };
650
+ let i = 0;
651
+ while (i < defs.length) {
652
+ let j = i;
653
+ while (j < defs.length && defs[j] === defs[i])
654
+ j++;
655
+ push((j - i) * 2);
656
+ parts.push(defs[i]);
657
+ i = j;
658
+ }
659
+ const body = Buffer.from(parts);
660
+ const len = Buffer.alloc(4);
661
+ len.writeUInt32LE(body.length);
662
+ return Buffer.concat([len, body]);
663
+ }
664
+ const ROW_GROUP = 1_000_000;
665
+ /** Write a flat table of optional columns (strings as UTF-8, int64, double), ZSTD-compressed. */
666
+ export function writeParquet(file, cols, rows, createdBy) {
667
+ const fd = fs.openSync(file, "w");
668
+ let off = 0;
669
+ const put = (b) => {
670
+ fs.writeSync(fd, b);
671
+ off += b.length;
672
+ };
673
+ const zstd = zlib.zstdCompressSync;
674
+ const codec = zstd ? 6 : 2;
675
+ const compress = (b) => (zstd ? zstd(b) : zlib.gzipSync(b));
676
+ try {
677
+ put(Buffer.from("PAR1", "latin1"));
678
+ const groups = [];
679
+ for (let start = 0; start < Math.max(rows, 1) && (start < rows || rows === 0); start += ROW_GROUP) {
680
+ const n = Math.min(ROW_GROUP, rows - start);
681
+ const chunks = [];
682
+ let groupBytes = 0;
683
+ for (const c of cols) {
684
+ const defs = new Uint8Array(n);
685
+ const vals = [];
686
+ for (let i = 0; i < n; i++) {
687
+ const v = c.values[start + i];
688
+ if (v === null || v === undefined || v === "" && c.kind !== "string")
689
+ continue;
690
+ defs[i] = 1;
691
+ if (c.kind === "string") {
692
+ const s = Buffer.from(String(v), "utf8");
693
+ const l = Buffer.alloc(4);
694
+ l.writeUInt32LE(s.length);
695
+ vals.push(l, s);
696
+ }
697
+ else if (c.kind === "int64") {
698
+ const x = Buffer.alloc(8);
699
+ x.writeBigInt64LE(BigInt(Math.trunc(Number(v))));
700
+ vals.push(x);
701
+ }
702
+ else {
703
+ const x = Buffer.alloc(8);
704
+ x.writeDoubleLE(Number(v));
705
+ vals.push(x);
706
+ }
707
+ }
708
+ const raw = Buffer.concat([rleLevels(defs), ...vals]);
709
+ const comp = compress(raw);
710
+ const header = new TWriter().i32(1, 0).i32(2, raw.length).i32(3, comp.length)
711
+ .struct(5, (w) => w.i32(1, n).i32(2, 0).i32(3, 3).i32(4, 3)).end();
712
+ const pageOffset = off;
713
+ put(header);
714
+ put(comp);
715
+ const usize = header.length + raw.length, csize = header.length + comp.length;
716
+ groupBytes += usize;
717
+ const type = c.kind === "string" ? 6 : c.kind === "int64" ? 2 : 5;
718
+ chunks.push((w) => w.i64(2, pageOffset).struct(3, (m) => m.i32(1, type).listI32(2, [0, 3]).listBin(3, [c.name])
719
+ .i32(4, codec).i64(5, n).i64(6, usize).i64(7, csize).i64(9, pageOffset)));
720
+ }
721
+ groups.push((w) => w.listStruct(1, chunks).i64(2, groupBytes).i64(3, n));
722
+ if (rows === 0)
723
+ break;
724
+ }
725
+ const schema = [(w) => w.bin(4, "schema").i32(5, cols.length)];
726
+ for (const c of cols) {
727
+ schema.push((w) => {
728
+ w.i32(1, c.kind === "string" ? 6 : c.kind === "int64" ? 2 : 5).i32(3, 1).bin(4, c.name);
729
+ if (c.kind === "string")
730
+ w.i32(6, 0).struct(10, (l) => l.struct(1, () => undefined));
731
+ });
732
+ }
733
+ const meta = new TWriter().i32(1, 1).listStruct(2, schema).i64(3, rows).listStruct(4, groups).bin(6, createdBy).end();
734
+ put(meta);
735
+ const len = Buffer.alloc(4);
736
+ len.writeUInt32LE(meta.length);
737
+ put(len);
738
+ put(Buffer.from("PAR1", "latin1"));
739
+ }
740
+ finally {
741
+ fs.closeSync(fd);
742
+ }
743
+ }
744
+ //# sourceMappingURL=_opm_parquet.js.map