data-analyst-mcp 0.0.0-stage → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (640) hide show
  1. package/README.md +262 -3
  2. package/dist/adapters/_acf_tidy.d.ts +18 -0
  3. package/dist/adapters/_acf_tidy.js +432 -0
  4. package/dist/adapters/_acf_tidy.js.map +1 -0
  5. package/dist/adapters/_actuaries_read.d.ts +38 -0
  6. package/dist/adapters/_actuaries_read.js +728 -0
  7. package/dist/adapters/_actuaries_read.js.map +1 -0
  8. package/dist/adapters/_census_micro.d.ts +70 -0
  9. package/dist/adapters/_census_micro.js +622 -0
  10. package/dist/adapters/_census_micro.js.map +1 -0
  11. package/dist/adapters/_census_util.d.ts +40 -0
  12. package/dist/adapters/_census_util.js +275 -0
  13. package/dist/adapters/_census_util.js.map +1 -0
  14. package/dist/adapters/_dol_states.d.ts +6 -0
  15. package/dist/adapters/_dol_states.js +58 -0
  16. package/dist/adapters/_dol_states.js.map +1 -0
  17. package/dist/adapters/_govinfo_entities.d.ts +4 -0
  18. package/dist/adapters/_govinfo_entities.js +6 -0
  19. package/dist/adapters/_govinfo_entities.js.map +1 -0
  20. package/dist/adapters/_govinfo_json.d.ts +13 -0
  21. package/dist/adapters/_govinfo_json.js +107 -0
  22. package/dist/adapters/_govinfo_json.js.map +1 -0
  23. package/dist/adapters/_govinfo_py.d.ts +22 -0
  24. package/dist/adapters/_govinfo_py.js +272 -0
  25. package/dist/adapters/_govinfo_py.js.map +1 -0
  26. package/dist/adapters/_govinfo_xml.d.ts +22 -0
  27. package/dist/adapters/_govinfo_xml.js +233 -0
  28. package/dist/adapters/_govinfo_xml.js.map +1 -0
  29. package/dist/adapters/_hud_excel.d.ts +2 -0
  30. package/dist/adapters/_hud_excel.js +166 -0
  31. package/dist/adapters/_hud_excel.js.map +1 -0
  32. package/dist/adapters/_omb_grid.d.ts +17 -0
  33. package/dist/adapters/_omb_grid.js +132 -0
  34. package/dist/adapters/_omb_grid.js.map +1 -0
  35. package/dist/adapters/_omb_util.d.ts +40 -0
  36. package/dist/adapters/_omb_util.js +192 -0
  37. package/dist/adapters/_omb_util.js.map +1 -0
  38. package/dist/adapters/_usaspending_state.d.ts +6 -0
  39. package/dist/adapters/_usaspending_state.js +54 -0
  40. package/dist/adapters/_usaspending_state.js.map +1 -0
  41. package/dist/adapters/_usda-fna_book.d.ts +59 -0
  42. package/dist/adapters/_usda-fna_book.js +869 -0
  43. package/dist/adapters/_usda-fna_book.js.map +1 -0
  44. package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
  45. package/dist/adapters/_usda-fna_pdf.js +361 -0
  46. package/dist/adapters/_usda-fna_pdf.js.map +1 -0
  47. package/dist/adapters/_usda-outlook_read.d.ts +70 -0
  48. package/dist/adapters/_usda-outlook_read.js +362 -0
  49. package/dist/adapters/_usda-outlook_read.js.map +1 -0
  50. package/dist/adapters/acf.d.ts +3 -0
  51. package/dist/adapters/acf.js +187 -0
  52. package/dist/adapters/acf.js.map +1 -0
  53. package/dist/adapters/actuaries.d.ts +3 -0
  54. package/dist/adapters/actuaries.js +293 -0
  55. package/dist/adapters/actuaries.js.map +1 -0
  56. package/dist/adapters/bts.d.ts +3 -0
  57. package/dist/adapters/bts.js +26 -0
  58. package/dist/adapters/bts.js.map +1 -0
  59. package/dist/adapters/census-geocoder.d.ts +10 -0
  60. package/dist/adapters/census-geocoder.js +47 -0
  61. package/dist/adapters/census-geocoder.js.map +1 -0
  62. package/dist/adapters/census.d.ts +17 -0
  63. package/dist/adapters/census.js +983 -0
  64. package/dist/adapters/census.js.map +1 -0
  65. package/dist/adapters/cfpb.d.ts +7 -0
  66. package/dist/adapters/cfpb.js +348 -0
  67. package/dist/adapters/cfpb.js.map +1 -0
  68. package/dist/adapters/clinical-trials.d.ts +3 -0
  69. package/dist/adapters/clinical-trials.js +135 -0
  70. package/dist/adapters/clinical-trials.js.map +1 -0
  71. package/dist/adapters/cms.d.ts +3 -0
  72. package/dist/adapters/cms.js +292 -0
  73. package/dist/adapters/cms.js.map +1 -0
  74. package/dist/adapters/college-scorecard.d.ts +3 -0
  75. package/dist/adapters/college-scorecard.js +122 -0
  76. package/dist/adapters/college-scorecard.js.map +1 -0
  77. package/dist/adapters/congress.d.ts +33 -0
  78. package/dist/adapters/congress.js +589 -0
  79. package/dist/adapters/congress.js.map +1 -0
  80. package/dist/adapters/dhs-ohss.d.ts +13 -0
  81. package/dist/adapters/dhs-ohss.js +598 -0
  82. package/dist/adapters/dhs-ohss.js.map +1 -0
  83. package/dist/adapters/doj-news.d.ts +7 -0
  84. package/dist/adapters/doj-news.js +286 -0
  85. package/dist/adapters/doj-news.js.map +1 -0
  86. package/dist/adapters/dol.d.ts +4 -0
  87. package/dist/adapters/dol.js +634 -0
  88. package/dist/adapters/dol.js.map +1 -0
  89. package/dist/adapters/eia.d.ts +15 -0
  90. package/dist/adapters/eia.js +456 -0
  91. package/dist/adapters/eia.js.map +1 -0
  92. package/dist/adapters/epa-aqs.d.ts +3 -0
  93. package/dist/adapters/epa-aqs.js +357 -0
  94. package/dist/adapters/epa-aqs.js.map +1 -0
  95. package/dist/adapters/epa.d.ts +3 -0
  96. package/dist/adapters/epa.js +298 -0
  97. package/dist/adapters/epa.js.map +1 -0
  98. package/dist/adapters/fbi.d.ts +3 -0
  99. package/dist/adapters/fbi.js +192 -0
  100. package/dist/adapters/fbi.js.map +1 -0
  101. package/dist/adapters/fda.d.ts +5 -0
  102. package/dist/adapters/fda.js +184 -0
  103. package/dist/adapters/fda.js.map +1 -0
  104. package/dist/adapters/fdic.d.ts +4 -0
  105. package/dist/adapters/fdic.js +95 -0
  106. package/dist/adapters/fdic.js.map +1 -0
  107. package/dist/adapters/fec.d.ts +3 -0
  108. package/dist/adapters/fec.js +142 -0
  109. package/dist/adapters/fec.js.map +1 -0
  110. package/dist/adapters/fed-board.d.ts +12 -0
  111. package/dist/adapters/fed-board.js +581 -0
  112. package/dist/adapters/fed-board.js.map +1 -0
  113. package/dist/adapters/federal-register.d.ts +11 -0
  114. package/dist/adapters/federal-register.js +43 -0
  115. package/dist/adapters/federal-register.js.map +1 -0
  116. package/dist/adapters/fhfa.d.ts +3 -0
  117. package/dist/adapters/fhfa.js +253 -0
  118. package/dist/adapters/fhfa.js.map +1 -0
  119. package/dist/adapters/fsa.d.ts +9 -0
  120. package/dist/adapters/fsa.js +335 -0
  121. package/dist/adapters/fsa.js.map +1 -0
  122. package/dist/adapters/govinfo.d.ts +30 -0
  123. package/dist/adapters/govinfo.js +636 -0
  124. package/dist/adapters/govinfo.js.map +1 -0
  125. package/dist/adapters/gsa-calc.d.ts +10 -0
  126. package/dist/adapters/gsa-calc.js +318 -0
  127. package/dist/adapters/gsa-calc.js.map +1 -0
  128. package/dist/adapters/hud.d.ts +3 -0
  129. package/dist/adapters/hud.js +200 -0
  130. package/dist/adapters/hud.js.map +1 -0
  131. package/dist/adapters/index.d.ts +3 -0
  132. package/dist/adapters/index.js +25 -0
  133. package/dist/adapters/index.js.map +1 -0
  134. package/dist/adapters/irs-soi.d.ts +6 -0
  135. package/dist/adapters/irs-soi.js +576 -0
  136. package/dist/adapters/irs-soi.js.map +1 -0
  137. package/dist/adapters/naep.d.ts +15 -0
  138. package/dist/adapters/naep.js +341 -0
  139. package/dist/adapters/naep.js.map +1 -0
  140. package/dist/adapters/nhtsa.d.ts +10 -0
  141. package/dist/adapters/nhtsa.js +239 -0
  142. package/dist/adapters/nhtsa.js.map +1 -0
  143. package/dist/adapters/nih.d.ts +5 -0
  144. package/dist/adapters/nih.js +196 -0
  145. package/dist/adapters/nih.js.map +1 -0
  146. package/dist/adapters/noaa.d.ts +3 -0
  147. package/dist/adapters/noaa.js +148 -0
  148. package/dist/adapters/noaa.js.map +1 -0
  149. package/dist/adapters/nrel.d.ts +3 -0
  150. package/dist/adapters/nrel.js +186 -0
  151. package/dist/adapters/nrel.js.map +1 -0
  152. package/dist/adapters/nws.d.ts +3 -0
  153. package/dist/adapters/nws.js +119 -0
  154. package/dist/adapters/nws.js.map +1 -0
  155. package/dist/adapters/omb.d.ts +15 -0
  156. package/dist/adapters/omb.js +517 -0
  157. package/dist/adapters/omb.js.map +1 -0
  158. package/dist/adapters/open-payments.d.ts +5 -0
  159. package/dist/adapters/open-payments.js +149 -0
  160. package/dist/adapters/open-payments.js.map +1 -0
  161. package/dist/adapters/pbgc.d.ts +13 -0
  162. package/dist/adapters/pbgc.js +283 -0
  163. package/dist/adapters/pbgc.js.map +1 -0
  164. package/dist/adapters/philfed.d.ts +12 -0
  165. package/dist/adapters/philfed.js +59 -0
  166. package/dist/adapters/philfed.js.map +1 -0
  167. package/dist/adapters/regulations.d.ts +6 -0
  168. package/dist/adapters/regulations.js +58 -0
  169. package/dist/adapters/regulations.js.map +1 -0
  170. package/dist/adapters/sec.d.ts +13 -0
  171. package/dist/adapters/sec.js +504 -0
  172. package/dist/adapters/sec.js.map +1 -0
  173. package/dist/adapters/senate-lobbying.d.ts +3 -0
  174. package/dist/adapters/senate-lobbying.js +181 -0
  175. package/dist/adapters/senate-lobbying.js.map +1 -0
  176. package/dist/adapters/snapqc.d.ts +32 -0
  177. package/dist/adapters/snapqc.js +701 -0
  178. package/dist/adapters/snapqc.js.map +1 -0
  179. package/dist/adapters/treasury-yields.d.ts +3 -0
  180. package/dist/adapters/treasury-yields.js +170 -0
  181. package/dist/adapters/treasury-yields.js.map +1 -0
  182. package/dist/adapters/usaspending.d.ts +3 -0
  183. package/dist/adapters/usaspending.js +340 -0
  184. package/dist/adapters/usaspending.js.map +1 -0
  185. package/dist/adapters/usda-fna.d.ts +7 -0
  186. package/dist/adapters/usda-fna.js +707 -0
  187. package/dist/adapters/usda-fna.js.map +1 -0
  188. package/dist/adapters/usda-fooddata.d.ts +3 -0
  189. package/dist/adapters/usda-fooddata.js +101 -0
  190. package/dist/adapters/usda-fooddata.js.map +1 -0
  191. package/dist/adapters/usda-nass.d.ts +3 -0
  192. package/dist/adapters/usda-nass.js +108 -0
  193. package/dist/adapters/usda-nass.js.map +1 -0
  194. package/dist/adapters/usda-outlook.d.ts +12 -0
  195. package/dist/adapters/usda-outlook.js +696 -0
  196. package/dist/adapters/usda-outlook.js.map +1 -0
  197. package/dist/adapters/usgs.d.ts +9 -0
  198. package/dist/adapters/usgs.js +265 -0
  199. package/dist/adapters/usgs.js.map +1 -0
  200. package/dist/adapters/usitc.d.ts +18 -0
  201. package/dist/adapters/usitc.js +170 -0
  202. package/dist/adapters/usitc.js.map +1 -0
  203. package/dist/adapters/uspto.d.ts +3 -0
  204. package/dist/adapters/uspto.js +47 -0
  205. package/dist/adapters/uspto.js.map +1 -0
  206. package/dist/adapters/world-bank.d.ts +3 -0
  207. package/dist/adapters/world-bank.js +163 -0
  208. package/dist/adapters/world-bank.js.map +1 -0
  209. package/dist/analysis/cite.d.ts +31 -0
  210. package/dist/analysis/cite.js +229 -0
  211. package/dist/analysis/cite.js.map +1 -0
  212. package/dist/analysis/combine.d.ts +27 -0
  213. package/dist/analysis/combine.js +198 -0
  214. package/dist/analysis/combine.js.map +1 -0
  215. package/dist/analysis/compute.d.ts +62 -0
  216. package/dist/analysis/compute.js +583 -0
  217. package/dist/analysis/compute.js.map +1 -0
  218. package/dist/analysis/expr.d.ts +69 -0
  219. package/dist/analysis/expr.js +780 -0
  220. package/dist/analysis/expr.js.map +1 -0
  221. package/dist/analysis/factfmt.d.ts +16 -0
  222. package/dist/analysis/factfmt.js +187 -0
  223. package/dist/analysis/factfmt.js.map +1 -0
  224. package/dist/analysis/facts.d.ts +63 -0
  225. package/dist/analysis/facts.js +355 -0
  226. package/dist/analysis/facts.js.map +1 -0
  227. package/dist/analysis/handoff.d.ts +58 -0
  228. package/dist/analysis/handoff.js +208 -0
  229. package/dist/analysis/handoff.js.map +1 -0
  230. package/dist/analysis/pyfmt.d.ts +16 -0
  231. package/dist/analysis/pyfmt.js +156 -0
  232. package/dist/analysis/pyfmt.js.map +1 -0
  233. package/dist/analysis/rerun.d.ts +13 -0
  234. package/dist/analysis/rerun.js +125 -0
  235. package/dist/analysis/rerun.js.map +1 -0
  236. package/dist/analysis/table.d.ts +18 -0
  237. package/dist/analysis/table.js +109 -0
  238. package/dist/analysis/table.js.map +1 -0
  239. package/dist/analysis/tools.d.ts +67 -0
  240. package/dist/analysis/tools.js +217 -0
  241. package/dist/analysis/tools.js.map +1 -0
  242. package/dist/analysis/write_table.d.ts +13 -0
  243. package/dist/analysis/write_table.js +59 -0
  244. package/dist/analysis/write_table.js.map +1 -0
  245. package/dist/bin/cli.d.ts +10 -0
  246. package/dist/bin/cli.js +177 -0
  247. package/dist/bin/cli.js.map +1 -0
  248. package/dist/core/cache.d.ts +14 -0
  249. package/dist/core/cache.js +90 -0
  250. package/dist/core/cache.js.map +1 -0
  251. package/dist/core/context.d.ts +12 -0
  252. package/dist/core/context.js +15 -0
  253. package/dist/core/context.js.map +1 -0
  254. package/dist/core/env.d.ts +14 -0
  255. package/dist/core/env.js +52 -0
  256. package/dist/core/env.js.map +1 -0
  257. package/dist/core/generic-rest.d.ts +61 -0
  258. package/dist/core/generic-rest.js +639 -0
  259. package/dist/core/generic-rest.js.map +1 -0
  260. package/dist/core/http.d.ts +120 -0
  261. package/dist/core/http.js +514 -0
  262. package/dist/core/http.js.map +1 -0
  263. package/dist/core/keys.d.ts +43 -0
  264. package/dist/core/keys.js +174 -0
  265. package/dist/core/keys.js.map +1 -0
  266. package/dist/core/manual.d.ts +41 -0
  267. package/dist/core/manual.js +218 -0
  268. package/dist/core/manual.js.map +1 -0
  269. package/dist/core/package.d.ts +58 -0
  270. package/dist/core/package.js +228 -0
  271. package/dist/core/package.js.map +1 -0
  272. package/dist/core/packs.d.ts +139 -0
  273. package/dist/core/packs.js +173 -0
  274. package/dist/core/packs.js.map +1 -0
  275. package/dist/core/policy.d.ts +69 -0
  276. package/dist/core/policy.js +189 -0
  277. package/dist/core/policy.js.map +1 -0
  278. package/dist/core/py.d.ts +40 -0
  279. package/dist/core/py.js +171 -0
  280. package/dist/core/py.js.map +1 -0
  281. package/dist/core/routing.d.ts +10 -0
  282. package/dist/core/routing.js +39 -0
  283. package/dist/core/routing.js.map +1 -0
  284. package/dist/core/tabular.d.ts +21 -0
  285. package/dist/core/tabular.js +229 -0
  286. package/dist/core/tabular.js.map +1 -0
  287. package/dist/deep/_bea_api.d.ts +102 -0
  288. package/dist/deep/_bea_api.js +457 -0
  289. package/dist/deep/_bea_api.js.map +1 -0
  290. package/dist/deep/_bea_fetch.d.ts +151 -0
  291. package/dist/deep/_bea_fetch.js +1274 -0
  292. package/dist/deep/_bea_fetch.js.map +1 -0
  293. package/dist/deep/_bea_util.d.ts +14 -0
  294. package/dist/deep/_bea_util.js +156 -0
  295. package/dist/deep/_bea_util.js.map +1 -0
  296. package/dist/deep/_bls_api.d.ts +84 -0
  297. package/dist/deep/_bls_api.js +231 -0
  298. package/dist/deep/_bls_api.js.map +1 -0
  299. package/dist/deep/_bls_ids.d.ts +43 -0
  300. package/dist/deep/_bls_ids.js +188 -0
  301. package/dist/deep/_bls_ids.js.map +1 -0
  302. package/dist/deep/_bls_lib.d.ts +61 -0
  303. package/dist/deep/_bls_lib.js +442 -0
  304. package/dist/deep/_bls_lib.js.map +1 -0
  305. package/dist/deep/_fred_api.d.ts +148 -0
  306. package/dist/deep/_fred_api.js +545 -0
  307. package/dist/deep/_fred_api.js.map +1 -0
  308. package/dist/deep/_fred_pack.d.ts +162 -0
  309. package/dist/deep/_fred_pack.js +1027 -0
  310. package/dist/deep/_fred_pack.js.map +1 -0
  311. package/dist/deep/_fred_py.d.ts +40 -0
  312. package/dist/deep/_fred_py.js +199 -0
  313. package/dist/deep/_fred_py.js.map +1 -0
  314. package/dist/deep/_ipums_bundle.d.ts +29 -0
  315. package/dist/deep/_ipums_bundle.js +194 -0
  316. package/dist/deep/_ipums_bundle.js.map +1 -0
  317. package/dist/deep/_ipums_ddi.d.ts +23 -0
  318. package/dist/deep/_ipums_ddi.js +162 -0
  319. package/dist/deep/_ipums_ddi.js.map +1 -0
  320. package/dist/deep/_opm_io.d.ts +26 -0
  321. package/dist/deep/_opm_io.js +383 -0
  322. package/dist/deep/_opm_io.js.map +1 -0
  323. package/dist/deep/_opm_parquet.d.ts +31 -0
  324. package/dist/deep/_opm_parquet.js +744 -0
  325. package/dist/deep/_opm_parquet.js.map +1 -0
  326. package/dist/deep/_opm_py.d.ts +6 -0
  327. package/dist/deep/_opm_py.js +74 -0
  328. package/dist/deep/_opm_py.js.map +1 -0
  329. package/dist/deep/_treasury_api.d.ts +48 -0
  330. package/dist/deep/_treasury_api.js +221 -0
  331. package/dist/deep/_treasury_api.js.map +1 -0
  332. package/dist/deep/_treasury_context.d.ts +34 -0
  333. package/dist/deep/_treasury_context.js +142 -0
  334. package/dist/deep/_treasury_context.js.map +1 -0
  335. package/dist/deep/_treasury_lib.d.ts +109 -0
  336. package/dist/deep/_treasury_lib.js +686 -0
  337. package/dist/deep/_treasury_lib.js.map +1 -0
  338. package/dist/deep/bea.d.ts +3 -0
  339. package/dist/deep/bea.js +248 -0
  340. package/dist/deep/bea.js.map +1 -0
  341. package/dist/deep/bls.d.ts +3 -0
  342. package/dist/deep/bls.js +220 -0
  343. package/dist/deep/bls.js.map +1 -0
  344. package/dist/deep/fred.d.ts +3 -0
  345. package/dist/deep/fred.js +325 -0
  346. package/dist/deep/fred.js.map +1 -0
  347. package/dist/deep/ipums.d.ts +59 -0
  348. package/dist/deep/ipums.js +744 -0
  349. package/dist/deep/ipums.js.map +1 -0
  350. package/dist/deep/opm.d.ts +56 -0
  351. package/dist/deep/opm.js +1033 -0
  352. package/dist/deep/opm.js.map +1 -0
  353. package/dist/deep/treasury.d.ts +3 -0
  354. package/dist/deep/treasury.js +218 -0
  355. package/dist/deep/treasury.js.map +1 -0
  356. package/dist/envelope.d.ts +47 -0
  357. package/dist/envelope.js +88 -0
  358. package/dist/envelope.js.map +1 -0
  359. package/dist/index.d.ts +277 -0
  360. package/dist/index.js +106 -0
  361. package/dist/index.js.map +1 -0
  362. package/dist/remote/app.d.ts +21 -0
  363. package/dist/remote/app.js +218 -0
  364. package/dist/remote/app.js.map +1 -0
  365. package/dist/remote/oauth.d.ts +14 -0
  366. package/dist/remote/oauth.js +90 -0
  367. package/dist/remote/oauth.js.map +1 -0
  368. package/dist/remote/pages.d.ts +8 -0
  369. package/dist/remote/pages.js +56 -0
  370. package/dist/remote/pages.js.map +1 -0
  371. package/dist/remote/sign.d.ts +4 -0
  372. package/dist/remote/sign.js +40 -0
  373. package/dist/remote/sign.js.map +1 -0
  374. package/dist/remote/store.d.ts +31 -0
  375. package/dist/remote/store.js +129 -0
  376. package/dist/remote/store.js.map +1 -0
  377. package/dist/report/audit_answer.d.ts +94 -0
  378. package/dist/report/audit_answer.js +1349 -0
  379. package/dist/report/audit_answer.js.map +1 -0
  380. package/dist/report/audit_report.d.ts +23 -0
  381. package/dist/report/audit_report.js +637 -0
  382. package/dist/report/audit_report.js.map +1 -0
  383. package/dist/report/build_report.d.ts +27 -0
  384. package/dist/report/build_report.js +1168 -0
  385. package/dist/report/build_report.js.map +1 -0
  386. package/dist/report/method_audit.d.ts +47 -0
  387. package/dist/report/method_audit.js +710 -0
  388. package/dist/report/method_audit.js.map +1 -0
  389. package/dist/report/policyset.d.ts +50 -0
  390. package/dist/report/policyset.js +421 -0
  391. package/dist/report/policyset.js.map +1 -0
  392. package/dist/report/py.d.ts +58 -0
  393. package/dist/report/py.js +311 -0
  394. package/dist/report/py.js.map +1 -0
  395. package/dist/report/scan_secrets.d.ts +21 -0
  396. package/dist/report/scan_secrets.js +234 -0
  397. package/dist/report/scan_secrets.js.map +1 -0
  398. package/dist/report/style_check.d.ts +29 -0
  399. package/dist/report/style_check.js +173 -0
  400. package/dist/report/style_check.js.map +1 -0
  401. package/dist/report/tools.d.ts +87 -0
  402. package/dist/report/tools.js +167 -0
  403. package/dist/report/tools.js.map +1 -0
  404. package/dist/server.d.ts +5 -0
  405. package/dist/server.js +246 -0
  406. package/dist/server.js.map +1 -0
  407. package/dist/session.d.ts +47 -0
  408. package/dist/session.js +107 -0
  409. package/dist/session.js.map +1 -0
  410. package/dist/toolkit.d.ts +21 -0
  411. package/dist/toolkit.js +2 -0
  412. package/dist/toolkit.js.map +1 -0
  413. package/dist/tools.d.ts +299 -0
  414. package/dist/tools.js +491 -0
  415. package/dist/tools.js.map +1 -0
  416. package/guide/WORKFLOW.md +134 -0
  417. package/guide/assets/disclaimer.txt +1 -0
  418. package/guide/assets/download_data.ps1 +162 -0
  419. package/guide/assets/download_data.sh +143 -0
  420. package/guide/assets/report_template.html +597 -0
  421. package/guide/references/report_patterns.md +199 -0
  422. package/guide/references/routing.md +171 -0
  423. package/guide/references/style_guide.md +398 -0
  424. package/package.json +56 -6
  425. package/packs/acf/endpoints.json +114 -0
  426. package/packs/acf/policy.json +104 -0
  427. package/packs/acf/source.md +138 -0
  428. package/packs/actuaries/endpoints.json +82 -0
  429. package/packs/actuaries/policy.json +109 -0
  430. package/packs/actuaries/source.md +137 -0
  431. package/packs/bea/assets/download_bea.ps1 +182 -0
  432. package/packs/bea/assets/download_bea.sh +144 -0
  433. package/packs/bea/docs/bea_api.md +461 -0
  434. package/packs/bea/docs/datasets.md +157 -0
  435. package/packs/bea/docs/report_patterns.md +112 -0
  436. package/packs/bea/policy.json +170 -0
  437. package/packs/bea/source.md +152 -0
  438. package/packs/bls/assets/download_bls.ps1 +152 -0
  439. package/packs/bls/assets/download_bls.sh +167 -0
  440. package/packs/bls/docs/analysis_patterns.md +108 -0
  441. package/packs/bls/docs/bls_api.md +73 -0
  442. package/packs/bls/docs/series_ids.md +149 -0
  443. package/packs/bls/policy.json +238 -0
  444. package/packs/bls/series_ids.json +1375 -0
  445. package/packs/bls/source.md +156 -0
  446. package/packs/bts/endpoints.json +347 -0
  447. package/packs/bts/policy.json +94 -0
  448. package/packs/bts/source.md +97 -0
  449. package/packs/cdc/endpoints.json +197 -0
  450. package/packs/cdc/policy.json +83 -0
  451. package/packs/cdc/source.md +74 -0
  452. package/packs/census/endpoints.json +99 -0
  453. package/packs/census/policy.json +238 -0
  454. package/packs/census/source.md +304 -0
  455. package/packs/census-geocoder/endpoints.json +70 -0
  456. package/packs/census-geocoder/policy.json +80 -0
  457. package/packs/census-geocoder/source.md +53 -0
  458. package/packs/cfpb/endpoints.json +327 -0
  459. package/packs/cfpb/policy.json +119 -0
  460. package/packs/cfpb/source.md +96 -0
  461. package/packs/clinical-trials/endpoints.json +167 -0
  462. package/packs/clinical-trials/policy.json +84 -0
  463. package/packs/clinical-trials/source.md +82 -0
  464. package/packs/cms/endpoints.json +69 -0
  465. package/packs/cms/policy.json +96 -0
  466. package/packs/cms/source.md +112 -0
  467. package/packs/college-scorecard/endpoints.json +253 -0
  468. package/packs/college-scorecard/policy.json +106 -0
  469. package/packs/college-scorecard/source.md +92 -0
  470. package/packs/congress/endpoints.json +1701 -0
  471. package/packs/congress/policy.json +117 -0
  472. package/packs/congress/source.md +190 -0
  473. package/packs/dhs-ohss/endpoints.json +66 -0
  474. package/packs/dhs-ohss/policy.json +107 -0
  475. package/packs/dhs-ohss/source.md +120 -0
  476. package/packs/doj-news/endpoints.json +59 -0
  477. package/packs/doj-news/policy.json +96 -0
  478. package/packs/doj-news/source.md +99 -0
  479. package/packs/dol/endpoints.json +818 -0
  480. package/packs/dol/policy.json +135 -0
  481. package/packs/dol/source.md +181 -0
  482. package/packs/eia/endpoints.json +718 -0
  483. package/packs/eia/policy.json +112 -0
  484. package/packs/eia/source.md +112 -0
  485. package/packs/epa/endpoints.json +292 -0
  486. package/packs/epa/policy.json +99 -0
  487. package/packs/epa/source.md +99 -0
  488. package/packs/epa-aqs/endpoints.json +174 -0
  489. package/packs/epa-aqs/policy.json +96 -0
  490. package/packs/epa-aqs/source.md +97 -0
  491. package/packs/fbi/endpoints.json +1842 -0
  492. package/packs/fbi/policy.json +118 -0
  493. package/packs/fbi/source.md +108 -0
  494. package/packs/fda/endpoints.json +238 -0
  495. package/packs/fda/policy.json +121 -0
  496. package/packs/fda/source.md +117 -0
  497. package/packs/fdic/endpoints.json +417 -0
  498. package/packs/fdic/policy.json +83 -0
  499. package/packs/fdic/source.md +92 -0
  500. package/packs/fec/endpoints.json +708 -0
  501. package/packs/fec/policy.json +93 -0
  502. package/packs/fec/source.md +90 -0
  503. package/packs/fed-board/endpoints.json +89 -0
  504. package/packs/fed-board/policy.json +136 -0
  505. package/packs/fed-board/source.md +118 -0
  506. package/packs/federal-register/endpoints.json +143 -0
  507. package/packs/federal-register/policy.json +86 -0
  508. package/packs/federal-register/source.md +98 -0
  509. package/packs/fema/endpoints.json +646 -0
  510. package/packs/fema/policy.json +103 -0
  511. package/packs/fema/source.md +103 -0
  512. package/packs/fhfa/endpoints.json +105 -0
  513. package/packs/fhfa/policy.json +109 -0
  514. package/packs/fhfa/source.md +108 -0
  515. package/packs/fred/assets/download_fred.ps1 +122 -0
  516. package/packs/fred/assets/download_fred.sh +123 -0
  517. package/packs/fred/docs/analysis_patterns.md +124 -0
  518. package/packs/fred/docs/fred_api.md +94 -0
  519. package/packs/fred/docs/series.md +188 -0
  520. package/packs/fred/policy.json +253 -0
  521. package/packs/fred/source.md +142 -0
  522. package/packs/fsa/endpoints.json +150 -0
  523. package/packs/fsa/policy.json +99 -0
  524. package/packs/fsa/source.md +102 -0
  525. package/packs/govinfo/endpoints.json +520 -0
  526. package/packs/govinfo/policy.json +116 -0
  527. package/packs/govinfo/source.md +241 -0
  528. package/packs/gsa-calc/endpoints.json +93 -0
  529. package/packs/gsa-calc/policy.json +97 -0
  530. package/packs/gsa-calc/source.md +108 -0
  531. package/packs/hud/endpoints.json +147 -0
  532. package/packs/hud/policy.json +84 -0
  533. package/packs/hud/source.md +97 -0
  534. package/packs/ipums/assets/download_ipums.ps1 +215 -0
  535. package/packs/ipums/assets/download_ipums.sh +192 -0
  536. package/packs/ipums/docs/ipums_api.md +67 -0
  537. package/packs/ipums/docs/report_patterns.md +117 -0
  538. package/packs/ipums/docs/variables.md +284 -0
  539. package/packs/ipums/lib/fetch_text.json +112 -0
  540. package/packs/ipums/policy.json +215 -0
  541. package/packs/ipums/source.md +169 -0
  542. package/packs/irs-soi/endpoints.json +114 -0
  543. package/packs/irs-soi/policy.json +107 -0
  544. package/packs/irs-soi/source.md +124 -0
  545. package/packs/medicaid/endpoints.json +412 -0
  546. package/packs/medicaid/policy.json +108 -0
  547. package/packs/medicaid/source.md +136 -0
  548. package/packs/naep/endpoints.json +217 -0
  549. package/packs/naep/policy.json +100 -0
  550. package/packs/naep/source.md +120 -0
  551. package/packs/nhtsa/endpoints.json +132 -0
  552. package/packs/nhtsa/policy.json +93 -0
  553. package/packs/nhtsa/source.md +109 -0
  554. package/packs/nih/endpoints.json +453 -0
  555. package/packs/nih/policy.json +86 -0
  556. package/packs/nih/source.md +108 -0
  557. package/packs/noaa/endpoints.json +432 -0
  558. package/packs/noaa/policy.json +95 -0
  559. package/packs/noaa/source.md +88 -0
  560. package/packs/nrel/endpoints.json +328 -0
  561. package/packs/nrel/policy.json +96 -0
  562. package/packs/nrel/source.md +91 -0
  563. package/packs/nws/endpoints.json +272 -0
  564. package/packs/nws/policy.json +82 -0
  565. package/packs/nws/source.md +81 -0
  566. package/packs/nyfed/endpoints.json +80 -0
  567. package/packs/nyfed/policy.json +92 -0
  568. package/packs/nyfed/source.md +89 -0
  569. package/packs/omb/endpoints.json +460 -0
  570. package/packs/omb/policy.json +106 -0
  571. package/packs/omb/source.md +130 -0
  572. package/packs/open-payments/endpoints.json +298 -0
  573. package/packs/open-payments/policy.json +101 -0
  574. package/packs/open-payments/source.md +99 -0
  575. package/packs/opm/assets/download_opm.ps1 +125 -0
  576. package/packs/opm/assets/download_opm.sh +85 -0
  577. package/packs/opm/docs/fields.md +184 -0
  578. package/packs/opm/docs/opm_api.md +71 -0
  579. package/packs/opm/docs/report_patterns.md +83 -0
  580. package/packs/opm/lib/data_dictionary.csv +73 -0
  581. package/packs/opm/policy.json +142 -0
  582. package/packs/opm/source.md +137 -0
  583. package/packs/pbgc/endpoints.json +483 -0
  584. package/packs/pbgc/policy.json +94 -0
  585. package/packs/pbgc/source.md +95 -0
  586. package/packs/philfed/endpoints.json +85 -0
  587. package/packs/philfed/policy.json +94 -0
  588. package/packs/philfed/source.md +125 -0
  589. package/packs/port.json +287 -0
  590. package/packs/regulations/endpoints.json +288 -0
  591. package/packs/regulations/policy.json +86 -0
  592. package/packs/regulations/source.md +84 -0
  593. package/packs/sec/endpoints.json +237 -0
  594. package/packs/sec/policy.json +86 -0
  595. package/packs/sec/source.md +101 -0
  596. package/packs/senate-lobbying/endpoints.json +554 -0
  597. package/packs/senate-lobbying/policy.json +99 -0
  598. package/packs/senate-lobbying/source.md +100 -0
  599. package/packs/snapqc/endpoints.json +1426 -0
  600. package/packs/snapqc/policy.json +92 -0
  601. package/packs/snapqc/source.md +96 -0
  602. package/packs/treasury/catalog.json +2267 -0
  603. package/packs/treasury/docs/api_guide.md +245 -0
  604. package/packs/treasury/docs/datasets.md +56 -0
  605. package/packs/treasury/docs/report_patterns.md +126 -0
  606. package/packs/treasury/policy.json +159 -0
  607. package/packs/treasury/source.md +165 -0
  608. package/packs/treasury-yields/endpoints.json +226 -0
  609. package/packs/treasury-yields/policy.json +86 -0
  610. package/packs/treasury-yields/source.md +80 -0
  611. package/packs/usaspending/endpoints.json +1316 -0
  612. package/packs/usaspending/policy.json +109 -0
  613. package/packs/usaspending/source.md +103 -0
  614. package/packs/usda-fna/endpoints.json +427 -0
  615. package/packs/usda-fna/policy.json +133 -0
  616. package/packs/usda-fna/source.md +217 -0
  617. package/packs/usda-fooddata/endpoints.json +181 -0
  618. package/packs/usda-fooddata/policy.json +94 -0
  619. package/packs/usda-fooddata/source.md +105 -0
  620. package/packs/usda-nass/endpoints.json +324 -0
  621. package/packs/usda-nass/policy.json +95 -0
  622. package/packs/usda-nass/source.md +85 -0
  623. package/packs/usda-outlook/endpoints.json +86 -0
  624. package/packs/usda-outlook/policy.json +109 -0
  625. package/packs/usda-outlook/source.md +144 -0
  626. package/packs/usgs/endpoints.json +313 -0
  627. package/packs/usgs/policy.json +95 -0
  628. package/packs/usgs/source.md +96 -0
  629. package/packs/usitc/endpoints.json +74 -0
  630. package/packs/usitc/policy.json +111 -0
  631. package/packs/usitc/source.md +90 -0
  632. package/packs/uspto/endpoints.json +141 -0
  633. package/packs/uspto/policy.json +95 -0
  634. package/packs/uspto/source.md +101 -0
  635. package/packs/va/endpoints.json +414 -0
  636. package/packs/va/policy.json +101 -0
  637. package/packs/va/source.md +129 -0
  638. package/packs/world-bank/endpoints.json +122 -0
  639. package/packs/world-bank/policy.json +101 -0
  640. package/packs/world-bank/source.md +93 -0
@@ -0,0 +1,639 @@
1
+ /**
2
+ * Declarative fetching for JSON/CSV REST APIs (Socrata, DKAN, CKAN, plain REST) from a pack's endpoints.json.
3
+ * The endpoints.json format:
4
+ *
5
+ * {
6
+ * "base_url": "https://data.cdc.gov/resource", "timeout": 30, "retry_html_400": 3, "headers": {...},
7
+ * "endpoints": {
8
+ * "<name>": {
9
+ * "path": "/{dataset}.json", "method": "GET" | "POST", "description": "...",
10
+ * "params": {"dataset": {"in": "path", "required": true, "desc": "...", "catalog": true},
11
+ * // "in": "query" (GET default) | "path" | "body" (POST default) | "spec" (never sent) | "records"
12
+ * // "flag": true (a bare ?flagName, sent only when true), "raw": true (path value keeps slashes)
13
+ * "$where": {"desc": "SoQL filter"}, "state": {"desc": "...", "enum": [...]}},
14
+ * "defaults": {"$order": ":id"}, "body": {...},
15
+ * "records": "results" | "" | "$table" | "$csv" | "$xlsx" | "$xls" | "$zip:<glob>",
16
+ * "sheet": "Table 1", "header_row": 1, "total": "meta.count",
17
+ * "paging": {"type": "offset", "limit_param": "$limit", "offset_param": "$offset", "size": 1000, "manual_size": 50000}
18
+ * | {"type": "page", "page_param": "page", "size_param": "per_page", "size": 100, "start": 1, "min_size": 5}
19
+ * | {"type": "cursor", "cursor_param": "cursor", "next": "pagination.next_cursor", "size_param": "limit"}
20
+ * | {"type": "link", "next": "links.next"} | {"type": "none"},
21
+ * "max_rows": 100000, "unique_key": "appl_id", "truncated_note": {"when_default": "sort_field", "text": "..."},
22
+ * "missing": ["", "(NA)", "."], "flatten": true
23
+ * }
24
+ * },
25
+ * "catalog": {"<alias>": {"id": "bi63-dtpu", "name": "...", "description": "...", "endpoint": "<name>"}}
26
+ * }
27
+ *
28
+ * spec: {"description": "...", "requests": [{"name": "deaths", "endpoint": "query",
29
+ * "params": {"dataset": "leading_death", "$where": "year = '2017'"}, "max_rows": 5000}]}
30
+ */
31
+ import fs from "node:fs";
32
+ import path from "node:path";
33
+ import { Cache } from "./cache.js";
34
+ import { Client, DataError } from "./http.js";
35
+ import { parsePyJson, pyJsonDumps, pyQuote, pyStr, pyUrlencode, sleep } from "./py.js";
36
+ import { decodeText, fnmatch, readCsv, sheetRecords, unzip, xlsRecords } from "./tabular.js";
37
+ /** A note containing this marks its table incomplete (status partial), like a max_rows cut. */
38
+ export const INCOMPLETE = ": incomplete:";
39
+ export const FOR_EACH_MAX = 2000;
40
+ export function dig(obj, p) {
41
+ if (p === "" || p === null || p === undefined)
42
+ return obj;
43
+ for (const part of p.split(".")) {
44
+ if (obj === null || obj === undefined)
45
+ return null;
46
+ if (Array.isArray(obj)) {
47
+ const i = Number(part);
48
+ obj = Number.isInteger(i) && i >= 0 && i < obj.length ? obj[i] : null;
49
+ }
50
+ else
51
+ obj = obj && typeof obj === "object" ? obj[part] ?? null : null;
52
+ }
53
+ return obj;
54
+ }
55
+ const scalar = (x) => typeof x === "string" || typeof x === "number" || typeof x === "boolean";
56
+ export function flatten(rec, prefix = "") {
57
+ if (!rec || typeof rec !== "object" || Array.isArray(rec))
58
+ return { [prefix || "value"]: rec };
59
+ const out = {};
60
+ for (const [k, v] of Object.entries(rec)) {
61
+ const key = prefix ? `${prefix}.${k}` : k;
62
+ if (v && typeof v === "object" && !Array.isArray(v))
63
+ Object.assign(out, flatten(v, key));
64
+ else if (Array.isArray(v)) {
65
+ out[key] = v.length && !v.every(scalar) ? pyJsonDumps(v, false, false) : v.map(pyStr).join("; ");
66
+ }
67
+ else
68
+ out[key] = v;
69
+ }
70
+ return out;
71
+ }
72
+ export function clientFor(pack, eps, extra = {}) {
73
+ const P = pack.policy;
74
+ const e = eps ?? pack.endpoints ?? {};
75
+ const L = P.limits;
76
+ return new Client(e.base_url || P.routes?.api?.base || "", {
77
+ name: pack.id, auth: P.auth, rps: L.rps || 5, burst: L.burst || 10,
78
+ checkError: pack.adapter?.checkError ?? null, errorsPerMin: L.errors_per_min ?? null,
79
+ bytesPerMin: L.bytes_per_min ?? null, headers: e.headers ?? null, timeout: e.timeout ?? 30,
80
+ retryHtml400: e.retry_html_400 ?? 0,
81
+ // a persistent cache only where the source's terms allow it (cache.mode "persistent"); never for test fixtures
82
+ cache: P.cache.mode === "persistent" && !process.env.DATA_ANALYST_FIXTURES ? new Cache(pack.id, P) : null, ...extra,
83
+ });
84
+ }
85
+ // ---------------------------------------------------------------------- spec -> requests
86
+ /** [endpoint definition, merged params, problems] for one spec request. */
87
+ export function resolve(pack, req) {
88
+ const eps = pack.endpoints?.endpoints || {};
89
+ const cat = pack.endpoints?.catalog || {};
90
+ const probs = [];
91
+ let name = req.endpoint;
92
+ if (!name && req.dataset && cat[req.dataset])
93
+ name = cat[req.dataset].endpoint;
94
+ const ep = eps[name || ""];
95
+ if (!ep) {
96
+ return [{}, {}, [`request ${JSON.stringify(req.name)}: unknown endpoint ${JSON.stringify(name ?? null)} (known: ${Object.keys(eps).sort().join(", ")})`]];
97
+ }
98
+ const given = req.params || {};
99
+ const defaults = { ...(ep.defaults || {}) };
100
+ if ("$group" in given && !("$order" in given))
101
+ delete defaults.$order; // SoQL: an aggregate cannot keep a row-id order
102
+ const params = { ...defaults, ...given };
103
+ for (const [pname, d] of Object.entries(ep.params || {})) {
104
+ if (d.catalog) {
105
+ let v = pname in params ? params[pname] : req.dataset;
106
+ if (typeof v === "string" && cat[v])
107
+ v = cat[v].id;
108
+ if (v !== undefined && v !== null)
109
+ params[pname] = v;
110
+ }
111
+ const v = params[pname];
112
+ if (d.required && (v === undefined || v === null || v === "")) {
113
+ probs.push(`request ${JSON.stringify(req.name)}: parameter ${JSON.stringify(pname)} is required (${d.desc || ""})`);
114
+ }
115
+ if (d.enum && v !== undefined && v !== null && v !== "" && !d.enum.map(String).includes(pyStr(v))) {
116
+ probs.push(`request ${JSON.stringify(req.name)}: ${pname}=${JSON.stringify(v)} is not one of ${JSON.stringify(d.enum)}`);
117
+ }
118
+ }
119
+ if (!req.name || !/^[A-Za-z0-9_-]+$/.test(String(req.name))) {
120
+ probs.push("every request needs a name of letters, digits, _ or - (it names the CSV file)");
121
+ }
122
+ return [ep, params, probs];
123
+ }
124
+ /** [endpoint path with {x} filled, query params, JSON body]. */
125
+ export function split(ep, params) {
126
+ const pdefs = ep.params || {};
127
+ let p = ep.path;
128
+ const query = {};
129
+ let body = ep.body !== undefined && ep.body !== null ? structuredClone(ep.body) : null;
130
+ const flags = [];
131
+ for (const [k, v] of Object.entries(params)) {
132
+ const d = pdefs[k] || {};
133
+ const where = d.in || (ep.method === "POST" ? "body" : "query");
134
+ if (where === "spec" || where === "records")
135
+ continue;
136
+ if (d.flag) {
137
+ if (v === true || ["true", "1", "yes"].includes(String(v).trim().toLowerCase()))
138
+ flags.push(pyQuote(k));
139
+ continue;
140
+ }
141
+ if (where === "path" || p.includes(`{${k}}`))
142
+ p = p.replace(`{${k}}`, pyQuote(pyStr(v), d.raw ? "/" : ""));
143
+ else if (where === "body") {
144
+ body ??= {};
145
+ body[k] = v;
146
+ }
147
+ else
148
+ query[k] = v;
149
+ }
150
+ if (flags.length)
151
+ p += (p.includes("?") ? "&" : "?") + flags.join("&");
152
+ return [p, query, body];
153
+ }
154
+ export function checkSpec(pack, spec) {
155
+ const probs = [];
156
+ const reqs = spec.requests || [];
157
+ if (!reqs.length)
158
+ probs.push('spec needs a non-empty "requests" list');
159
+ const names = reqs.map((r) => r.name);
160
+ if (new Set(names).size !== names.length)
161
+ probs.push("request names must be unique");
162
+ for (const r of reqs)
163
+ probs.push(...resolve(pack, r)[2]);
164
+ if (pack.adapter?.checkSpec)
165
+ probs.push(...(pack.adapter.checkSpec(spec, pack) || []));
166
+ return probs;
167
+ }
168
+ // ---------------------------------------------------------------------- one request per row of a table
169
+ /** for_each requests as one request per distinct value (or combination) of a column of an earlier fetch's CSV. */
170
+ export function expandForEach(spec, base) {
171
+ const reqs = [];
172
+ for (const r of spec.requests || []) {
173
+ const fe = r.for_each;
174
+ if (!fe) {
175
+ reqs.push(r);
176
+ continue;
177
+ }
178
+ const csvPath = path.isAbsolute(fe.csv || "") ? fe.csv : path.join(base, fe.csv || "");
179
+ if (!fs.existsSync(csvPath)) {
180
+ throw new DataError(`request ${JSON.stringify(r.name)}: for_each csv ${JSON.stringify(fe.csv)} not found (fetch that ` +
181
+ "table first; the path is relative to the workspace)", "bad_request");
182
+ }
183
+ const pairs = fe.map && Object.keys(fe.map).length ? { ...fe.map } : { [fe.param || fe.column]: fe.column };
184
+ const rows = readCsv(decodeText(fs.readFileSync(csvPath)));
185
+ const cols = rows.length ? Object.keys(rows[0]) : [];
186
+ const miss = Object.values(pairs).filter((c) => !cols.includes(c));
187
+ if (miss.length) {
188
+ throw new DataError(`request ${JSON.stringify(r.name)}: for_each column(s) ${JSON.stringify(miss)} not in ${fe.csv} ` +
189
+ `(columns: ${JSON.stringify(cols)})`, "bad_request");
190
+ }
191
+ const low = !!fe.lower, pre = String(fe.strip || "");
192
+ const combos = new Map();
193
+ for (const x of rows) {
194
+ let vals = Object.values(pairs).map((c) => (low ? String(x[c] ?? "").trim().toLowerCase() : String(x[c] ?? "").trim()));
195
+ vals = vals.map((v) => (pre && v.startsWith(pre) ? v.slice(pre.length) : v));
196
+ if (vals.every(Boolean))
197
+ combos.set(JSON.stringify(vals), vals);
198
+ }
199
+ const max = Number(fe.max ?? FOR_EACH_MAX);
200
+ if (combos.size > max) {
201
+ throw new DataError(`request ${JSON.stringify(r.name)}: for_each gives ${combos.size} requests (max ${max}): filter the ` +
202
+ 'table first, or set "max"', "bad_request");
203
+ }
204
+ const names = Object.keys(pairs);
205
+ let i = 0;
206
+ for (const vals of combos.values()) {
207
+ i++;
208
+ const { for_each: _drop, ...rest } = r;
209
+ const key = names.map((n, j) => [n, vals[j]]);
210
+ reqs.push({ ...rest, name: `${r.name}__${i}`, params: { ...(r.params || {}), ...Object.fromEntries(key) },
211
+ _group: r.name, _key: key, _orig: r });
212
+ }
213
+ }
214
+ return { ...spec, requests: reqs };
215
+ }
216
+ /** The spec as written (each for_each request once). */
217
+ export function collapseForEach(spec) {
218
+ const reqs = [];
219
+ const seen = new Set();
220
+ for (const r of spec.requests || []) {
221
+ if (r._group === undefined)
222
+ reqs.push(r);
223
+ else if (!seen.has(r._group)) {
224
+ seen.add(r._group);
225
+ reqs.push(r._orig);
226
+ }
227
+ }
228
+ return { ...spec, requests: reqs };
229
+ }
230
+ /** The expanded requests' tables as one table per for_each request; each row keeps the value it was asked for. */
231
+ export function mergeForEach(spec, tables) {
232
+ const out = {};
233
+ for (const r of spec.requests || []) {
234
+ const rows = tables[r.name] || [];
235
+ if (r._group === undefined) {
236
+ out[r.name] = rows;
237
+ continue;
238
+ }
239
+ if (!Array.isArray(rows))
240
+ throw new DataError(`request '${r.name}': for_each cannot merge a streamed table`, "spec");
241
+ const asked = Object.fromEntries(r._key);
242
+ const merged = (out[r._group] ??= []);
243
+ for (const row of rows) {
244
+ const add = Object.fromEntries(Object.entries(asked).filter(([k]) => !(k in row)));
245
+ merged.push({ ...add, ...row });
246
+ }
247
+ }
248
+ const named = new Set((spec.requests || []).map((r) => r.name));
249
+ for (const [name, rows] of Object.entries(tables))
250
+ if (!named.has(name) && !(name in out))
251
+ out[name] = rows;
252
+ return out;
253
+ }
254
+ export function records(ep, resp) {
255
+ const p = ep.records ?? "";
256
+ if (p === "$csv")
257
+ return readCsv(resp.text());
258
+ if (p === "$xlsx")
259
+ return sheetRecords(ep, resp.body);
260
+ if (p === "$xls") {
261
+ if (resp.body.subarray(0, 4).equals(Buffer.from("PK\x03\x04", "latin1")))
262
+ return sheetRecords(ep, resp.body);
263
+ if (!resp.body.subarray(0, 8).equals(Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]))) {
264
+ throw new DataError("expected an .xls workbook, got something else (an HTML page?)", "api");
265
+ }
266
+ return xlsRecords(ep, resp.body);
267
+ }
268
+ if (p.startsWith("$zip:")) {
269
+ const z = unzip(resp.body);
270
+ const glob = p.slice(5).toLowerCase() || "*";
271
+ const members = Object.keys(z).filter((n) => fnmatch(n.toLowerCase(), glob) && !n.endsWith("/")).sort();
272
+ if (!members.length) {
273
+ throw new DataError(`no member matching ${JSON.stringify(p.slice(5))} in the zip (${Object.keys(z).slice(0, 10).join(", ")})`, "api");
274
+ }
275
+ const rows = [];
276
+ for (const n of members) {
277
+ if (n.toLowerCase().endsWith(".xlsx"))
278
+ rows.push(...sheetRecords(ep, z[n]));
279
+ else if (n.toLowerCase().endsWith(".xls"))
280
+ rows.push(...xlsRecords(ep, z[n]));
281
+ else
282
+ rows.push(...readCsv(decodeText(z[n])));
283
+ }
284
+ return rows;
285
+ }
286
+ const data = resp.json();
287
+ if (p === "$table") {
288
+ if (!data || !data.length)
289
+ return [];
290
+ const [head, ...rows] = data;
291
+ return rows.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i]])));
292
+ }
293
+ const recs = dig(data, p);
294
+ if (recs === null || recs === undefined)
295
+ return [];
296
+ return Array.isArray(recs) ? recs : [recs];
297
+ }
298
+ export function clean(rows, ep) {
299
+ const missing = new Set(ep.missing ?? [""]);
300
+ return rows.map((r0) => {
301
+ const r = ep.flatten ?? true ? flatten(r0) : r0 && typeof r0 === "object" ? r0 : { value: r0 };
302
+ const o = {};
303
+ for (const [k, v] of Object.entries(r)) {
304
+ o[k] = v === null || v === undefined || (typeof v === "string" && missing.has(v.trim())) ? "" : v;
305
+ }
306
+ return o;
307
+ });
308
+ }
309
+ /** The endpoint with the spec's file-reading params ("in": "records": sheet, header_row) applied. */
310
+ export function withRecordParams(ep, params) {
311
+ const pdefs = ep.params || {};
312
+ const over = Object.fromEntries(Object.entries(params).filter(([k, v]) => pdefs[k]?.in === "records" && v !== null && v !== undefined && v !== ""));
313
+ return Object.keys(over).length ? { ...ep, ...over } : ep;
314
+ }
315
+ /** All pages of one request. */
316
+ export async function fetchRequest(client, ep0, params, maxRows) {
317
+ const ep = withRecordParams(ep0, params);
318
+ const pg = ep.paging || { type: "none" };
319
+ const kind = pg.type || "none";
320
+ const limit = Math.min(Number(maxRows || ep.max_rows || 100000), Number(ep.max_rows || 1e9));
321
+ const size = Number(pg.size || 1000);
322
+ const floor = Number(pg.min_size || 1);
323
+ const [p, query, body] = split(ep, params);
324
+ const method = ep.method || "GET";
325
+ let rows = [];
326
+ const meta = { pages: 0, total: null, truncated: false };
327
+ let page = Number(pg.start ?? 1);
328
+ let offset = kind === "offset" ? Number(query[pg.offset_param || ""] || 0) || 0 : 0;
329
+ let cursor = null, nextUrl = null, prev = null;
330
+ for (;;) {
331
+ const q = { ...query };
332
+ const b = body !== null ? { ...body } : null;
333
+ const tgt = method === "POST" && (pg.in ?? "body") === "body" && b ? b : q;
334
+ const put = (k, v) => {
335
+ if (k)
336
+ tgt[k] = v;
337
+ };
338
+ const asked = kind === "offset" ? Math.min(size, limit - rows.length) : Math.min(size, limit);
339
+ if (kind === "offset") {
340
+ put(pg.limit_param, asked);
341
+ put(pg.offset_param, offset);
342
+ }
343
+ else if (kind === "page") {
344
+ put(pg.page_param, page);
345
+ put(pg.size_param, Math.max(Math.min(size, limit), floor));
346
+ }
347
+ else if (kind === "cursor") {
348
+ put(pg.size_param, Math.max(Math.min(size, limit), floor));
349
+ if (cursor)
350
+ put(pg.cursor_param, cursor);
351
+ }
352
+ const resp = nextUrl ? await client.request(method, nextUrl, null, { jsonBody: b ?? undefined })
353
+ : await client.request(method, p, q, { jsonBody: b ?? undefined });
354
+ meta.pages++;
355
+ const got = records(ep, resp);
356
+ const sig = JSON.stringify(got);
357
+ if (got.length && (kind === "offset" || kind === "page") && sig === prev) {
358
+ meta.truncated = meta.repeated_page = true;
359
+ break;
360
+ }
361
+ prev = sig;
362
+ rows = rows.concat(got);
363
+ const head = resp.body.subarray(0, 1).toString("latin1");
364
+ if (ep.total && (head === "{" || head === "[")) {
365
+ const t = Number(dig(resp.json(), ep.total));
366
+ if (Number.isFinite(t))
367
+ meta.total = Math.trunc(t);
368
+ }
369
+ if (rows.length >= limit) {
370
+ meta.truncated = (meta.total || 0) > limit || (kind !== "none" && got.length >= asked) || (kind === "none" && got.length > limit);
371
+ rows = rows.slice(0, limit);
372
+ break;
373
+ }
374
+ if (kind === "none" || !got.length)
375
+ break;
376
+ if (kind === "offset") {
377
+ if (got.length < Math.min(size, limit))
378
+ break;
379
+ offset += got.length;
380
+ }
381
+ else if (kind === "page") {
382
+ if (got.length < Math.min(size, limit) || (meta.total !== null && rows.length >= meta.total))
383
+ break;
384
+ page++;
385
+ }
386
+ else if (kind === "cursor") {
387
+ cursor = dig(resp.json(), pg.next);
388
+ if (!cursor)
389
+ break;
390
+ }
391
+ else if (kind === "link") {
392
+ nextUrl = dig(resp.json(), pg.next);
393
+ if (!nextUrl)
394
+ break;
395
+ }
396
+ if (meta.pages >= Number(pg.max_pages ?? 1000)) {
397
+ meta.truncated = true;
398
+ break;
399
+ }
400
+ }
401
+ return [clean(rows, ep), meta];
402
+ }
403
+ /** Rows repeated across page boundaries (same unique_key) are dropped, with a note that as many may be missing. */
404
+ export function dedupePages(rows, ep, meta, name) {
405
+ const col = ep.unique_key;
406
+ if (!col || meta.pages < 2 || !rows.length || rows.some((r) => !(col in r)))
407
+ return [rows, null];
408
+ const seen = new Set();
409
+ const out = rows.filter((r) => {
410
+ const k = pyStr(r[col]);
411
+ if (seen.has(k))
412
+ return false;
413
+ seen.add(k);
414
+ return true;
415
+ });
416
+ if (out.length === rows.length)
417
+ return [rows, null];
418
+ const gap = rows.length - out.length;
419
+ return [out, `${name}: ${gap.toLocaleString("en-US")} row(s) came back twice across page boundaries (same ${col}), so the ` +
420
+ "source's order shifted while paging and about as many rows may be missing; the repeats were dropped " +
421
+ `(${out.length.toLocaleString("en-US")} rows kept` + (meta.total ? ` of ${meta.total.toLocaleString("en-US")}` : "") +
422
+ "). Sort by a unique field (or narrow the request to one page) and fetch again (force)."];
423
+ }
424
+ /** fetchRequest, waiting out a server that stays busy past the client's own retries. */
425
+ async function fetchPatiently(c, ep, params, maxRows, waits = [20, 60]) {
426
+ for (const wait of [...waits, null]) {
427
+ try {
428
+ return await fetchRequest(c, ep, params, maxRows);
429
+ }
430
+ catch (e) {
431
+ if (wait === null || !(e instanceof DataError) || e.kind !== "http" || ![500, 502, 503, 504].includes(e.status ?? 0) ||
432
+ process.env.DATA_ANALYST_FIXTURES)
433
+ throw e;
434
+ await sleep(wait * 1000);
435
+ }
436
+ }
437
+ throw new Error("unreachable");
438
+ }
439
+ /** Every request runs; the missing (404) and empty ones are named together at the end. */
440
+ export class Misses {
441
+ missing = [];
442
+ empty = [];
443
+ notFound(r, tables, notes) {
444
+ if (r.allow_empty) {
445
+ notes.push(`${r.name}: not found at the source (404); kept as an empty table (allow_empty).`);
446
+ tables[r.name] = [];
447
+ }
448
+ else
449
+ this.missing.push(r.name);
450
+ }
451
+ checkEmpty(r, rows) {
452
+ if (!rows.length && !r.allow_empty) {
453
+ const key = r._key;
454
+ this.empty.push(r.name + (key ? ` (${key.map(([k, v]) => `${k}=${v}`).join(", ")})` : ""));
455
+ }
456
+ }
457
+ raiseIfAny(missingHint = "", emptyHint = "") {
458
+ const msgs = [];
459
+ if (this.missing.length) {
460
+ msgs.push(`request(s) ${this.missing.map((m) => `'${m}'`).join(", ")} were not found at the source (404): ` +
461
+ `${missingHint || "check the IDs"}. If the item really does not exist yet (e.g. a volume not yet published), add ` +
462
+ '"allow_empty": true to that request and run again; the answer must say it is missing.');
463
+ }
464
+ if (this.empty.length) {
465
+ msgs.push(`request(s) ${this.empty.map((m) => `'${m}'`).join(", ")} returned no rows: ` +
466
+ `${emptyHint || "check the filter values (see source.md)"}. If none is a correct answer (e.g. no private laws in ` +
467
+ 'that Congress), add "allow_empty": true to that request and run again.');
468
+ }
469
+ if (msgs.length)
470
+ throw new DataError(msgs.join(" "), this.missing.length ? "not_found" : "bad_request");
471
+ }
472
+ }
473
+ /** Every request in the spec: [{name: rows}, notes]. */
474
+ export async function fetchSpec(pack, spec, client) {
475
+ const c = client ?? clientFor(pack);
476
+ const tables = {};
477
+ const notes = [];
478
+ const miss = new Misses();
479
+ for (const r of spec.requests) {
480
+ const [ep, params0, probs] = resolve(pack, r);
481
+ if (probs.length)
482
+ throw new DataError(probs.join("; "), "bad_request");
483
+ let params = params0;
484
+ const extra = Object.keys(r.params || {}).filter((k) => !(k in (ep.params || {})) && !(k in (ep.defaults || {})));
485
+ if (extra.length) {
486
+ notes.push(`${r.name}: parameter(s) ${extra.join(", ")} not declared for endpoint '${r.endpoint}' (endpoints.json); ` +
487
+ "the API may ignore them: check the rows");
488
+ }
489
+ if (pack.adapter?.beforeRequest)
490
+ params = pack.adapter.beforeRequest(r, ep, params) || params;
491
+ let rows, meta;
492
+ try {
493
+ [rows, meta] = await fetchPatiently(c, ep, params, r.max_rows);
494
+ }
495
+ catch (e) {
496
+ if (!(e instanceof DataError))
497
+ throw e;
498
+ if (e.kind === "not_found") {
499
+ miss.notFound(r, tables, notes);
500
+ continue;
501
+ }
502
+ throw new DataError(`request '${r.name}' (${r.endpoint}): ${e.message}`, e.kind, e.status, e.retryAfter);
503
+ }
504
+ let dup;
505
+ [rows, dup] = dedupePages(rows, ep, meta, r.name);
506
+ if (dup)
507
+ notes.push(dup);
508
+ if (pack.adapter?.requestNote) {
509
+ const n = pack.adapter.requestNote(r, ep, params);
510
+ if (n)
511
+ notes.push(n);
512
+ }
513
+ if (pack.adapter?.normalize)
514
+ rows = pack.adapter.normalize(r, ep, rows);
515
+ if (meta.repeated_page) {
516
+ notes.push(`${r.name}${INCOMPLETE} stopped at ${rows.length.toLocaleString("en-US")} rows: page ${meta.pages} repeated ` +
517
+ "the page before it (the API ignored the paging parameter), so later rows may be missing. Narrow the request so it fits one page.");
518
+ }
519
+ else if (meta.truncated) {
520
+ const tn = ep.truncated_note || {};
521
+ const k = tn.when_default;
522
+ const own = !!(tn.text && k && params[k] !== undefined && params[k] !== null && params[k] !== "" &&
523
+ ((r.params || {})[k] === undefined || (r.params || {})[k] === null || (r.params || {})[k] === ""));
524
+ notes.push(`${r.name}: stopped at ${rows.length.toLocaleString("en-US")} rows (max_rows); the source has more` +
525
+ (meta.total ? ` (${meta.total.toLocaleString("en-US")} in total)` : "") + ". Narrow the filter or raise max_rows." +
526
+ (own ? " " + tn.text : ""));
527
+ }
528
+ miss.checkEmpty(r, rows);
529
+ tables[r.name] = rows;
530
+ }
531
+ miss.raiseIfAny();
532
+ return [tables, notes];
533
+ }
534
+ // ---------------------------------------------------------------------- manual download
535
+ /** One link per page the request would need (up to 25), with the key as {API_KEY}. */
536
+ export function manualItems(pack, spec) {
537
+ const eps = pack.endpoints || {};
538
+ const base = String(eps.base_url || pack.policy.routes?.api?.base || "").replace(/\/+$/, "");
539
+ const a = pack.policy.auth;
540
+ const needsKey = a.required === "required";
541
+ const items = [];
542
+ for (const r of spec.requests || []) {
543
+ const [ep, params0, probs] = resolve(pack, r);
544
+ if (probs.length)
545
+ continue;
546
+ const params = (pack.adapter?.beforeRequest && pack.adapter.beforeRequest(r, ep, params0)) || params0;
547
+ const [p, query, body] = split(ep, params);
548
+ const pg = ep.paging || { type: "none" };
549
+ const size = Number(pg.manual_size || pg.size || 1000);
550
+ const limit = Number(r.max_rows || size);
551
+ const pages = ["none", "cursor", "link"].includes(pg.type) ? 1 : Math.min(25, Math.max(1, Math.ceil(limit / size)));
552
+ const rec = ep.records || "";
553
+ const ext = { $csv: "csv", $xlsx: "xlsx", $xls: "xls" }[rec] ?? (rec.startsWith("$zip:") ? "zip" : "json");
554
+ for (let i = 0; i < pages; i++) {
555
+ const q = { ...query };
556
+ const b = body !== null ? { ...body } : null;
557
+ const tgt = ep.method === "POST" && (pg.in ?? "body") === "body" && b ? b : q;
558
+ if (pg.type === "offset")
559
+ [tgt[pg.limit_param], tgt[pg.offset_param]] = [size, i * size];
560
+ else if (pg.type === "page")
561
+ [tgt[pg.page_param], tgt[pg.size_param]] = [Number(pg.start ?? 1) + i, size];
562
+ let url = /^https?:\/\//.test(p) ? p : `${base}/${p.replace(/^\/+/, "")}`;
563
+ if (Object.keys(q).length)
564
+ url += (url.includes("?") ? "&" : "?") + pyUrlencode(q);
565
+ if (needsKey && a.placement === "query")
566
+ url += (url.includes("?") ? "&" : "?") + `${a.param}={API_KEY}`;
567
+ else if (a.required === "optional" && a.demo_key && a.placement === "query")
568
+ url += (url.includes("?") ? "&" : "?") + `${a.param}=${a.demo_key}`;
569
+ const filename = pages === 1 ? `${r.name}.${ext}` : `${r.name}_${i + 1}.${ext}`;
570
+ let note = pages === 1 ? "" : `page ${i + 1} of up to ${pages}; only needed if the previous page held ${size.toLocaleString("en-US")} rows`;
571
+ if (pages === 1 && ["offset", "page"].includes(pg.type)) {
572
+ note = `holds up to ${size.toLocaleString("en-US")} rows; if it holds exactly that many, more rows exist: narrow the filter`;
573
+ }
574
+ if (["cursor", "link"].includes(pg.type))
575
+ note = "first page only: this API pages with a cursor, so the script route is needed for more rows";
576
+ items.push({ name: `${r.name}#${i + 1}`, request: r.name, url, filename, also_ext: ext === "json" ? [".csv"] : [],
577
+ needs_key: needsKey, method: ep.method || "GET", body: b, note, optional: i > 0 });
578
+ }
579
+ }
580
+ return items;
581
+ }
582
+ /** Rows from uploaded files for one request (its pages in natural order). */
583
+ export function parseUpload(pack, req, files) {
584
+ const [ep0, params] = resolve(pack, req);
585
+ const ep = withRecordParams(ep0, params);
586
+ const natural = (s) => s.split(/(\d+)/).map((x) => (/^\d+$/.test(x) ? x.padStart(20, "0") : x)).join("");
587
+ let rows = [];
588
+ for (const f of [...files].sort((a, b) => (natural(a) < natural(b) ? -1 : 1))) {
589
+ const data = fs.readFileSync(f);
590
+ const r = { body: data, text: () => decodeText(data), json: () => parsePyJson(data.toString("utf8").replace(/^/, "")) };
591
+ rows = rows.concat(f.toLowerCase().endsWith(".csv") && ep.records !== "$csv" ? readCsv(r.text()) : records(ep, r));
592
+ }
593
+ let out = clean(rows, ep);
594
+ if (pack.adapter?.normalize)
595
+ out = pack.adapter.normalize(req, ep, out);
596
+ return out;
597
+ }
598
+ // ---------------------------------------------------------------------- describing the API to the model
599
+ /** One line per endpoint: the request index of source_brief. */
600
+ export function endpointIndex(pack) {
601
+ return Object.entries(pack.endpoints?.endpoints || {}).sort(([a], [b]) => (a < b ? -1 : 1))
602
+ .map(([name, e]) => ({ name, summary: String(e.description || "").split(/(?<=\.)\s/)[0].slice(0, 200) }));
603
+ }
604
+ /** Full argument schema of named endpoints (what request_schema returns). */
605
+ export function endpointSchema(pack, name) {
606
+ const e = pack.endpoints?.endpoints?.[name];
607
+ if (!e)
608
+ return null;
609
+ const pg = e.paging || { type: "none" };
610
+ return {
611
+ name, method: e.method || "GET", description: e.description || "",
612
+ params: Object.fromEntries(Object.entries(e.params || {}).map(([k, d]) => [k, {
613
+ desc: d.desc || "", ...(d.required ? { required: true } : {}), ...(d.enum ? { enum: d.enum } : {}),
614
+ ...(d.catalog ? { catalog: "an alias from the catalog (search with request_schema search) or a dataset id" } : {}),
615
+ ...(d.in === "spec" ? { note: "read by the pack, never sent" } : {}),
616
+ }])),
617
+ defaults: e.defaults || {}, paging: pg.type, max_rows: e.max_rows ?? 100000,
618
+ returns: e.records?.startsWith?.("$") ? e.records : "json records",
619
+ };
620
+ }
621
+ export function searchCatalog(pack, query, limit = 20) {
622
+ const words = (query.toLowerCase().match(/[a-z0-9]+/g) || []).filter((w) => w.length > 1);
623
+ const count = (text, w) => text.split(w).length - 1;
624
+ const hits = [];
625
+ for (const [alias, c] of Object.entries(pack.endpoints?.catalog || {})) {
626
+ const text = `${alias} ${c.id ?? ""} ${c.name ?? ""} ${c.description ?? ""}`.toLowerCase();
627
+ const s = words.reduce((n, w) => n + count(text, w), 0);
628
+ if (s)
629
+ hits.push([s, { alias, ...c }]);
630
+ }
631
+ for (const [n, e] of Object.entries(pack.endpoints?.endpoints || {})) {
632
+ const text = `${n} ${e.description ?? ""}`.toLowerCase();
633
+ const s = words.reduce((k, w) => k + count(text, w), 0);
634
+ if (s)
635
+ hits.push([s, { endpoint: n, description: e.description || "" }]);
636
+ }
637
+ return hits.sort((a, b) => b[0] - a[0]).slice(0, limit).map(([, h]) => h);
638
+ }
639
+ //# sourceMappingURL=generic-rest.js.map