data-analyst-mcp 0.0.0-stage → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (640) hide show
  1. package/README.md +261 -2
  2. package/dist/adapters/_acf_tidy.d.ts +18 -0
  3. package/dist/adapters/_acf_tidy.js +432 -0
  4. package/dist/adapters/_acf_tidy.js.map +1 -0
  5. package/dist/adapters/_actuaries_read.d.ts +38 -0
  6. package/dist/adapters/_actuaries_read.js +728 -0
  7. package/dist/adapters/_actuaries_read.js.map +1 -0
  8. package/dist/adapters/_census_micro.d.ts +70 -0
  9. package/dist/adapters/_census_micro.js +622 -0
  10. package/dist/adapters/_census_micro.js.map +1 -0
  11. package/dist/adapters/_census_util.d.ts +40 -0
  12. package/dist/adapters/_census_util.js +275 -0
  13. package/dist/adapters/_census_util.js.map +1 -0
  14. package/dist/adapters/_dol_states.d.ts +6 -0
  15. package/dist/adapters/_dol_states.js +58 -0
  16. package/dist/adapters/_dol_states.js.map +1 -0
  17. package/dist/adapters/_govinfo_entities.d.ts +4 -0
  18. package/dist/adapters/_govinfo_entities.js +6 -0
  19. package/dist/adapters/_govinfo_entities.js.map +1 -0
  20. package/dist/adapters/_govinfo_json.d.ts +13 -0
  21. package/dist/adapters/_govinfo_json.js +107 -0
  22. package/dist/adapters/_govinfo_json.js.map +1 -0
  23. package/dist/adapters/_govinfo_py.d.ts +22 -0
  24. package/dist/adapters/_govinfo_py.js +272 -0
  25. package/dist/adapters/_govinfo_py.js.map +1 -0
  26. package/dist/adapters/_govinfo_xml.d.ts +22 -0
  27. package/dist/adapters/_govinfo_xml.js +233 -0
  28. package/dist/adapters/_govinfo_xml.js.map +1 -0
  29. package/dist/adapters/_hud_excel.d.ts +2 -0
  30. package/dist/adapters/_hud_excel.js +166 -0
  31. package/dist/adapters/_hud_excel.js.map +1 -0
  32. package/dist/adapters/_omb_grid.d.ts +17 -0
  33. package/dist/adapters/_omb_grid.js +132 -0
  34. package/dist/adapters/_omb_grid.js.map +1 -0
  35. package/dist/adapters/_omb_util.d.ts +40 -0
  36. package/dist/adapters/_omb_util.js +192 -0
  37. package/dist/adapters/_omb_util.js.map +1 -0
  38. package/dist/adapters/_usaspending_state.d.ts +6 -0
  39. package/dist/adapters/_usaspending_state.js +54 -0
  40. package/dist/adapters/_usaspending_state.js.map +1 -0
  41. package/dist/adapters/_usda-fna_book.d.ts +59 -0
  42. package/dist/adapters/_usda-fna_book.js +869 -0
  43. package/dist/adapters/_usda-fna_book.js.map +1 -0
  44. package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
  45. package/dist/adapters/_usda-fna_pdf.js +361 -0
  46. package/dist/adapters/_usda-fna_pdf.js.map +1 -0
  47. package/dist/adapters/_usda-outlook_read.d.ts +70 -0
  48. package/dist/adapters/_usda-outlook_read.js +362 -0
  49. package/dist/adapters/_usda-outlook_read.js.map +1 -0
  50. package/dist/adapters/acf.d.ts +3 -0
  51. package/dist/adapters/acf.js +187 -0
  52. package/dist/adapters/acf.js.map +1 -0
  53. package/dist/adapters/actuaries.d.ts +3 -0
  54. package/dist/adapters/actuaries.js +293 -0
  55. package/dist/adapters/actuaries.js.map +1 -0
  56. package/dist/adapters/bts.d.ts +3 -0
  57. package/dist/adapters/bts.js +26 -0
  58. package/dist/adapters/bts.js.map +1 -0
  59. package/dist/adapters/census-geocoder.d.ts +10 -0
  60. package/dist/adapters/census-geocoder.js +47 -0
  61. package/dist/adapters/census-geocoder.js.map +1 -0
  62. package/dist/adapters/census.d.ts +17 -0
  63. package/dist/adapters/census.js +983 -0
  64. package/dist/adapters/census.js.map +1 -0
  65. package/dist/adapters/cfpb.d.ts +7 -0
  66. package/dist/adapters/cfpb.js +348 -0
  67. package/dist/adapters/cfpb.js.map +1 -0
  68. package/dist/adapters/clinical-trials.d.ts +3 -0
  69. package/dist/adapters/clinical-trials.js +135 -0
  70. package/dist/adapters/clinical-trials.js.map +1 -0
  71. package/dist/adapters/cms.d.ts +3 -0
  72. package/dist/adapters/cms.js +292 -0
  73. package/dist/adapters/cms.js.map +1 -0
  74. package/dist/adapters/college-scorecard.d.ts +3 -0
  75. package/dist/adapters/college-scorecard.js +122 -0
  76. package/dist/adapters/college-scorecard.js.map +1 -0
  77. package/dist/adapters/congress.d.ts +33 -0
  78. package/dist/adapters/congress.js +589 -0
  79. package/dist/adapters/congress.js.map +1 -0
  80. package/dist/adapters/dhs-ohss.d.ts +13 -0
  81. package/dist/adapters/dhs-ohss.js +598 -0
  82. package/dist/adapters/dhs-ohss.js.map +1 -0
  83. package/dist/adapters/doj-news.d.ts +7 -0
  84. package/dist/adapters/doj-news.js +286 -0
  85. package/dist/adapters/doj-news.js.map +1 -0
  86. package/dist/adapters/dol.d.ts +4 -0
  87. package/dist/adapters/dol.js +634 -0
  88. package/dist/adapters/dol.js.map +1 -0
  89. package/dist/adapters/eia.d.ts +15 -0
  90. package/dist/adapters/eia.js +456 -0
  91. package/dist/adapters/eia.js.map +1 -0
  92. package/dist/adapters/epa-aqs.d.ts +3 -0
  93. package/dist/adapters/epa-aqs.js +357 -0
  94. package/dist/adapters/epa-aqs.js.map +1 -0
  95. package/dist/adapters/epa.d.ts +3 -0
  96. package/dist/adapters/epa.js +298 -0
  97. package/dist/adapters/epa.js.map +1 -0
  98. package/dist/adapters/fbi.d.ts +3 -0
  99. package/dist/adapters/fbi.js +192 -0
  100. package/dist/adapters/fbi.js.map +1 -0
  101. package/dist/adapters/fda.d.ts +5 -0
  102. package/dist/adapters/fda.js +184 -0
  103. package/dist/adapters/fda.js.map +1 -0
  104. package/dist/adapters/fdic.d.ts +4 -0
  105. package/dist/adapters/fdic.js +95 -0
  106. package/dist/adapters/fdic.js.map +1 -0
  107. package/dist/adapters/fec.d.ts +3 -0
  108. package/dist/adapters/fec.js +142 -0
  109. package/dist/adapters/fec.js.map +1 -0
  110. package/dist/adapters/fed-board.d.ts +12 -0
  111. package/dist/adapters/fed-board.js +581 -0
  112. package/dist/adapters/fed-board.js.map +1 -0
  113. package/dist/adapters/federal-register.d.ts +11 -0
  114. package/dist/adapters/federal-register.js +43 -0
  115. package/dist/adapters/federal-register.js.map +1 -0
  116. package/dist/adapters/fhfa.d.ts +3 -0
  117. package/dist/adapters/fhfa.js +253 -0
  118. package/dist/adapters/fhfa.js.map +1 -0
  119. package/dist/adapters/fsa.d.ts +9 -0
  120. package/dist/adapters/fsa.js +335 -0
  121. package/dist/adapters/fsa.js.map +1 -0
  122. package/dist/adapters/govinfo.d.ts +30 -0
  123. package/dist/adapters/govinfo.js +636 -0
  124. package/dist/adapters/govinfo.js.map +1 -0
  125. package/dist/adapters/gsa-calc.d.ts +10 -0
  126. package/dist/adapters/gsa-calc.js +318 -0
  127. package/dist/adapters/gsa-calc.js.map +1 -0
  128. package/dist/adapters/hud.d.ts +3 -0
  129. package/dist/adapters/hud.js +200 -0
  130. package/dist/adapters/hud.js.map +1 -0
  131. package/dist/adapters/index.d.ts +3 -0
  132. package/dist/adapters/index.js +25 -0
  133. package/dist/adapters/index.js.map +1 -0
  134. package/dist/adapters/irs-soi.d.ts +6 -0
  135. package/dist/adapters/irs-soi.js +576 -0
  136. package/dist/adapters/irs-soi.js.map +1 -0
  137. package/dist/adapters/naep.d.ts +15 -0
  138. package/dist/adapters/naep.js +341 -0
  139. package/dist/adapters/naep.js.map +1 -0
  140. package/dist/adapters/nhtsa.d.ts +10 -0
  141. package/dist/adapters/nhtsa.js +239 -0
  142. package/dist/adapters/nhtsa.js.map +1 -0
  143. package/dist/adapters/nih.d.ts +5 -0
  144. package/dist/adapters/nih.js +196 -0
  145. package/dist/adapters/nih.js.map +1 -0
  146. package/dist/adapters/noaa.d.ts +3 -0
  147. package/dist/adapters/noaa.js +148 -0
  148. package/dist/adapters/noaa.js.map +1 -0
  149. package/dist/adapters/nrel.d.ts +3 -0
  150. package/dist/adapters/nrel.js +186 -0
  151. package/dist/adapters/nrel.js.map +1 -0
  152. package/dist/adapters/nws.d.ts +3 -0
  153. package/dist/adapters/nws.js +119 -0
  154. package/dist/adapters/nws.js.map +1 -0
  155. package/dist/adapters/omb.d.ts +15 -0
  156. package/dist/adapters/omb.js +517 -0
  157. package/dist/adapters/omb.js.map +1 -0
  158. package/dist/adapters/open-payments.d.ts +5 -0
  159. package/dist/adapters/open-payments.js +149 -0
  160. package/dist/adapters/open-payments.js.map +1 -0
  161. package/dist/adapters/pbgc.d.ts +13 -0
  162. package/dist/adapters/pbgc.js +283 -0
  163. package/dist/adapters/pbgc.js.map +1 -0
  164. package/dist/adapters/philfed.d.ts +12 -0
  165. package/dist/adapters/philfed.js +59 -0
  166. package/dist/adapters/philfed.js.map +1 -0
  167. package/dist/adapters/regulations.d.ts +6 -0
  168. package/dist/adapters/regulations.js +58 -0
  169. package/dist/adapters/regulations.js.map +1 -0
  170. package/dist/adapters/sec.d.ts +13 -0
  171. package/dist/adapters/sec.js +504 -0
  172. package/dist/adapters/sec.js.map +1 -0
  173. package/dist/adapters/senate-lobbying.d.ts +3 -0
  174. package/dist/adapters/senate-lobbying.js +181 -0
  175. package/dist/adapters/senate-lobbying.js.map +1 -0
  176. package/dist/adapters/snapqc.d.ts +32 -0
  177. package/dist/adapters/snapqc.js +701 -0
  178. package/dist/adapters/snapqc.js.map +1 -0
  179. package/dist/adapters/treasury-yields.d.ts +3 -0
  180. package/dist/adapters/treasury-yields.js +170 -0
  181. package/dist/adapters/treasury-yields.js.map +1 -0
  182. package/dist/adapters/usaspending.d.ts +3 -0
  183. package/dist/adapters/usaspending.js +340 -0
  184. package/dist/adapters/usaspending.js.map +1 -0
  185. package/dist/adapters/usda-fna.d.ts +7 -0
  186. package/dist/adapters/usda-fna.js +707 -0
  187. package/dist/adapters/usda-fna.js.map +1 -0
  188. package/dist/adapters/usda-fooddata.d.ts +3 -0
  189. package/dist/adapters/usda-fooddata.js +101 -0
  190. package/dist/adapters/usda-fooddata.js.map +1 -0
  191. package/dist/adapters/usda-nass.d.ts +3 -0
  192. package/dist/adapters/usda-nass.js +108 -0
  193. package/dist/adapters/usda-nass.js.map +1 -0
  194. package/dist/adapters/usda-outlook.d.ts +12 -0
  195. package/dist/adapters/usda-outlook.js +696 -0
  196. package/dist/adapters/usda-outlook.js.map +1 -0
  197. package/dist/adapters/usgs.d.ts +9 -0
  198. package/dist/adapters/usgs.js +265 -0
  199. package/dist/adapters/usgs.js.map +1 -0
  200. package/dist/adapters/usitc.d.ts +18 -0
  201. package/dist/adapters/usitc.js +170 -0
  202. package/dist/adapters/usitc.js.map +1 -0
  203. package/dist/adapters/uspto.d.ts +3 -0
  204. package/dist/adapters/uspto.js +47 -0
  205. package/dist/adapters/uspto.js.map +1 -0
  206. package/dist/adapters/world-bank.d.ts +3 -0
  207. package/dist/adapters/world-bank.js +163 -0
  208. package/dist/adapters/world-bank.js.map +1 -0
  209. package/dist/analysis/cite.d.ts +31 -0
  210. package/dist/analysis/cite.js +229 -0
  211. package/dist/analysis/cite.js.map +1 -0
  212. package/dist/analysis/combine.d.ts +27 -0
  213. package/dist/analysis/combine.js +198 -0
  214. package/dist/analysis/combine.js.map +1 -0
  215. package/dist/analysis/compute.d.ts +62 -0
  216. package/dist/analysis/compute.js +583 -0
  217. package/dist/analysis/compute.js.map +1 -0
  218. package/dist/analysis/expr.d.ts +69 -0
  219. package/dist/analysis/expr.js +780 -0
  220. package/dist/analysis/expr.js.map +1 -0
  221. package/dist/analysis/factfmt.d.ts +16 -0
  222. package/dist/analysis/factfmt.js +187 -0
  223. package/dist/analysis/factfmt.js.map +1 -0
  224. package/dist/analysis/facts.d.ts +63 -0
  225. package/dist/analysis/facts.js +355 -0
  226. package/dist/analysis/facts.js.map +1 -0
  227. package/dist/analysis/handoff.d.ts +58 -0
  228. package/dist/analysis/handoff.js +208 -0
  229. package/dist/analysis/handoff.js.map +1 -0
  230. package/dist/analysis/pyfmt.d.ts +16 -0
  231. package/dist/analysis/pyfmt.js +156 -0
  232. package/dist/analysis/pyfmt.js.map +1 -0
  233. package/dist/analysis/rerun.d.ts +13 -0
  234. package/dist/analysis/rerun.js +125 -0
  235. package/dist/analysis/rerun.js.map +1 -0
  236. package/dist/analysis/table.d.ts +18 -0
  237. package/dist/analysis/table.js +109 -0
  238. package/dist/analysis/table.js.map +1 -0
  239. package/dist/analysis/tools.d.ts +67 -0
  240. package/dist/analysis/tools.js +217 -0
  241. package/dist/analysis/tools.js.map +1 -0
  242. package/dist/analysis/write_table.d.ts +13 -0
  243. package/dist/analysis/write_table.js +59 -0
  244. package/dist/analysis/write_table.js.map +1 -0
  245. package/dist/bin/cli.d.ts +10 -0
  246. package/dist/bin/cli.js +177 -0
  247. package/dist/bin/cli.js.map +1 -0
  248. package/dist/core/cache.d.ts +14 -0
  249. package/dist/core/cache.js +90 -0
  250. package/dist/core/cache.js.map +1 -0
  251. package/dist/core/context.d.ts +12 -0
  252. package/dist/core/context.js +15 -0
  253. package/dist/core/context.js.map +1 -0
  254. package/dist/core/env.d.ts +14 -0
  255. package/dist/core/env.js +52 -0
  256. package/dist/core/env.js.map +1 -0
  257. package/dist/core/generic-rest.d.ts +61 -0
  258. package/dist/core/generic-rest.js +639 -0
  259. package/dist/core/generic-rest.js.map +1 -0
  260. package/dist/core/http.d.ts +120 -0
  261. package/dist/core/http.js +514 -0
  262. package/dist/core/http.js.map +1 -0
  263. package/dist/core/keys.d.ts +43 -0
  264. package/dist/core/keys.js +174 -0
  265. package/dist/core/keys.js.map +1 -0
  266. package/dist/core/manual.d.ts +41 -0
  267. package/dist/core/manual.js +218 -0
  268. package/dist/core/manual.js.map +1 -0
  269. package/dist/core/package.d.ts +58 -0
  270. package/dist/core/package.js +228 -0
  271. package/dist/core/package.js.map +1 -0
  272. package/dist/core/packs.d.ts +139 -0
  273. package/dist/core/packs.js +173 -0
  274. package/dist/core/packs.js.map +1 -0
  275. package/dist/core/policy.d.ts +69 -0
  276. package/dist/core/policy.js +189 -0
  277. package/dist/core/policy.js.map +1 -0
  278. package/dist/core/py.d.ts +40 -0
  279. package/dist/core/py.js +171 -0
  280. package/dist/core/py.js.map +1 -0
  281. package/dist/core/routing.d.ts +10 -0
  282. package/dist/core/routing.js +39 -0
  283. package/dist/core/routing.js.map +1 -0
  284. package/dist/core/tabular.d.ts +21 -0
  285. package/dist/core/tabular.js +229 -0
  286. package/dist/core/tabular.js.map +1 -0
  287. package/dist/deep/_bea_api.d.ts +102 -0
  288. package/dist/deep/_bea_api.js +457 -0
  289. package/dist/deep/_bea_api.js.map +1 -0
  290. package/dist/deep/_bea_fetch.d.ts +151 -0
  291. package/dist/deep/_bea_fetch.js +1274 -0
  292. package/dist/deep/_bea_fetch.js.map +1 -0
  293. package/dist/deep/_bea_util.d.ts +14 -0
  294. package/dist/deep/_bea_util.js +156 -0
  295. package/dist/deep/_bea_util.js.map +1 -0
  296. package/dist/deep/_bls_api.d.ts +84 -0
  297. package/dist/deep/_bls_api.js +231 -0
  298. package/dist/deep/_bls_api.js.map +1 -0
  299. package/dist/deep/_bls_ids.d.ts +43 -0
  300. package/dist/deep/_bls_ids.js +188 -0
  301. package/dist/deep/_bls_ids.js.map +1 -0
  302. package/dist/deep/_bls_lib.d.ts +61 -0
  303. package/dist/deep/_bls_lib.js +442 -0
  304. package/dist/deep/_bls_lib.js.map +1 -0
  305. package/dist/deep/_fred_api.d.ts +148 -0
  306. package/dist/deep/_fred_api.js +545 -0
  307. package/dist/deep/_fred_api.js.map +1 -0
  308. package/dist/deep/_fred_pack.d.ts +162 -0
  309. package/dist/deep/_fred_pack.js +1027 -0
  310. package/dist/deep/_fred_pack.js.map +1 -0
  311. package/dist/deep/_fred_py.d.ts +40 -0
  312. package/dist/deep/_fred_py.js +199 -0
  313. package/dist/deep/_fred_py.js.map +1 -0
  314. package/dist/deep/_ipums_bundle.d.ts +29 -0
  315. package/dist/deep/_ipums_bundle.js +194 -0
  316. package/dist/deep/_ipums_bundle.js.map +1 -0
  317. package/dist/deep/_ipums_ddi.d.ts +23 -0
  318. package/dist/deep/_ipums_ddi.js +162 -0
  319. package/dist/deep/_ipums_ddi.js.map +1 -0
  320. package/dist/deep/_opm_io.d.ts +26 -0
  321. package/dist/deep/_opm_io.js +383 -0
  322. package/dist/deep/_opm_io.js.map +1 -0
  323. package/dist/deep/_opm_parquet.d.ts +31 -0
  324. package/dist/deep/_opm_parquet.js +744 -0
  325. package/dist/deep/_opm_parquet.js.map +1 -0
  326. package/dist/deep/_opm_py.d.ts +6 -0
  327. package/dist/deep/_opm_py.js +74 -0
  328. package/dist/deep/_opm_py.js.map +1 -0
  329. package/dist/deep/_treasury_api.d.ts +48 -0
  330. package/dist/deep/_treasury_api.js +221 -0
  331. package/dist/deep/_treasury_api.js.map +1 -0
  332. package/dist/deep/_treasury_context.d.ts +34 -0
  333. package/dist/deep/_treasury_context.js +142 -0
  334. package/dist/deep/_treasury_context.js.map +1 -0
  335. package/dist/deep/_treasury_lib.d.ts +109 -0
  336. package/dist/deep/_treasury_lib.js +686 -0
  337. package/dist/deep/_treasury_lib.js.map +1 -0
  338. package/dist/deep/bea.d.ts +3 -0
  339. package/dist/deep/bea.js +248 -0
  340. package/dist/deep/bea.js.map +1 -0
  341. package/dist/deep/bls.d.ts +3 -0
  342. package/dist/deep/bls.js +220 -0
  343. package/dist/deep/bls.js.map +1 -0
  344. package/dist/deep/fred.d.ts +3 -0
  345. package/dist/deep/fred.js +325 -0
  346. package/dist/deep/fred.js.map +1 -0
  347. package/dist/deep/ipums.d.ts +59 -0
  348. package/dist/deep/ipums.js +744 -0
  349. package/dist/deep/ipums.js.map +1 -0
  350. package/dist/deep/opm.d.ts +56 -0
  351. package/dist/deep/opm.js +1033 -0
  352. package/dist/deep/opm.js.map +1 -0
  353. package/dist/deep/treasury.d.ts +3 -0
  354. package/dist/deep/treasury.js +218 -0
  355. package/dist/deep/treasury.js.map +1 -0
  356. package/dist/envelope.d.ts +47 -0
  357. package/dist/envelope.js +88 -0
  358. package/dist/envelope.js.map +1 -0
  359. package/dist/index.d.ts +277 -0
  360. package/dist/index.js +106 -0
  361. package/dist/index.js.map +1 -0
  362. package/dist/remote/app.d.ts +21 -0
  363. package/dist/remote/app.js +218 -0
  364. package/dist/remote/app.js.map +1 -0
  365. package/dist/remote/oauth.d.ts +14 -0
  366. package/dist/remote/oauth.js +90 -0
  367. package/dist/remote/oauth.js.map +1 -0
  368. package/dist/remote/pages.d.ts +8 -0
  369. package/dist/remote/pages.js +56 -0
  370. package/dist/remote/pages.js.map +1 -0
  371. package/dist/remote/sign.d.ts +4 -0
  372. package/dist/remote/sign.js +40 -0
  373. package/dist/remote/sign.js.map +1 -0
  374. package/dist/remote/store.d.ts +31 -0
  375. package/dist/remote/store.js +129 -0
  376. package/dist/remote/store.js.map +1 -0
  377. package/dist/report/audit_answer.d.ts +94 -0
  378. package/dist/report/audit_answer.js +1349 -0
  379. package/dist/report/audit_answer.js.map +1 -0
  380. package/dist/report/audit_report.d.ts +23 -0
  381. package/dist/report/audit_report.js +637 -0
  382. package/dist/report/audit_report.js.map +1 -0
  383. package/dist/report/build_report.d.ts +27 -0
  384. package/dist/report/build_report.js +1168 -0
  385. package/dist/report/build_report.js.map +1 -0
  386. package/dist/report/method_audit.d.ts +47 -0
  387. package/dist/report/method_audit.js +716 -0
  388. package/dist/report/method_audit.js.map +1 -0
  389. package/dist/report/policyset.d.ts +50 -0
  390. package/dist/report/policyset.js +421 -0
  391. package/dist/report/policyset.js.map +1 -0
  392. package/dist/report/py.d.ts +58 -0
  393. package/dist/report/py.js +311 -0
  394. package/dist/report/py.js.map +1 -0
  395. package/dist/report/scan_secrets.d.ts +24 -0
  396. package/dist/report/scan_secrets.js +265 -0
  397. package/dist/report/scan_secrets.js.map +1 -0
  398. package/dist/report/style_check.d.ts +29 -0
  399. package/dist/report/style_check.js +173 -0
  400. package/dist/report/style_check.js.map +1 -0
  401. package/dist/report/tools.d.ts +87 -0
  402. package/dist/report/tools.js +167 -0
  403. package/dist/report/tools.js.map +1 -0
  404. package/dist/server.d.ts +5 -0
  405. package/dist/server.js +247 -0
  406. package/dist/server.js.map +1 -0
  407. package/dist/session.d.ts +47 -0
  408. package/dist/session.js +107 -0
  409. package/dist/session.js.map +1 -0
  410. package/dist/toolkit.d.ts +21 -0
  411. package/dist/toolkit.js +2 -0
  412. package/dist/toolkit.js.map +1 -0
  413. package/dist/tools.d.ts +299 -0
  414. package/dist/tools.js +491 -0
  415. package/dist/tools.js.map +1 -0
  416. package/guide/WORKFLOW.md +134 -0
  417. package/guide/assets/disclaimer.txt +1 -0
  418. package/guide/assets/download_data.ps1 +162 -0
  419. package/guide/assets/download_data.sh +143 -0
  420. package/guide/assets/report_template.html +597 -0
  421. package/guide/references/report_patterns.md +199 -0
  422. package/guide/references/routing.md +171 -0
  423. package/guide/references/style_guide.md +398 -0
  424. package/package.json +54 -4
  425. package/packs/acf/endpoints.json +114 -0
  426. package/packs/acf/policy.json +104 -0
  427. package/packs/acf/source.md +138 -0
  428. package/packs/actuaries/endpoints.json +82 -0
  429. package/packs/actuaries/policy.json +109 -0
  430. package/packs/actuaries/source.md +137 -0
  431. package/packs/bea/assets/download_bea.ps1 +182 -0
  432. package/packs/bea/assets/download_bea.sh +144 -0
  433. package/packs/bea/docs/bea_api.md +461 -0
  434. package/packs/bea/docs/datasets.md +157 -0
  435. package/packs/bea/docs/report_patterns.md +112 -0
  436. package/packs/bea/policy.json +170 -0
  437. package/packs/bea/source.md +152 -0
  438. package/packs/bls/assets/download_bls.ps1 +152 -0
  439. package/packs/bls/assets/download_bls.sh +167 -0
  440. package/packs/bls/docs/analysis_patterns.md +108 -0
  441. package/packs/bls/docs/bls_api.md +73 -0
  442. package/packs/bls/docs/series_ids.md +149 -0
  443. package/packs/bls/policy.json +238 -0
  444. package/packs/bls/series_ids.json +1375 -0
  445. package/packs/bls/source.md +156 -0
  446. package/packs/bts/endpoints.json +347 -0
  447. package/packs/bts/policy.json +94 -0
  448. package/packs/bts/source.md +97 -0
  449. package/packs/cdc/endpoints.json +197 -0
  450. package/packs/cdc/policy.json +83 -0
  451. package/packs/cdc/source.md +74 -0
  452. package/packs/census/endpoints.json +99 -0
  453. package/packs/census/policy.json +238 -0
  454. package/packs/census/source.md +304 -0
  455. package/packs/census-geocoder/endpoints.json +70 -0
  456. package/packs/census-geocoder/policy.json +80 -0
  457. package/packs/census-geocoder/source.md +53 -0
  458. package/packs/cfpb/endpoints.json +327 -0
  459. package/packs/cfpb/policy.json +119 -0
  460. package/packs/cfpb/source.md +96 -0
  461. package/packs/clinical-trials/endpoints.json +167 -0
  462. package/packs/clinical-trials/policy.json +84 -0
  463. package/packs/clinical-trials/source.md +82 -0
  464. package/packs/cms/endpoints.json +69 -0
  465. package/packs/cms/policy.json +96 -0
  466. package/packs/cms/source.md +112 -0
  467. package/packs/college-scorecard/endpoints.json +253 -0
  468. package/packs/college-scorecard/policy.json +106 -0
  469. package/packs/college-scorecard/source.md +92 -0
  470. package/packs/congress/endpoints.json +1701 -0
  471. package/packs/congress/policy.json +117 -0
  472. package/packs/congress/source.md +190 -0
  473. package/packs/dhs-ohss/endpoints.json +66 -0
  474. package/packs/dhs-ohss/policy.json +107 -0
  475. package/packs/dhs-ohss/source.md +120 -0
  476. package/packs/doj-news/endpoints.json +59 -0
  477. package/packs/doj-news/policy.json +96 -0
  478. package/packs/doj-news/source.md +99 -0
  479. package/packs/dol/endpoints.json +818 -0
  480. package/packs/dol/policy.json +135 -0
  481. package/packs/dol/source.md +181 -0
  482. package/packs/eia/endpoints.json +718 -0
  483. package/packs/eia/policy.json +112 -0
  484. package/packs/eia/source.md +112 -0
  485. package/packs/epa/endpoints.json +292 -0
  486. package/packs/epa/policy.json +99 -0
  487. package/packs/epa/source.md +99 -0
  488. package/packs/epa-aqs/endpoints.json +174 -0
  489. package/packs/epa-aqs/policy.json +96 -0
  490. package/packs/epa-aqs/source.md +97 -0
  491. package/packs/fbi/endpoints.json +1842 -0
  492. package/packs/fbi/policy.json +118 -0
  493. package/packs/fbi/source.md +108 -0
  494. package/packs/fda/endpoints.json +238 -0
  495. package/packs/fda/policy.json +121 -0
  496. package/packs/fda/source.md +117 -0
  497. package/packs/fdic/endpoints.json +417 -0
  498. package/packs/fdic/policy.json +83 -0
  499. package/packs/fdic/source.md +92 -0
  500. package/packs/fec/endpoints.json +708 -0
  501. package/packs/fec/policy.json +93 -0
  502. package/packs/fec/source.md +90 -0
  503. package/packs/fed-board/endpoints.json +89 -0
  504. package/packs/fed-board/policy.json +136 -0
  505. package/packs/fed-board/source.md +118 -0
  506. package/packs/federal-register/endpoints.json +143 -0
  507. package/packs/federal-register/policy.json +86 -0
  508. package/packs/federal-register/source.md +98 -0
  509. package/packs/fema/endpoints.json +646 -0
  510. package/packs/fema/policy.json +103 -0
  511. package/packs/fema/source.md +103 -0
  512. package/packs/fhfa/endpoints.json +105 -0
  513. package/packs/fhfa/policy.json +109 -0
  514. package/packs/fhfa/source.md +108 -0
  515. package/packs/fred/assets/download_fred.ps1 +122 -0
  516. package/packs/fred/assets/download_fred.sh +123 -0
  517. package/packs/fred/docs/analysis_patterns.md +124 -0
  518. package/packs/fred/docs/fred_api.md +94 -0
  519. package/packs/fred/docs/series.md +188 -0
  520. package/packs/fred/policy.json +253 -0
  521. package/packs/fred/source.md +142 -0
  522. package/packs/fsa/endpoints.json +150 -0
  523. package/packs/fsa/policy.json +99 -0
  524. package/packs/fsa/source.md +102 -0
  525. package/packs/govinfo/endpoints.json +520 -0
  526. package/packs/govinfo/policy.json +116 -0
  527. package/packs/govinfo/source.md +241 -0
  528. package/packs/gsa-calc/endpoints.json +93 -0
  529. package/packs/gsa-calc/policy.json +97 -0
  530. package/packs/gsa-calc/source.md +108 -0
  531. package/packs/hud/endpoints.json +147 -0
  532. package/packs/hud/policy.json +84 -0
  533. package/packs/hud/source.md +97 -0
  534. package/packs/ipums/assets/download_ipums.ps1 +215 -0
  535. package/packs/ipums/assets/download_ipums.sh +192 -0
  536. package/packs/ipums/docs/ipums_api.md +67 -0
  537. package/packs/ipums/docs/report_patterns.md +117 -0
  538. package/packs/ipums/docs/variables.md +284 -0
  539. package/packs/ipums/lib/fetch_text.json +112 -0
  540. package/packs/ipums/policy.json +215 -0
  541. package/packs/ipums/source.md +169 -0
  542. package/packs/irs-soi/endpoints.json +114 -0
  543. package/packs/irs-soi/policy.json +107 -0
  544. package/packs/irs-soi/source.md +124 -0
  545. package/packs/medicaid/endpoints.json +412 -0
  546. package/packs/medicaid/policy.json +108 -0
  547. package/packs/medicaid/source.md +136 -0
  548. package/packs/naep/endpoints.json +217 -0
  549. package/packs/naep/policy.json +100 -0
  550. package/packs/naep/source.md +120 -0
  551. package/packs/nhtsa/endpoints.json +132 -0
  552. package/packs/nhtsa/policy.json +93 -0
  553. package/packs/nhtsa/source.md +109 -0
  554. package/packs/nih/endpoints.json +453 -0
  555. package/packs/nih/policy.json +86 -0
  556. package/packs/nih/source.md +108 -0
  557. package/packs/noaa/endpoints.json +432 -0
  558. package/packs/noaa/policy.json +95 -0
  559. package/packs/noaa/source.md +88 -0
  560. package/packs/nrel/endpoints.json +328 -0
  561. package/packs/nrel/policy.json +96 -0
  562. package/packs/nrel/source.md +91 -0
  563. package/packs/nws/endpoints.json +272 -0
  564. package/packs/nws/policy.json +82 -0
  565. package/packs/nws/source.md +81 -0
  566. package/packs/nyfed/endpoints.json +80 -0
  567. package/packs/nyfed/policy.json +92 -0
  568. package/packs/nyfed/source.md +89 -0
  569. package/packs/omb/endpoints.json +460 -0
  570. package/packs/omb/policy.json +106 -0
  571. package/packs/omb/source.md +130 -0
  572. package/packs/open-payments/endpoints.json +298 -0
  573. package/packs/open-payments/policy.json +101 -0
  574. package/packs/open-payments/source.md +99 -0
  575. package/packs/opm/assets/download_opm.ps1 +125 -0
  576. package/packs/opm/assets/download_opm.sh +85 -0
  577. package/packs/opm/docs/fields.md +184 -0
  578. package/packs/opm/docs/opm_api.md +71 -0
  579. package/packs/opm/docs/report_patterns.md +83 -0
  580. package/packs/opm/lib/data_dictionary.csv +73 -0
  581. package/packs/opm/policy.json +142 -0
  582. package/packs/opm/source.md +137 -0
  583. package/packs/pbgc/endpoints.json +483 -0
  584. package/packs/pbgc/policy.json +94 -0
  585. package/packs/pbgc/source.md +95 -0
  586. package/packs/philfed/endpoints.json +85 -0
  587. package/packs/philfed/policy.json +94 -0
  588. package/packs/philfed/source.md +125 -0
  589. package/packs/port.json +287 -0
  590. package/packs/regulations/endpoints.json +288 -0
  591. package/packs/regulations/policy.json +86 -0
  592. package/packs/regulations/source.md +84 -0
  593. package/packs/sec/endpoints.json +237 -0
  594. package/packs/sec/policy.json +86 -0
  595. package/packs/sec/source.md +101 -0
  596. package/packs/senate-lobbying/endpoints.json +554 -0
  597. package/packs/senate-lobbying/policy.json +99 -0
  598. package/packs/senate-lobbying/source.md +100 -0
  599. package/packs/snapqc/endpoints.json +1426 -0
  600. package/packs/snapqc/policy.json +92 -0
  601. package/packs/snapqc/source.md +96 -0
  602. package/packs/treasury/catalog.json +2267 -0
  603. package/packs/treasury/docs/api_guide.md +245 -0
  604. package/packs/treasury/docs/datasets.md +56 -0
  605. package/packs/treasury/docs/report_patterns.md +126 -0
  606. package/packs/treasury/policy.json +159 -0
  607. package/packs/treasury/source.md +170 -0
  608. package/packs/treasury-yields/endpoints.json +226 -0
  609. package/packs/treasury-yields/policy.json +86 -0
  610. package/packs/treasury-yields/source.md +80 -0
  611. package/packs/usaspending/endpoints.json +1316 -0
  612. package/packs/usaspending/policy.json +109 -0
  613. package/packs/usaspending/source.md +103 -0
  614. package/packs/usda-fna/endpoints.json +427 -0
  615. package/packs/usda-fna/policy.json +133 -0
  616. package/packs/usda-fna/source.md +217 -0
  617. package/packs/usda-fooddata/endpoints.json +181 -0
  618. package/packs/usda-fooddata/policy.json +94 -0
  619. package/packs/usda-fooddata/source.md +105 -0
  620. package/packs/usda-nass/endpoints.json +324 -0
  621. package/packs/usda-nass/policy.json +95 -0
  622. package/packs/usda-nass/source.md +85 -0
  623. package/packs/usda-outlook/endpoints.json +86 -0
  624. package/packs/usda-outlook/policy.json +109 -0
  625. package/packs/usda-outlook/source.md +144 -0
  626. package/packs/usgs/endpoints.json +313 -0
  627. package/packs/usgs/policy.json +95 -0
  628. package/packs/usgs/source.md +96 -0
  629. package/packs/usitc/endpoints.json +74 -0
  630. package/packs/usitc/policy.json +111 -0
  631. package/packs/usitc/source.md +90 -0
  632. package/packs/uspto/endpoints.json +141 -0
  633. package/packs/uspto/policy.json +95 -0
  634. package/packs/uspto/source.md +101 -0
  635. package/packs/va/endpoints.json +414 -0
  636. package/packs/va/policy.json +101 -0
  637. package/packs/va/source.md +129 -0
  638. package/packs/world-bank/endpoints.json +122 -0
  639. package/packs/world-bank/policy.json +101 -0
  640. package/packs/world-bank/source.md +93 -0
@@ -0,0 +1,174 @@
1
+ /**
2
+ * API key lookup and redaction for every source pack.
3
+ *
4
+ * A pack names its key variable in policy.json (auth.env, e.g. "FRED_API_KEY"). findKey() looks, in order:
5
+ * 0. the per-request key provider (remote mode: the signed-in user's own keys), when one is set
6
+ * 1. the <ENV> environment variable
7
+ * local mode only:
8
+ * 2. the file named by the <ENV>_FILE environment variable
9
+ * 3. "env": {"<ENV>": "..."} in ~/.config/data-analyst/keys.json ($DATA_ANALYST_CONFIG_DIR), written by
10
+ * `data-analyst-mcp setup`
11
+ * 4. "env": {"<ENV>": "..."} in Claude Code's settings.json (~/.claude or $CLAUDE_CONFIG_DIR)
12
+ * 5. ~/.<home>/api_key (a file holding only the key); <home> is auth.home in policy.json
13
+ * Files are read on every lookup, so a key stored a moment ago works without a restart.
14
+ * Returns [key, source label]. The label may be shown; the key never is. Every result and error that leaves the
15
+ * server goes through redact(), which removes every key this process has seen plus key-shaped query parameters,
16
+ * headers and JSON fields.
17
+ *
18
+ * DATA_ACCESS_FORCE=restricted hides every key and makes every request fail as a network error (sandbox replay).
19
+ */
20
+ import { AsyncLocalStorage } from "node:async_hooks";
21
+ import fs from "node:fs";
22
+ import os from "node:os";
23
+ import path from "node:path";
24
+ import { mode } from "./env.js";
25
+ const KNOWN = new Set();
26
+ export const KEY_FIELDS = ["api_key", "apikey", "api-key", "key", "registrationkey", "userid", "token", "access_token",
27
+ "x-api-key", "authorization", "subscription-key", "user_token"];
28
+ /** Patterns a pack adds for keys its API echoes in odd shapes, by pack id ("" = every pack). */
29
+ const EXTRA_PATTERNS = [];
30
+ const packScope = new AsyncLocalStorage();
31
+ /** Run fn as a call for one pack: that pack's own redaction patterns apply inside it. */
32
+ export function withPackScope(pack, fn) {
33
+ return packScope.run(pack, fn);
34
+ }
35
+ const provider = new AsyncLocalStorage();
36
+ export function withKeyProvider(p, fn) {
37
+ return provider.run(p, fn);
38
+ }
39
+ export function restricted() {
40
+ return (process.env.DATA_ACCESS_FORCE || "").trim().toLowerCase() === "restricted";
41
+ }
42
+ export function configPath() {
43
+ const cfg = process.env.DATA_ANALYST_CONFIG_DIR || path.join(os.homedir(), ".config", "data-analyst");
44
+ return path.join(cfg, "keys.json");
45
+ }
46
+ export function settingsPath() {
47
+ const cfg = process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), ".claude");
48
+ return path.join(cfg, "settings.json");
49
+ }
50
+ function readEnvJson(p, env) {
51
+ try {
52
+ const data = JSON.parse(fs.readFileSync(p, "utf8").replace(/^/, ""));
53
+ const v = data && typeof data === "object" ? (data.env || {})[env] : undefined;
54
+ return String(v ?? "").trim();
55
+ }
56
+ catch {
57
+ return "";
58
+ }
59
+ }
60
+ function readKeyFile(p) {
61
+ try {
62
+ return fs.readFileSync(p, "utf8").replace(/^/, "").split(/\r?\n/)[0].trim();
63
+ }
64
+ catch {
65
+ return "";
66
+ }
67
+ }
68
+ export function defaultHome(env) {
69
+ return env.toLowerCase().replace("_api_key", "").replaceAll("_", "-");
70
+ }
71
+ export function keyLocations(env, home) {
72
+ if (mode() === "remote")
73
+ return [`the ${env} secret on the server, or your own key on the server's settings page`];
74
+ const h = home || defaultHome(env);
75
+ return [`the ${env} environment variable`, `the file named by the ${env}_FILE environment variable`,
76
+ `"env": {"${env}": "..."} in ${configPath()} (data-analyst-mcp setup writes it)`,
77
+ `"env": {"${env}": "..."} in Claude Code's ${settingsPath()}`, `~/.${h}/api_key (a file holding only the key)`];
78
+ }
79
+ /** Locate the key named env. Returns [key, source]; [null, null] when absent or restricted. */
80
+ export function findKey(env, home) {
81
+ if (!env || restricted())
82
+ return [null, null];
83
+ let found = [null, null];
84
+ const p = provider.getStore();
85
+ const own = p ? (p(env) || "").trim() : "";
86
+ if (own)
87
+ found = [own, "your stored key"];
88
+ const ev = (process.env[env] || "").trim();
89
+ if (!found[0] && ev)
90
+ found = [ev, `${env} environment variable`];
91
+ if (!found[0] && mode() === "local") {
92
+ const kf = process.env[`${env}_FILE`];
93
+ if (kf) {
94
+ const k = readKeyFile(kf);
95
+ if (k)
96
+ found = [k, `file named by ${env}_FILE`];
97
+ }
98
+ for (const [label, file] of [["keys.json", configPath()], ["settings.json", settingsPath()]]) {
99
+ if (found[0])
100
+ break;
101
+ const k = readEnvJson(file, env);
102
+ if (k)
103
+ found = [k, `${label} (${file})`];
104
+ }
105
+ if (!found[0]) {
106
+ const file = path.join(os.homedir(), `.${home || defaultHome(env)}`, "api_key");
107
+ const k = readKeyFile(file);
108
+ if (k)
109
+ found = [k, file];
110
+ }
111
+ }
112
+ if (found[0])
113
+ remember(found[0]);
114
+ return found;
115
+ }
116
+ /** The key for a pack's auth block: auth.env first, then each auth.alt_env. */
117
+ export function findAuthKey(auth) {
118
+ let [key, src] = findKey(auth.env, auth.home);
119
+ for (const alt of key ? [] : auth.alt_env || []) {
120
+ [key, src] = findKey(alt);
121
+ if (key)
122
+ break;
123
+ }
124
+ return [key, src];
125
+ }
126
+ /** Add a secret to the redaction set (keys passed in, tokens returned by an API). */
127
+ export function remember(secret) {
128
+ if (secret && secret.length >= 6)
129
+ KNOWN.add(secret);
130
+ }
131
+ /** A pack whose API echoes keys in odd shapes adds a pattern (BLS: 32-hex tokens in error text). */
132
+ /**
133
+ * A pack whose API echoes keys in odd shapes adds a pattern (BLS: 32-hex tokens in error text). It applies only
134
+ * inside that pack's calls (withPackScope): in one server process every pack is loaded, and a pattern from one
135
+ * pack must not redact another pack's data (a 32-hex series hash). pack "" applies everywhere.
136
+ */
137
+ export function addPattern(regex, pack) {
138
+ EXTRA_PATTERNS.push({ re: new RegExp(regex, "g"), pack });
139
+ }
140
+ const esc = (s) => s.replace(/[.*+?^${}()|[\]\\-]/g, "\\$&");
141
+ export function redact(value) {
142
+ let text = typeof value === "string" ? value : String(value);
143
+ for (const k of [...KNOWN].sort((a, b) => b.length - a.length))
144
+ text = text.split(k).join("[REDACTED]");
145
+ const names = KEY_FIELDS.map(esc).join("|");
146
+ text = text.replace(new RegExp(`([?&](?:${names})=)[^&\\s'"#]+`, "gi"), "$1[REDACTED]");
147
+ text = text.replace(new RegExp(`(["'](?:${names})["']\\s*:\\s*["'])[^"']+`, "gi"), "$1[REDACTED]");
148
+ // a credential is a run of token characters; prose such as "Bearer <token>" is left alone
149
+ text = text.replace(/(authorization:\s*(?:(?:bearer|basic|token)\s+)?)(?!(?:bearer|basic|token)\b)[A-Za-z0-9._~+/=-]{6,}/gi, "$1[REDACTED]");
150
+ const scope = packScope.getStore();
151
+ for (const p of EXTRA_PATTERNS)
152
+ if (!p.pack || p.pack === scope)
153
+ text = text.replace(p.re, "[REDACTED]");
154
+ return text;
155
+ }
156
+ /** Redact every string inside a JSON-like value (tool results). */
157
+ export function redactDeep(value) {
158
+ if (typeof value === "string")
159
+ return redact(value);
160
+ if (Array.isArray(value))
161
+ return value.map(redactDeep);
162
+ if (value && typeof value === "object") {
163
+ const out = {};
164
+ for (const [k, v] of Object.entries(value))
165
+ out[k] = redactDeep(v);
166
+ return out;
167
+ }
168
+ return value;
169
+ }
170
+ /** Every secret this process knows, for scanning outputs (never printed). */
171
+ export function secretValues() {
172
+ return [...KNOWN].sort();
173
+ }
174
+ //# sourceMappingURL=keys.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"keys.js","sourceRoot":"","sources":["../../src/core/keys.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAEhC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;AAChC,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc;IACpH,WAAW,EAAE,eAAe,EAAE,kBAAkB,EAAE,YAAY,CAAC,CAAC;AAClE,gGAAgG;AAChG,MAAM,cAAc,GAAmC,EAAE,CAAC;AAC1D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAU,CAAC;AAElD,yFAAyF;AACzF,MAAM,UAAU,aAAa,CAAI,IAAY,EAAE,EAAW;IACxD,OAAO,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AACjC,CAAC;AAWD,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAe,CAAC;AAEtD,MAAM,UAAU,eAAe,CAAI,CAAc,EAAE,EAAW;IAC5D,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,YAAY,CAAC;AACrF,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC;IACtG,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC,CAAC;IAChF,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,WAAW,CAAC,CAAS,EAAE,GAAW;IACzC,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;QACtE,MAAM,CAAC,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/E,OAAO,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC5B,IAAI,CAAC;QACH,OAAO,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,OAAO,GAAG,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW,EAAE,IAAoB;IAC5D,IAAI,IAAI,EAAE,KAAK,QAAQ;QAAE,OAAO,CAAC,OAAO,GAAG,sEAAsE,CAAC,CAAC;IACnH,MAAM,CAAC,GAAG,IAAI,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC;IACnC,OAAO,CAAC,OAAO,GAAG,uBAAuB,EAAE,yBAAyB,GAAG,4BAA4B;QACjG,YAAY,GAAG,gBAAgB,UAAU,EAAE,qCAAqC;QAChF,YAAY,GAAG,8BAA8B,YAAY,EAAE,EAAE,EAAE,MAAM,CAAC,wCAAwC,CAAC,CAAC;AACpH,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,OAAO,CAAC,GAAmB,EAAE,IAAoB;IAC/D,IAAI,CAAC,GAAG,IAAI,UAAU,EAAE;QAAE,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC9C,IAAI,KAAK,GAAmC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACzD,MAAM,CAAC,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;IAC9B,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,IAAI,GAAG;QAAE,KAAK,GAAG,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;IAC1C,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3C,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE;QAAE,KAAK,GAAG,CAAC,EAAE,EAAE,GAAG,GAAG,uBAAuB,CAAC,CAAC;IACjE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,KAAK,OAAO,EAAE,CAAC;QACpC,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC;QACtC,IAAI,EAAE,EAAE,CAAC;YACP,MAAM,CAAC,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC;YAC1B,IAAI,CAAC;gBAAE,KAAK,GAAG,CAAC,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,CAAC;QAClD,CAAC;QACD,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,eAAe,EAAE,YAAY,EAAE,CAAC,CAAU,EAAE,CAAC;YACtG,IAAI,KAAK,CAAC,CAAC,CAAC;gBAAE,MAAM;YACpB,MAAM,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACjC,IAAI,CAAC;gBAAE,KAAK,GAAG,CAAC,CAAC,EAAE,GAAG,KAAK,KAAK,IAAI,GAAG,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACd,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,IAAI,IAAI,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;YAChF,MAAM,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,CAAC;gBAAE,KAAK,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,CAAC,CAAC;QAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,WAAW,CAAC,IAAU;IACpC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9C,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAChD,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,GAAG;YAAE,MAAM;IACjB,CAAC;IACD,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AACpB,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,QAAQ,CAAC,MAAiC;IACxD,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;QAAE,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AACtD,CAAC;AAED,oGAAoG;AACpG;;;;GAIG;AACH,MAAM,UAAU,UAAU,CAAC,KAAsB,EAAE,IAAY;IAC7D,cAAc,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;AAErE,MAAM,UAAU,MAAM,CAAC,KAAc;IACnC,IAAI,IAAI,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7D,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;QAAE,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACxG,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,WAAW,KAAK,gBAAgB,EAAE,IAAI,CAAC,EAAE,cAAc,CAAC,CAAC;IACxF,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,WAAW,KAAK,2BAA2B,EAAE,IAAI,CAAC,EAAE,cAAc,CAAC,CAAC;IACnG,0FAA0F;IAC1F,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,uGAAuG,EACzH,cAAc,CAAC,CAAC;IAClB,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,cAAc;QAAE,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK;YAAE,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;IACzG,OAAO,IAAI,CAAC;AACd,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,UAAU,CAAI,KAAQ;IACpC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,KAAK,CAAM,CAAC;IACzD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,UAAU,CAAM,CAAC;IAC5D,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACnE,OAAO,GAAQ,CAAC;IAClB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,YAAY;IAC1B,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;AAC3B,CAAC"}
@@ -0,0 +1,41 @@
1
+ import type { Pack } from "./packs.js";
2
+ export declare function bundleDir(out: string): string;
3
+ /** A file name the scripts can write on any system (ASCII, no path separators or tabs). */
4
+ export declare function safeName(name: string): string;
5
+ /** [values, requests]: the tab-separated blocks the script templates embed. "-" marks an empty field. */
6
+ export declare function scriptRows(items: any[]): [string[], string[]];
7
+ /** <stem>.sh and <stem>.ps1 with every item built in. */
8
+ export declare function writeScripts(d: string, items: any[], stem?: string): {
9
+ bash: string;
10
+ powershell: string;
11
+ };
12
+ export declare function runLines(stem: string, folder?: string): {
13
+ bash: string;
14
+ powershell: string;
15
+ };
16
+ export declare function runSection(stem: string, keys?: [string, string][], folder?: string, specName?: string): string[];
17
+ export type Bundle = {
18
+ dir: string;
19
+ instructions: string;
20
+ scripts: {
21
+ bash: string;
22
+ powershell: string;
23
+ };
24
+ spec: string | null;
25
+ commands: {
26
+ bash: string;
27
+ powershell: string;
28
+ };
29
+ items: any[];
30
+ };
31
+ /** groups: [[pack, items], ...]; specs: {pack id: spec}. One bundle for every source. */
32
+ export declare function writeBundle(out: string, groups: [Pack, any[]][], reason: string, specs?: Record<string, any>): Bundle;
33
+ /** The given files, and every file inside the given folders. */
34
+ export declare function candidateFiles(paths: string[]): string[];
35
+ /**
36
+ * Map each expected item to an uploaded file: exact filename, then the same name with a browser suffix
37
+ * ("file (1).csv") or another readable extension, then a unique file with the same extension... then the item's
38
+ * alt_filenames. Returns [found {item name: path}, missing filenames].
39
+ */
40
+ export declare function matchUploads(paths: string[], items: any[]): [Record<string, string>, string[]];
41
+ export declare function copyIn(found: Record<string, string>, dest: string): Record<string, string>;
@@ -0,0 +1,218 @@
1
+ /**
2
+ * Manual-download bundles and ingest, for when this server cannot get the data itself (no key stored here, a
3
+ * host that refuses scripts, a network that blocks it). The user downloads the files on their own computer
4
+ * with one command and uploads them; then `ingest` packages them like a fetch.
5
+ *
6
+ * An item is {name, url, filename, needs_key, method, body, note, optional, also_ext?, alt_filenames?}. A URL that
7
+ * needs a key carries {API_KEY}; the scripts fill it on the user's own computer (from the key variable or a
8
+ * hidden prompt), so the key never reaches this server or the chat. The bundle (<workspace>/manual_download/):
9
+ * INSTRUCTIONS.md, download_data.sh / download_data.ps1 (curl / Windows PowerShell, no Python),
10
+ * spec.json (one source) or plan.json (several), manifest.json
11
+ */
12
+ import fs from "node:fs";
13
+ import path from "node:path";
14
+ import { GUIDE_DIR } from "./env.js";
15
+ export function bundleDir(out) {
16
+ return path.join(path.resolve(out), "manual_download");
17
+ }
18
+ /** A file name the scripts can write on any system (ASCII, no path separators or tabs). */
19
+ export function safeName(name) {
20
+ return String(name).replace(/[^A-Za-z0-9._()+ -]/g, "_").trim() || "file";
21
+ }
22
+ function item(pack, it) {
23
+ const a = pack.policy.auth;
24
+ const needs = !!it.needs_key;
25
+ const x = { source: pack.id, name: it.name, url: it.url, filename: safeName(it.filename), method: it.method || "GET",
26
+ note: it.note || "", needs_key: needs, optional: !!it.optional, key_env: needs ? a.env ?? null : null,
27
+ signup_url: needs ? a.signup_url ?? null : null };
28
+ if (it.body !== undefined && it.body !== null)
29
+ x.body = it.body;
30
+ if (a.extra)
31
+ x.extra_env = Object.values(a.extra).sort();
32
+ if (needs && a.placement === "body")
33
+ Object.assign(x, { body_has_key: true, key_param: a.param });
34
+ if (needs && a.placement === "header")
35
+ Object.assign(x, { key_header: a.param, key_prefix: a.prefix || "" });
36
+ return x;
37
+ }
38
+ /** Percent-encode what would break a script line or a Windows PowerShell 5.1 file (non-ASCII, blanks). */
39
+ function ascii(s) {
40
+ let out = "";
41
+ for (const c of String(s)) {
42
+ const code = c.codePointAt(0);
43
+ out += code > 32 && code < 127 ? c : [...Buffer.from(c, "utf8")].map((b) => "%" + b.toString(16).toUpperCase().padStart(2, "0")).join("");
44
+ }
45
+ return out;
46
+ }
47
+ /** [values, requests]: the tab-separated blocks the script templates embed. "-" marks an empty field. */
48
+ export function scriptRows(items) {
49
+ const values = [], rows = [];
50
+ const seen = new Set();
51
+ for (const it of items) {
52
+ const needs = !!it.needs_key && !!it.key_env;
53
+ if (needs && !seen.has(it.key_env)) {
54
+ seen.add(it.key_env);
55
+ values.push([it.key_env, "secret", "free key: " + ascii(it.signup_url || "")].join("\t"));
56
+ }
57
+ const extras = (it.extra_env || []).filter((v) => it.url.includes(`{${v}}`));
58
+ for (const v of extras) {
59
+ if (!seen.has(v)) {
60
+ seen.add(v);
61
+ values.push([v, "plain", "the value registered with your key"].join("\t"));
62
+ }
63
+ }
64
+ const place = needs ? (it.body_has_key ? "body" : it.key_header ? "header" : "query") : "none";
65
+ let body = it.body;
66
+ if (body !== undefined && body !== null && place === "body")
67
+ body = { ...body, [it.key_param]: "{API_KEY}" };
68
+ const fields = [safeName(it.filename), String(it.method || "GET").toUpperCase(), it.optional ? "1" : "0",
69
+ needs ? it.key_env : "", place, needs ? it.key_header || it.key_param || "" : "", place === "header" ? it.key_prefix || "" : "",
70
+ extras.join(","), ascii(it.url),
71
+ body !== undefined && body !== null ? JSON.stringify(body).replace(/[\u007f-￿]/g, (c) => "\\u" + c.charCodeAt(0).toString(16).padStart(4, "0")) : ""];
72
+ rows.push(fields.map((f) => f.replace(/[\t\r\n]/g, " ") || "-").join("\t"));
73
+ }
74
+ return [values, rows];
75
+ }
76
+ /** <stem>.sh and <stem>.ps1 with every item built in. */
77
+ export function writeScripts(d, items, stem = "download_data") {
78
+ fs.mkdirSync(d, { recursive: true });
79
+ const [values, rows] = scriptRows(items);
80
+ const out = {};
81
+ for (const [kind, ext] of [["bash", ".sh"], ["powershell", ".ps1"]]) {
82
+ let text = fs.readFileSync(path.join(GUIDE_DIR, "assets", "download_data" + ext), "ascii");
83
+ text = text.replaceAll("@STEM@", stem).replaceAll("@VALUES@", values.join("\n")).replaceAll("@ITEMS@", rows.join("\n"));
84
+ text = text.replace(/\r\n/g, "\n");
85
+ if (kind === "powershell")
86
+ text = text.replace(/\n/g, "\r\n");
87
+ const p = path.join(d, stem + ext);
88
+ fs.writeFileSync(p, text, { encoding: "ascii", mode: kind === "bash" ? 0o755 : 0o644 });
89
+ out[kind] = p;
90
+ }
91
+ return out;
92
+ }
93
+ export function runLines(stem, folder = "") {
94
+ if (folder) {
95
+ const f = folder.replace(/^[/\\]+|[/\\]+$/g, "").replaceAll("\\", "/");
96
+ return { bash: `bash ${f}/${stem}.sh`,
97
+ powershell: `powershell -NoProfile -ExecutionPolicy Bypass -File ".\\${f.replaceAll("/", "\\")}\\${stem}.ps1"` };
98
+ }
99
+ return { bash: `bash ~/Downloads/${stem}.sh`,
100
+ powershell: `powershell -NoProfile -ExecutionPolicy Bypass -File "$HOME\\Downloads\\${stem}.ps1"` };
101
+ }
102
+ export function runSection(stem, keys = [], folder = "", specName = "") {
103
+ const c = runLines(stem, folder);
104
+ let lead, tail;
105
+ if (folder) {
106
+ lead = `From the folder that holds \`${folder}/\`, copy and paste the command for your system:`;
107
+ const d = runLines(stem);
108
+ tail = `If you downloaded only the script, run it where you saved it instead: \`${d.bash}\` or \`${d.powershell}\`.`;
109
+ }
110
+ else {
111
+ lead = `Save \`${stem}.sh\` (macOS, Linux) or \`${stem}.ps1\` (Windows) in your Downloads folder, then copy and paste the ` +
112
+ "command for your system into a terminal:";
113
+ tail = "If you saved the script in another folder, put that folder in the command instead of Downloads.";
114
+ }
115
+ const lines = [lead, "", "macOS, Linux, Git Bash or WSL (needs only curl):", "", "```bash", c.bash, "```", "",
116
+ "Windows PowerShell:", "", "```powershell", c.powershell, "```", "", tail, ""];
117
+ const ref = specName ? ` (\`${specName}\` holds the same request, for reference)` : "";
118
+ lines.push(`The requests are built into the script${ref}. It saves every file next to itself and prints their full paths ` +
119
+ "when it finishes.", "");
120
+ if (keys.length) {
121
+ lines.push("It needs your own free API key: the script asks for it with hidden input (or uses the environment variable " +
122
+ "when it is set in that terminal), and the key goes only to the source. Never paste a key into the chat.", "");
123
+ lines.push(...keys.map(([env, url]) => `- \`${env}\`: get one at ${url}`), "");
124
+ }
125
+ return lines;
126
+ }
127
+ function writeSpec(d, specs) {
128
+ const ids = Object.keys(specs || {});
129
+ if (!ids.length)
130
+ return null;
131
+ const [name, data] = ids.length === 1 ? ["spec.json", specs[ids[0]]]
132
+ : ["plan.json", { sources: ids.map((k) => ({ source: k, spec: specs[k] })) }];
133
+ fs.writeFileSync(path.join(d, name), JSON.stringify(data, null, 2) + "\n");
134
+ return name;
135
+ }
136
+ /** groups: [[pack, items], ...]; specs: {pack id: spec}. One bundle for every source. */
137
+ export function writeBundle(out, groups, reason, specs) {
138
+ const d = bundleDir(out);
139
+ fs.mkdirSync(d, { recursive: true });
140
+ const items = groups.flatMap(([p, its]) => its.map((it) => item(p, it)));
141
+ fs.writeFileSync(path.join(d, "manifest.json"), JSON.stringify({ reason, items }, null, 2));
142
+ const scripts = writeScripts(d, items);
143
+ const specName = writeSpec(d, specs);
144
+ const keyed = [...new Map(items.filter((i) => i.needs_key).map((i) => [i.key_env, [i.key_env, i.signup_url]])).values()].sort();
145
+ const lines = ["# Download the data on your own computer", "", `The assistant cannot download these files itself (${reason}).`,
146
+ "Get them with one command on your own computer, then upload every downloaded file to the chat (or zip the folder " +
147
+ "and upload the zip).", "", "## Option A: one command (recommended)", "",
148
+ ...runSection("download_data", keyed, "", specName || ""),
149
+ "## Option B: open each link in a browser", "",
150
+ "Save each file under the name shown. Links that need a key are marked; replace `{API_KEY}` with yours in the browser only.", ""];
151
+ for (const i of items) {
152
+ let mark = i.needs_key ? " (needs key)" : "";
153
+ const how = i.method === "GET" ? "" : ` (a ${i.method} request: use Option A)`;
154
+ if (i.optional)
155
+ mark += " (optional)";
156
+ lines.push(`- **${i.filename}** [${i.source}]${mark}${how}: ${i.url}` + (i.note ? ` \n ${i.note}` : ""));
157
+ }
158
+ lines.push("", "## Upload", "", "Upload every downloaded file (do not rename or edit them). The assistant then reads them and continues.");
159
+ const instructions = path.join(d, "INSTRUCTIONS.md");
160
+ fs.writeFileSync(instructions, lines.join("\n") + "\n");
161
+ return { dir: d, instructions, scripts, spec: specName ? path.join(d, specName) : null, commands: runLines("download_data"), items };
162
+ }
163
+ /** The given files, and every file inside the given folders. */
164
+ export function candidateFiles(paths) {
165
+ const out = [];
166
+ const walk = (p) => {
167
+ if (!fs.existsSync(p))
168
+ return;
169
+ if (fs.statSync(p).isDirectory())
170
+ for (const n of fs.readdirSync(p))
171
+ walk(path.join(p, n));
172
+ else
173
+ out.push(p);
174
+ };
175
+ paths.forEach(walk);
176
+ return out;
177
+ }
178
+ /**
179
+ * Map each expected item to an uploaded file: exact filename, then the same name with a browser suffix
180
+ * ("file (1).csv") or another readable extension, then a unique file with the same extension... then the item's
181
+ * alt_filenames. Returns [found {item name: path}, missing filenames].
182
+ */
183
+ export function matchUploads(paths, items) {
184
+ const files = candidateFiles(paths);
185
+ const byName = new Map(files.map((f) => [path.basename(f).toLowerCase(), f]));
186
+ const found = {};
187
+ let missing = [];
188
+ const look = (filename, also = []) => {
189
+ const want = safeName(filename).toLowerCase();
190
+ const ext = path.extname(want), stem = want.slice(0, want.length - ext.length);
191
+ const exts = [ext, ...also.map((e) => e.toLowerCase())];
192
+ return byName.get(want) ?? [...byName].find(([n]) => n.startsWith(stem) && exts.some((e) => n.endsWith(e)))?.[1];
193
+ };
194
+ for (const it of items) {
195
+ const hit = look(it.filename, it.also_ext || []) ?? (it.alt_filenames || []).map((a) => look(a)).find(Boolean);
196
+ if (hit)
197
+ found[it.name] = hit;
198
+ else if (!it.optional)
199
+ missing.push(it.filename + (it.alt_filenames?.length ? ` (or ${it.alt_filenames.join(" / ")})` : ""));
200
+ }
201
+ if (missing.length && items.length === 1 && files.length === 1) {
202
+ found[items[0].name] = files[0];
203
+ missing = [];
204
+ }
205
+ return [found, missing];
206
+ }
207
+ export function copyIn(found, dest) {
208
+ fs.mkdirSync(dest, { recursive: true });
209
+ const res = {};
210
+ for (const [name, src] of Object.entries(found)) {
211
+ const tgt = path.join(dest, path.basename(src));
212
+ if (path.resolve(src) !== path.resolve(tgt))
213
+ fs.copyFileSync(src, tgt);
214
+ res[name] = tgt;
215
+ }
216
+ return res;
217
+ }
218
+ //# sourceMappingURL=manual.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"manual.js","sourceRoot":"","sources":["../../src/core/manual.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAGrC,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,iBAAiB,CAAC,CAAC;AACzD,CAAC;AAED,2FAA2F;AAC3F,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC;AAC5E,CAAC;AAED,SAAS,IAAI,CAAC,IAAU,EAAE,EAAO;IAC/B,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IAC3B,MAAM,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC;IAC7B,MAAM,CAAC,GAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,IAAI,KAAK;QACvH,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI;QACrG,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpD,IAAI,EAAE,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI;QAAE,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC;IAChE,IAAI,CAAC,CAAC,KAAK;QAAE,CAAC,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,IAAI,KAAK,IAAI,CAAC,CAAC,SAAS,KAAK,MAAM;QAAE,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IAClG,IAAI,KAAK,IAAI,CAAC,CAAC,SAAS,KAAK,QAAQ;QAAE,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC,CAAC;IAC7G,OAAO,CAAC,CAAC;AACX,CAAC;AAED,0GAA0G;AAC1G,SAAS,KAAK,CAAC,CAAU;IACvB,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC;QAC/B,GAAG,IAAI,IAAI,GAAG,EAAE,IAAI,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5I,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,yGAAyG;AACzG,MAAM,UAAU,UAAU,CAAC,KAAY;IACrC,MAAM,MAAM,GAAa,EAAE,EAAE,IAAI,GAAa,EAAE,CAAC;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC,SAAS,IAAI,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC;QAC7C,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;YACnC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,YAAY,GAAG,KAAK,CAAC,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5F,CAAC;QACD,MAAM,MAAM,GAAG,CAAC,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACrF,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACZ,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,oCAAoC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/F,IAAI,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC;QACnB,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,KAAK,MAAM;YAAE,IAAI,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;QAC7G,MAAM,MAAM,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;YACtG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE;YAC/H,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC;YAC/B,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACxJ,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AACxB,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,KAAY,EAAE,IAAI,GAAG,eAAe;IAC1E,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACrC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,GAAG,GAAQ,EAAE,CAAC;IACpB,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,CAAC,CAAU,EAAE,CAAC;QAC7E,IAAI,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,eAAe,GAAG,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC;QAC3F,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACxH,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,YAAY;YAAE,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9D,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,GAAG,GAAG,CAAC,CAAC;QACnC,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACxF,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,MAAM,GAAG,EAAE;IAChD,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACvE,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,KAAK;YACnC,UAAU,EAAE,2DAA2D,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,IAAI,OAAO,EAAE,CAAC;IACrH,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,oBAAoB,IAAI,KAAK;QAC1C,UAAU,EAAE,0EAA0E,IAAI,OAAO,EAAE,CAAC;AACxG,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,OAA2B,EAAE,EAAE,MAAM,GAAG,EAAE,EAAE,QAAQ,GAAG,EAAE;IAChG,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACjC,IAAI,IAAY,EAAE,IAAY,CAAC;IAC/B,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,GAAG,gCAAgC,MAAM,kDAAkD,CAAC;QAChG,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QACzB,IAAI,GAAG,2EAA2E,CAAC,CAAC,IAAI,WAAW,CAAC,CAAC,UAAU,KAAK,CAAC;IACvH,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,UAAU,IAAI,6BAA6B,IAAI,qEAAqE;YACzH,0CAA0C,CAAC;QAC7C,IAAI,GAAG,iGAAiG,CAAC;IAC3G,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,IAAI,EAAE,EAAE,EAAE,kDAAkD,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC3G,qBAAqB,EAAE,EAAE,EAAE,eAAe,EAAE,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IACjF,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,QAAQ,2CAA2C,CAAC,CAAC,CAAC,EAAE,CAAC;IACvF,KAAK,CAAC,IAAI,CAAC,yCAAyC,GAAG,mEAAmE;QACxH,mBAAmB,EAAE,EAAE,CAAC,CAAC;IAC3B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,6GAA6G;YACtH,yGAAyG,EAAE,EAAE,CAAC,CAAC;QACjH,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,OAAO,GAAG,kBAAkB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,CAAS,EAAE,KAA2B;IACvD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACrC,IAAI,CAAC,GAAG,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC7B,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,KAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,KAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;IACjF,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAC3E,OAAO,IAAI,CAAC;AACd,CAAC;AAKD,yFAAyF;AACzF,MAAM,UAAU,WAAW,CAAC,GAAW,EAAE,MAAuB,EAAE,MAAc,EAAE,KAA2B;IAC3G,MAAM,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IACzB,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACzE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5F,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,UAAU,CAAqB,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACpJ,MAAM,KAAK,GAAG,CAAC,0CAA0C,EAAE,EAAE,EAAE,qDAAqD,MAAM,IAAI;QAC5H,mHAAmH;YACnH,sBAAsB,EAAE,EAAE,EAAE,wCAAwC,EAAE,EAAE;QACxE,GAAG,UAAU,CAAC,eAAe,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,IAAI,EAAE,CAAC;QACzD,0CAA0C,EAAE,EAAE;QAC9C,4HAA4H,EAAE,EAAE,CAAC,CAAC;IACpI,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,IAAI,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,yBAAyB,CAAC;QAC/E,IAAI,CAAC,CAAC,QAAQ;YAAE,IAAI,IAAI,aAAa,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,OAAO,CAAC,CAAC,MAAM,IAAI,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7G,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,yGAAyG,CAAC,CAAC;IAC3I,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;IACrD,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;IACxD,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,CAAC;AACvI,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,cAAc,CAAC,KAAe;IAC5C,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE;QACzB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;YAAE,OAAO;QAC9B,IAAI,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE;YAAE,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;;YACtF,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC,CAAC;IACF,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACpB,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,KAAe,EAAE,KAAY;IACxD,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,IAAI,OAAO,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,QAAgB,EAAE,OAAiB,EAAE,EAAE,EAAE;QACrD,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;QAC/E,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACxD,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACnH,CAAC,CAAC;IACF,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvH,IAAI,GAAG;YAAE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;aACzB,IAAI,CAAC,EAAE,CAAC,QAAQ;YAAE,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/H,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/D,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAChC,OAAO,GAAG,EAAE,CAAC;IACf,CAAC;IACD,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,KAA6B,EAAE,IAAY;IAChE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxC,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAChD,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;YAAE,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACvE,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;IAClB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
@@ -0,0 +1,58 @@
1
+ import type { SourceRecord } from "./context.js";
2
+ import type { Pack, Row, Spec } from "./packs.js";
3
+ export declare function dataDir(out: string, source: string): string;
4
+ export declare function fingerprint(spec: unknown): string;
5
+ export type Existing = {
6
+ path: string;
7
+ matches: boolean;
8
+ complete: boolean;
9
+ retrieved: string;
10
+ age_days: number | null;
11
+ route: string;
12
+ problems: string[];
13
+ };
14
+ export declare function existing(out: string, source: string, spec: Spec): Existing | null;
15
+ /**
16
+ * A table already written to a CSV file (header row first), for microdata too large to hold as rows: an adapter
17
+ * streams the rows to `path` (a temporary file outside the package) and returns this in place of a list of rows;
18
+ * write() moves the file into data/<source>/ and builds the dictionary from `sample` and `nonMissing`.
19
+ */
20
+ export declare class FileTable {
21
+ path: string;
22
+ columns: string[];
23
+ rows: number;
24
+ sample: Row[];
25
+ nonMissing: Record<string, number>;
26
+ constructor(path: string, columns: string[], rows: number, sample?: Row[], nonMissing?: Record<string, number>);
27
+ get length(): number;
28
+ discard(): void;
29
+ }
30
+ export type Table = Row[] | FileTable;
31
+ /** The rows, marked with the column order to write (Python's dict order, for keys JS would reorder). */
32
+ export declare function withColumns<T extends Row[]>(rows: T, columns: string[]): T;
33
+ export declare function columnsOf(rows: unknown): string[] | undefined;
34
+ export type TableMeta = {
35
+ name: string;
36
+ file: string;
37
+ rows: number;
38
+ columns: string[];
39
+ sha256: string;
40
+ };
41
+ export type WriteResult = {
42
+ data_dir: string;
43
+ tables: TableMeta[];
44
+ retrieved: string;
45
+ citation: string;
46
+ };
47
+ export declare function write(out: string, pack: Pack, tables: Record<string, Table>, spec: Spec, o: {
48
+ route: string;
49
+ acquired?: string;
50
+ requests?: SourceRecord[];
51
+ raw?: Record<string, Buffer>;
52
+ notes?: string[];
53
+ problems?: string[];
54
+ vintage?: string;
55
+ extraReadme?: string;
56
+ }): WriteResult;
57
+ /** The provenance of earlier fetches whose tables are still in the folder and not replaced now. */
58
+ export declare function earlierFetches(d: string, newFiles: Set<string>): any[];