data-analyst-mcp 0.0.0-stage → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (640) hide show
  1. package/README.md +262 -3
  2. package/dist/adapters/_acf_tidy.d.ts +18 -0
  3. package/dist/adapters/_acf_tidy.js +432 -0
  4. package/dist/adapters/_acf_tidy.js.map +1 -0
  5. package/dist/adapters/_actuaries_read.d.ts +38 -0
  6. package/dist/adapters/_actuaries_read.js +728 -0
  7. package/dist/adapters/_actuaries_read.js.map +1 -0
  8. package/dist/adapters/_census_micro.d.ts +70 -0
  9. package/dist/adapters/_census_micro.js +622 -0
  10. package/dist/adapters/_census_micro.js.map +1 -0
  11. package/dist/adapters/_census_util.d.ts +40 -0
  12. package/dist/adapters/_census_util.js +275 -0
  13. package/dist/adapters/_census_util.js.map +1 -0
  14. package/dist/adapters/_dol_states.d.ts +6 -0
  15. package/dist/adapters/_dol_states.js +58 -0
  16. package/dist/adapters/_dol_states.js.map +1 -0
  17. package/dist/adapters/_govinfo_entities.d.ts +4 -0
  18. package/dist/adapters/_govinfo_entities.js +6 -0
  19. package/dist/adapters/_govinfo_entities.js.map +1 -0
  20. package/dist/adapters/_govinfo_json.d.ts +13 -0
  21. package/dist/adapters/_govinfo_json.js +107 -0
  22. package/dist/adapters/_govinfo_json.js.map +1 -0
  23. package/dist/adapters/_govinfo_py.d.ts +22 -0
  24. package/dist/adapters/_govinfo_py.js +272 -0
  25. package/dist/adapters/_govinfo_py.js.map +1 -0
  26. package/dist/adapters/_govinfo_xml.d.ts +22 -0
  27. package/dist/adapters/_govinfo_xml.js +233 -0
  28. package/dist/adapters/_govinfo_xml.js.map +1 -0
  29. package/dist/adapters/_hud_excel.d.ts +2 -0
  30. package/dist/adapters/_hud_excel.js +166 -0
  31. package/dist/adapters/_hud_excel.js.map +1 -0
  32. package/dist/adapters/_omb_grid.d.ts +17 -0
  33. package/dist/adapters/_omb_grid.js +132 -0
  34. package/dist/adapters/_omb_grid.js.map +1 -0
  35. package/dist/adapters/_omb_util.d.ts +40 -0
  36. package/dist/adapters/_omb_util.js +192 -0
  37. package/dist/adapters/_omb_util.js.map +1 -0
  38. package/dist/adapters/_usaspending_state.d.ts +6 -0
  39. package/dist/adapters/_usaspending_state.js +54 -0
  40. package/dist/adapters/_usaspending_state.js.map +1 -0
  41. package/dist/adapters/_usda-fna_book.d.ts +59 -0
  42. package/dist/adapters/_usda-fna_book.js +869 -0
  43. package/dist/adapters/_usda-fna_book.js.map +1 -0
  44. package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
  45. package/dist/adapters/_usda-fna_pdf.js +361 -0
  46. package/dist/adapters/_usda-fna_pdf.js.map +1 -0
  47. package/dist/adapters/_usda-outlook_read.d.ts +70 -0
  48. package/dist/adapters/_usda-outlook_read.js +362 -0
  49. package/dist/adapters/_usda-outlook_read.js.map +1 -0
  50. package/dist/adapters/acf.d.ts +3 -0
  51. package/dist/adapters/acf.js +187 -0
  52. package/dist/adapters/acf.js.map +1 -0
  53. package/dist/adapters/actuaries.d.ts +3 -0
  54. package/dist/adapters/actuaries.js +293 -0
  55. package/dist/adapters/actuaries.js.map +1 -0
  56. package/dist/adapters/bts.d.ts +3 -0
  57. package/dist/adapters/bts.js +26 -0
  58. package/dist/adapters/bts.js.map +1 -0
  59. package/dist/adapters/census-geocoder.d.ts +10 -0
  60. package/dist/adapters/census-geocoder.js +47 -0
  61. package/dist/adapters/census-geocoder.js.map +1 -0
  62. package/dist/adapters/census.d.ts +17 -0
  63. package/dist/adapters/census.js +983 -0
  64. package/dist/adapters/census.js.map +1 -0
  65. package/dist/adapters/cfpb.d.ts +7 -0
  66. package/dist/adapters/cfpb.js +348 -0
  67. package/dist/adapters/cfpb.js.map +1 -0
  68. package/dist/adapters/clinical-trials.d.ts +3 -0
  69. package/dist/adapters/clinical-trials.js +135 -0
  70. package/dist/adapters/clinical-trials.js.map +1 -0
  71. package/dist/adapters/cms.d.ts +3 -0
  72. package/dist/adapters/cms.js +292 -0
  73. package/dist/adapters/cms.js.map +1 -0
  74. package/dist/adapters/college-scorecard.d.ts +3 -0
  75. package/dist/adapters/college-scorecard.js +122 -0
  76. package/dist/adapters/college-scorecard.js.map +1 -0
  77. package/dist/adapters/congress.d.ts +33 -0
  78. package/dist/adapters/congress.js +589 -0
  79. package/dist/adapters/congress.js.map +1 -0
  80. package/dist/adapters/dhs-ohss.d.ts +13 -0
  81. package/dist/adapters/dhs-ohss.js +598 -0
  82. package/dist/adapters/dhs-ohss.js.map +1 -0
  83. package/dist/adapters/doj-news.d.ts +7 -0
  84. package/dist/adapters/doj-news.js +286 -0
  85. package/dist/adapters/doj-news.js.map +1 -0
  86. package/dist/adapters/dol.d.ts +4 -0
  87. package/dist/adapters/dol.js +634 -0
  88. package/dist/adapters/dol.js.map +1 -0
  89. package/dist/adapters/eia.d.ts +15 -0
  90. package/dist/adapters/eia.js +456 -0
  91. package/dist/adapters/eia.js.map +1 -0
  92. package/dist/adapters/epa-aqs.d.ts +3 -0
  93. package/dist/adapters/epa-aqs.js +357 -0
  94. package/dist/adapters/epa-aqs.js.map +1 -0
  95. package/dist/adapters/epa.d.ts +3 -0
  96. package/dist/adapters/epa.js +298 -0
  97. package/dist/adapters/epa.js.map +1 -0
  98. package/dist/adapters/fbi.d.ts +3 -0
  99. package/dist/adapters/fbi.js +192 -0
  100. package/dist/adapters/fbi.js.map +1 -0
  101. package/dist/adapters/fda.d.ts +5 -0
  102. package/dist/adapters/fda.js +184 -0
  103. package/dist/adapters/fda.js.map +1 -0
  104. package/dist/adapters/fdic.d.ts +4 -0
  105. package/dist/adapters/fdic.js +95 -0
  106. package/dist/adapters/fdic.js.map +1 -0
  107. package/dist/adapters/fec.d.ts +3 -0
  108. package/dist/adapters/fec.js +142 -0
  109. package/dist/adapters/fec.js.map +1 -0
  110. package/dist/adapters/fed-board.d.ts +12 -0
  111. package/dist/adapters/fed-board.js +581 -0
  112. package/dist/adapters/fed-board.js.map +1 -0
  113. package/dist/adapters/federal-register.d.ts +11 -0
  114. package/dist/adapters/federal-register.js +43 -0
  115. package/dist/adapters/federal-register.js.map +1 -0
  116. package/dist/adapters/fhfa.d.ts +3 -0
  117. package/dist/adapters/fhfa.js +253 -0
  118. package/dist/adapters/fhfa.js.map +1 -0
  119. package/dist/adapters/fsa.d.ts +9 -0
  120. package/dist/adapters/fsa.js +335 -0
  121. package/dist/adapters/fsa.js.map +1 -0
  122. package/dist/adapters/govinfo.d.ts +30 -0
  123. package/dist/adapters/govinfo.js +636 -0
  124. package/dist/adapters/govinfo.js.map +1 -0
  125. package/dist/adapters/gsa-calc.d.ts +10 -0
  126. package/dist/adapters/gsa-calc.js +318 -0
  127. package/dist/adapters/gsa-calc.js.map +1 -0
  128. package/dist/adapters/hud.d.ts +3 -0
  129. package/dist/adapters/hud.js +200 -0
  130. package/dist/adapters/hud.js.map +1 -0
  131. package/dist/adapters/index.d.ts +3 -0
  132. package/dist/adapters/index.js +25 -0
  133. package/dist/adapters/index.js.map +1 -0
  134. package/dist/adapters/irs-soi.d.ts +6 -0
  135. package/dist/adapters/irs-soi.js +576 -0
  136. package/dist/adapters/irs-soi.js.map +1 -0
  137. package/dist/adapters/naep.d.ts +15 -0
  138. package/dist/adapters/naep.js +341 -0
  139. package/dist/adapters/naep.js.map +1 -0
  140. package/dist/adapters/nhtsa.d.ts +10 -0
  141. package/dist/adapters/nhtsa.js +239 -0
  142. package/dist/adapters/nhtsa.js.map +1 -0
  143. package/dist/adapters/nih.d.ts +5 -0
  144. package/dist/adapters/nih.js +196 -0
  145. package/dist/adapters/nih.js.map +1 -0
  146. package/dist/adapters/noaa.d.ts +3 -0
  147. package/dist/adapters/noaa.js +148 -0
  148. package/dist/adapters/noaa.js.map +1 -0
  149. package/dist/adapters/nrel.d.ts +3 -0
  150. package/dist/adapters/nrel.js +186 -0
  151. package/dist/adapters/nrel.js.map +1 -0
  152. package/dist/adapters/nws.d.ts +3 -0
  153. package/dist/adapters/nws.js +119 -0
  154. package/dist/adapters/nws.js.map +1 -0
  155. package/dist/adapters/omb.d.ts +15 -0
  156. package/dist/adapters/omb.js +517 -0
  157. package/dist/adapters/omb.js.map +1 -0
  158. package/dist/adapters/open-payments.d.ts +5 -0
  159. package/dist/adapters/open-payments.js +149 -0
  160. package/dist/adapters/open-payments.js.map +1 -0
  161. package/dist/adapters/pbgc.d.ts +13 -0
  162. package/dist/adapters/pbgc.js +283 -0
  163. package/dist/adapters/pbgc.js.map +1 -0
  164. package/dist/adapters/philfed.d.ts +12 -0
  165. package/dist/adapters/philfed.js +59 -0
  166. package/dist/adapters/philfed.js.map +1 -0
  167. package/dist/adapters/regulations.d.ts +6 -0
  168. package/dist/adapters/regulations.js +58 -0
  169. package/dist/adapters/regulations.js.map +1 -0
  170. package/dist/adapters/sec.d.ts +13 -0
  171. package/dist/adapters/sec.js +504 -0
  172. package/dist/adapters/sec.js.map +1 -0
  173. package/dist/adapters/senate-lobbying.d.ts +3 -0
  174. package/dist/adapters/senate-lobbying.js +181 -0
  175. package/dist/adapters/senate-lobbying.js.map +1 -0
  176. package/dist/adapters/snapqc.d.ts +32 -0
  177. package/dist/adapters/snapqc.js +701 -0
  178. package/dist/adapters/snapqc.js.map +1 -0
  179. package/dist/adapters/treasury-yields.d.ts +3 -0
  180. package/dist/adapters/treasury-yields.js +170 -0
  181. package/dist/adapters/treasury-yields.js.map +1 -0
  182. package/dist/adapters/usaspending.d.ts +3 -0
  183. package/dist/adapters/usaspending.js +340 -0
  184. package/dist/adapters/usaspending.js.map +1 -0
  185. package/dist/adapters/usda-fna.d.ts +7 -0
  186. package/dist/adapters/usda-fna.js +707 -0
  187. package/dist/adapters/usda-fna.js.map +1 -0
  188. package/dist/adapters/usda-fooddata.d.ts +3 -0
  189. package/dist/adapters/usda-fooddata.js +101 -0
  190. package/dist/adapters/usda-fooddata.js.map +1 -0
  191. package/dist/adapters/usda-nass.d.ts +3 -0
  192. package/dist/adapters/usda-nass.js +108 -0
  193. package/dist/adapters/usda-nass.js.map +1 -0
  194. package/dist/adapters/usda-outlook.d.ts +12 -0
  195. package/dist/adapters/usda-outlook.js +696 -0
  196. package/dist/adapters/usda-outlook.js.map +1 -0
  197. package/dist/adapters/usgs.d.ts +9 -0
  198. package/dist/adapters/usgs.js +265 -0
  199. package/dist/adapters/usgs.js.map +1 -0
  200. package/dist/adapters/usitc.d.ts +18 -0
  201. package/dist/adapters/usitc.js +170 -0
  202. package/dist/adapters/usitc.js.map +1 -0
  203. package/dist/adapters/uspto.d.ts +3 -0
  204. package/dist/adapters/uspto.js +47 -0
  205. package/dist/adapters/uspto.js.map +1 -0
  206. package/dist/adapters/world-bank.d.ts +3 -0
  207. package/dist/adapters/world-bank.js +163 -0
  208. package/dist/adapters/world-bank.js.map +1 -0
  209. package/dist/analysis/cite.d.ts +31 -0
  210. package/dist/analysis/cite.js +229 -0
  211. package/dist/analysis/cite.js.map +1 -0
  212. package/dist/analysis/combine.d.ts +27 -0
  213. package/dist/analysis/combine.js +198 -0
  214. package/dist/analysis/combine.js.map +1 -0
  215. package/dist/analysis/compute.d.ts +62 -0
  216. package/dist/analysis/compute.js +583 -0
  217. package/dist/analysis/compute.js.map +1 -0
  218. package/dist/analysis/expr.d.ts +69 -0
  219. package/dist/analysis/expr.js +780 -0
  220. package/dist/analysis/expr.js.map +1 -0
  221. package/dist/analysis/factfmt.d.ts +16 -0
  222. package/dist/analysis/factfmt.js +187 -0
  223. package/dist/analysis/factfmt.js.map +1 -0
  224. package/dist/analysis/facts.d.ts +63 -0
  225. package/dist/analysis/facts.js +355 -0
  226. package/dist/analysis/facts.js.map +1 -0
  227. package/dist/analysis/handoff.d.ts +58 -0
  228. package/dist/analysis/handoff.js +208 -0
  229. package/dist/analysis/handoff.js.map +1 -0
  230. package/dist/analysis/pyfmt.d.ts +16 -0
  231. package/dist/analysis/pyfmt.js +156 -0
  232. package/dist/analysis/pyfmt.js.map +1 -0
  233. package/dist/analysis/rerun.d.ts +13 -0
  234. package/dist/analysis/rerun.js +125 -0
  235. package/dist/analysis/rerun.js.map +1 -0
  236. package/dist/analysis/table.d.ts +18 -0
  237. package/dist/analysis/table.js +109 -0
  238. package/dist/analysis/table.js.map +1 -0
  239. package/dist/analysis/tools.d.ts +67 -0
  240. package/dist/analysis/tools.js +217 -0
  241. package/dist/analysis/tools.js.map +1 -0
  242. package/dist/analysis/write_table.d.ts +13 -0
  243. package/dist/analysis/write_table.js +59 -0
  244. package/dist/analysis/write_table.js.map +1 -0
  245. package/dist/bin/cli.d.ts +10 -0
  246. package/dist/bin/cli.js +177 -0
  247. package/dist/bin/cli.js.map +1 -0
  248. package/dist/core/cache.d.ts +14 -0
  249. package/dist/core/cache.js +90 -0
  250. package/dist/core/cache.js.map +1 -0
  251. package/dist/core/context.d.ts +12 -0
  252. package/dist/core/context.js +15 -0
  253. package/dist/core/context.js.map +1 -0
  254. package/dist/core/env.d.ts +14 -0
  255. package/dist/core/env.js +52 -0
  256. package/dist/core/env.js.map +1 -0
  257. package/dist/core/generic-rest.d.ts +61 -0
  258. package/dist/core/generic-rest.js +639 -0
  259. package/dist/core/generic-rest.js.map +1 -0
  260. package/dist/core/http.d.ts +120 -0
  261. package/dist/core/http.js +514 -0
  262. package/dist/core/http.js.map +1 -0
  263. package/dist/core/keys.d.ts +43 -0
  264. package/dist/core/keys.js +174 -0
  265. package/dist/core/keys.js.map +1 -0
  266. package/dist/core/manual.d.ts +41 -0
  267. package/dist/core/manual.js +218 -0
  268. package/dist/core/manual.js.map +1 -0
  269. package/dist/core/package.d.ts +58 -0
  270. package/dist/core/package.js +228 -0
  271. package/dist/core/package.js.map +1 -0
  272. package/dist/core/packs.d.ts +139 -0
  273. package/dist/core/packs.js +173 -0
  274. package/dist/core/packs.js.map +1 -0
  275. package/dist/core/policy.d.ts +69 -0
  276. package/dist/core/policy.js +189 -0
  277. package/dist/core/policy.js.map +1 -0
  278. package/dist/core/py.d.ts +40 -0
  279. package/dist/core/py.js +171 -0
  280. package/dist/core/py.js.map +1 -0
  281. package/dist/core/routing.d.ts +10 -0
  282. package/dist/core/routing.js +39 -0
  283. package/dist/core/routing.js.map +1 -0
  284. package/dist/core/tabular.d.ts +21 -0
  285. package/dist/core/tabular.js +229 -0
  286. package/dist/core/tabular.js.map +1 -0
  287. package/dist/deep/_bea_api.d.ts +102 -0
  288. package/dist/deep/_bea_api.js +457 -0
  289. package/dist/deep/_bea_api.js.map +1 -0
  290. package/dist/deep/_bea_fetch.d.ts +151 -0
  291. package/dist/deep/_bea_fetch.js +1274 -0
  292. package/dist/deep/_bea_fetch.js.map +1 -0
  293. package/dist/deep/_bea_util.d.ts +14 -0
  294. package/dist/deep/_bea_util.js +156 -0
  295. package/dist/deep/_bea_util.js.map +1 -0
  296. package/dist/deep/_bls_api.d.ts +84 -0
  297. package/dist/deep/_bls_api.js +231 -0
  298. package/dist/deep/_bls_api.js.map +1 -0
  299. package/dist/deep/_bls_ids.d.ts +43 -0
  300. package/dist/deep/_bls_ids.js +188 -0
  301. package/dist/deep/_bls_ids.js.map +1 -0
  302. package/dist/deep/_bls_lib.d.ts +61 -0
  303. package/dist/deep/_bls_lib.js +442 -0
  304. package/dist/deep/_bls_lib.js.map +1 -0
  305. package/dist/deep/_fred_api.d.ts +148 -0
  306. package/dist/deep/_fred_api.js +545 -0
  307. package/dist/deep/_fred_api.js.map +1 -0
  308. package/dist/deep/_fred_pack.d.ts +162 -0
  309. package/dist/deep/_fred_pack.js +1027 -0
  310. package/dist/deep/_fred_pack.js.map +1 -0
  311. package/dist/deep/_fred_py.d.ts +40 -0
  312. package/dist/deep/_fred_py.js +199 -0
  313. package/dist/deep/_fred_py.js.map +1 -0
  314. package/dist/deep/_ipums_bundle.d.ts +29 -0
  315. package/dist/deep/_ipums_bundle.js +194 -0
  316. package/dist/deep/_ipums_bundle.js.map +1 -0
  317. package/dist/deep/_ipums_ddi.d.ts +23 -0
  318. package/dist/deep/_ipums_ddi.js +162 -0
  319. package/dist/deep/_ipums_ddi.js.map +1 -0
  320. package/dist/deep/_opm_io.d.ts +26 -0
  321. package/dist/deep/_opm_io.js +383 -0
  322. package/dist/deep/_opm_io.js.map +1 -0
  323. package/dist/deep/_opm_parquet.d.ts +31 -0
  324. package/dist/deep/_opm_parquet.js +744 -0
  325. package/dist/deep/_opm_parquet.js.map +1 -0
  326. package/dist/deep/_opm_py.d.ts +6 -0
  327. package/dist/deep/_opm_py.js +74 -0
  328. package/dist/deep/_opm_py.js.map +1 -0
  329. package/dist/deep/_treasury_api.d.ts +48 -0
  330. package/dist/deep/_treasury_api.js +221 -0
  331. package/dist/deep/_treasury_api.js.map +1 -0
  332. package/dist/deep/_treasury_context.d.ts +34 -0
  333. package/dist/deep/_treasury_context.js +142 -0
  334. package/dist/deep/_treasury_context.js.map +1 -0
  335. package/dist/deep/_treasury_lib.d.ts +109 -0
  336. package/dist/deep/_treasury_lib.js +686 -0
  337. package/dist/deep/_treasury_lib.js.map +1 -0
  338. package/dist/deep/bea.d.ts +3 -0
  339. package/dist/deep/bea.js +248 -0
  340. package/dist/deep/bea.js.map +1 -0
  341. package/dist/deep/bls.d.ts +3 -0
  342. package/dist/deep/bls.js +220 -0
  343. package/dist/deep/bls.js.map +1 -0
  344. package/dist/deep/fred.d.ts +3 -0
  345. package/dist/deep/fred.js +325 -0
  346. package/dist/deep/fred.js.map +1 -0
  347. package/dist/deep/ipums.d.ts +59 -0
  348. package/dist/deep/ipums.js +744 -0
  349. package/dist/deep/ipums.js.map +1 -0
  350. package/dist/deep/opm.d.ts +56 -0
  351. package/dist/deep/opm.js +1033 -0
  352. package/dist/deep/opm.js.map +1 -0
  353. package/dist/deep/treasury.d.ts +3 -0
  354. package/dist/deep/treasury.js +218 -0
  355. package/dist/deep/treasury.js.map +1 -0
  356. package/dist/envelope.d.ts +47 -0
  357. package/dist/envelope.js +88 -0
  358. package/dist/envelope.js.map +1 -0
  359. package/dist/index.d.ts +277 -0
  360. package/dist/index.js +106 -0
  361. package/dist/index.js.map +1 -0
  362. package/dist/remote/app.d.ts +21 -0
  363. package/dist/remote/app.js +218 -0
  364. package/dist/remote/app.js.map +1 -0
  365. package/dist/remote/oauth.d.ts +14 -0
  366. package/dist/remote/oauth.js +90 -0
  367. package/dist/remote/oauth.js.map +1 -0
  368. package/dist/remote/pages.d.ts +8 -0
  369. package/dist/remote/pages.js +56 -0
  370. package/dist/remote/pages.js.map +1 -0
  371. package/dist/remote/sign.d.ts +4 -0
  372. package/dist/remote/sign.js +40 -0
  373. package/dist/remote/sign.js.map +1 -0
  374. package/dist/remote/store.d.ts +31 -0
  375. package/dist/remote/store.js +129 -0
  376. package/dist/remote/store.js.map +1 -0
  377. package/dist/report/audit_answer.d.ts +94 -0
  378. package/dist/report/audit_answer.js +1349 -0
  379. package/dist/report/audit_answer.js.map +1 -0
  380. package/dist/report/audit_report.d.ts +23 -0
  381. package/dist/report/audit_report.js +637 -0
  382. package/dist/report/audit_report.js.map +1 -0
  383. package/dist/report/build_report.d.ts +27 -0
  384. package/dist/report/build_report.js +1168 -0
  385. package/dist/report/build_report.js.map +1 -0
  386. package/dist/report/method_audit.d.ts +47 -0
  387. package/dist/report/method_audit.js +710 -0
  388. package/dist/report/method_audit.js.map +1 -0
  389. package/dist/report/policyset.d.ts +50 -0
  390. package/dist/report/policyset.js +421 -0
  391. package/dist/report/policyset.js.map +1 -0
  392. package/dist/report/py.d.ts +58 -0
  393. package/dist/report/py.js +311 -0
  394. package/dist/report/py.js.map +1 -0
  395. package/dist/report/scan_secrets.d.ts +21 -0
  396. package/dist/report/scan_secrets.js +234 -0
  397. package/dist/report/scan_secrets.js.map +1 -0
  398. package/dist/report/style_check.d.ts +29 -0
  399. package/dist/report/style_check.js +173 -0
  400. package/dist/report/style_check.js.map +1 -0
  401. package/dist/report/tools.d.ts +87 -0
  402. package/dist/report/tools.js +167 -0
  403. package/dist/report/tools.js.map +1 -0
  404. package/dist/server.d.ts +5 -0
  405. package/dist/server.js +246 -0
  406. package/dist/server.js.map +1 -0
  407. package/dist/session.d.ts +47 -0
  408. package/dist/session.js +107 -0
  409. package/dist/session.js.map +1 -0
  410. package/dist/toolkit.d.ts +21 -0
  411. package/dist/toolkit.js +2 -0
  412. package/dist/toolkit.js.map +1 -0
  413. package/dist/tools.d.ts +299 -0
  414. package/dist/tools.js +491 -0
  415. package/dist/tools.js.map +1 -0
  416. package/guide/WORKFLOW.md +134 -0
  417. package/guide/assets/disclaimer.txt +1 -0
  418. package/guide/assets/download_data.ps1 +162 -0
  419. package/guide/assets/download_data.sh +143 -0
  420. package/guide/assets/report_template.html +597 -0
  421. package/guide/references/report_patterns.md +199 -0
  422. package/guide/references/routing.md +171 -0
  423. package/guide/references/style_guide.md +398 -0
  424. package/package.json +56 -6
  425. package/packs/acf/endpoints.json +114 -0
  426. package/packs/acf/policy.json +104 -0
  427. package/packs/acf/source.md +138 -0
  428. package/packs/actuaries/endpoints.json +82 -0
  429. package/packs/actuaries/policy.json +109 -0
  430. package/packs/actuaries/source.md +137 -0
  431. package/packs/bea/assets/download_bea.ps1 +182 -0
  432. package/packs/bea/assets/download_bea.sh +144 -0
  433. package/packs/bea/docs/bea_api.md +461 -0
  434. package/packs/bea/docs/datasets.md +157 -0
  435. package/packs/bea/docs/report_patterns.md +112 -0
  436. package/packs/bea/policy.json +170 -0
  437. package/packs/bea/source.md +152 -0
  438. package/packs/bls/assets/download_bls.ps1 +152 -0
  439. package/packs/bls/assets/download_bls.sh +167 -0
  440. package/packs/bls/docs/analysis_patterns.md +108 -0
  441. package/packs/bls/docs/bls_api.md +73 -0
  442. package/packs/bls/docs/series_ids.md +149 -0
  443. package/packs/bls/policy.json +238 -0
  444. package/packs/bls/series_ids.json +1375 -0
  445. package/packs/bls/source.md +156 -0
  446. package/packs/bts/endpoints.json +347 -0
  447. package/packs/bts/policy.json +94 -0
  448. package/packs/bts/source.md +97 -0
  449. package/packs/cdc/endpoints.json +197 -0
  450. package/packs/cdc/policy.json +83 -0
  451. package/packs/cdc/source.md +74 -0
  452. package/packs/census/endpoints.json +99 -0
  453. package/packs/census/policy.json +238 -0
  454. package/packs/census/source.md +304 -0
  455. package/packs/census-geocoder/endpoints.json +70 -0
  456. package/packs/census-geocoder/policy.json +80 -0
  457. package/packs/census-geocoder/source.md +53 -0
  458. package/packs/cfpb/endpoints.json +327 -0
  459. package/packs/cfpb/policy.json +119 -0
  460. package/packs/cfpb/source.md +96 -0
  461. package/packs/clinical-trials/endpoints.json +167 -0
  462. package/packs/clinical-trials/policy.json +84 -0
  463. package/packs/clinical-trials/source.md +82 -0
  464. package/packs/cms/endpoints.json +69 -0
  465. package/packs/cms/policy.json +96 -0
  466. package/packs/cms/source.md +112 -0
  467. package/packs/college-scorecard/endpoints.json +253 -0
  468. package/packs/college-scorecard/policy.json +106 -0
  469. package/packs/college-scorecard/source.md +92 -0
  470. package/packs/congress/endpoints.json +1701 -0
  471. package/packs/congress/policy.json +117 -0
  472. package/packs/congress/source.md +190 -0
  473. package/packs/dhs-ohss/endpoints.json +66 -0
  474. package/packs/dhs-ohss/policy.json +107 -0
  475. package/packs/dhs-ohss/source.md +120 -0
  476. package/packs/doj-news/endpoints.json +59 -0
  477. package/packs/doj-news/policy.json +96 -0
  478. package/packs/doj-news/source.md +99 -0
  479. package/packs/dol/endpoints.json +818 -0
  480. package/packs/dol/policy.json +135 -0
  481. package/packs/dol/source.md +181 -0
  482. package/packs/eia/endpoints.json +718 -0
  483. package/packs/eia/policy.json +112 -0
  484. package/packs/eia/source.md +112 -0
  485. package/packs/epa/endpoints.json +292 -0
  486. package/packs/epa/policy.json +99 -0
  487. package/packs/epa/source.md +99 -0
  488. package/packs/epa-aqs/endpoints.json +174 -0
  489. package/packs/epa-aqs/policy.json +96 -0
  490. package/packs/epa-aqs/source.md +97 -0
  491. package/packs/fbi/endpoints.json +1842 -0
  492. package/packs/fbi/policy.json +118 -0
  493. package/packs/fbi/source.md +108 -0
  494. package/packs/fda/endpoints.json +238 -0
  495. package/packs/fda/policy.json +121 -0
  496. package/packs/fda/source.md +117 -0
  497. package/packs/fdic/endpoints.json +417 -0
  498. package/packs/fdic/policy.json +83 -0
  499. package/packs/fdic/source.md +92 -0
  500. package/packs/fec/endpoints.json +708 -0
  501. package/packs/fec/policy.json +93 -0
  502. package/packs/fec/source.md +90 -0
  503. package/packs/fed-board/endpoints.json +89 -0
  504. package/packs/fed-board/policy.json +136 -0
  505. package/packs/fed-board/source.md +118 -0
  506. package/packs/federal-register/endpoints.json +143 -0
  507. package/packs/federal-register/policy.json +86 -0
  508. package/packs/federal-register/source.md +98 -0
  509. package/packs/fema/endpoints.json +646 -0
  510. package/packs/fema/policy.json +103 -0
  511. package/packs/fema/source.md +103 -0
  512. package/packs/fhfa/endpoints.json +105 -0
  513. package/packs/fhfa/policy.json +109 -0
  514. package/packs/fhfa/source.md +108 -0
  515. package/packs/fred/assets/download_fred.ps1 +122 -0
  516. package/packs/fred/assets/download_fred.sh +123 -0
  517. package/packs/fred/docs/analysis_patterns.md +124 -0
  518. package/packs/fred/docs/fred_api.md +94 -0
  519. package/packs/fred/docs/series.md +188 -0
  520. package/packs/fred/policy.json +253 -0
  521. package/packs/fred/source.md +142 -0
  522. package/packs/fsa/endpoints.json +150 -0
  523. package/packs/fsa/policy.json +99 -0
  524. package/packs/fsa/source.md +102 -0
  525. package/packs/govinfo/endpoints.json +520 -0
  526. package/packs/govinfo/policy.json +116 -0
  527. package/packs/govinfo/source.md +241 -0
  528. package/packs/gsa-calc/endpoints.json +93 -0
  529. package/packs/gsa-calc/policy.json +97 -0
  530. package/packs/gsa-calc/source.md +108 -0
  531. package/packs/hud/endpoints.json +147 -0
  532. package/packs/hud/policy.json +84 -0
  533. package/packs/hud/source.md +97 -0
  534. package/packs/ipums/assets/download_ipums.ps1 +215 -0
  535. package/packs/ipums/assets/download_ipums.sh +192 -0
  536. package/packs/ipums/docs/ipums_api.md +67 -0
  537. package/packs/ipums/docs/report_patterns.md +117 -0
  538. package/packs/ipums/docs/variables.md +284 -0
  539. package/packs/ipums/lib/fetch_text.json +112 -0
  540. package/packs/ipums/policy.json +215 -0
  541. package/packs/ipums/source.md +169 -0
  542. package/packs/irs-soi/endpoints.json +114 -0
  543. package/packs/irs-soi/policy.json +107 -0
  544. package/packs/irs-soi/source.md +124 -0
  545. package/packs/medicaid/endpoints.json +412 -0
  546. package/packs/medicaid/policy.json +108 -0
  547. package/packs/medicaid/source.md +136 -0
  548. package/packs/naep/endpoints.json +217 -0
  549. package/packs/naep/policy.json +100 -0
  550. package/packs/naep/source.md +120 -0
  551. package/packs/nhtsa/endpoints.json +132 -0
  552. package/packs/nhtsa/policy.json +93 -0
  553. package/packs/nhtsa/source.md +109 -0
  554. package/packs/nih/endpoints.json +453 -0
  555. package/packs/nih/policy.json +86 -0
  556. package/packs/nih/source.md +108 -0
  557. package/packs/noaa/endpoints.json +432 -0
  558. package/packs/noaa/policy.json +95 -0
  559. package/packs/noaa/source.md +88 -0
  560. package/packs/nrel/endpoints.json +328 -0
  561. package/packs/nrel/policy.json +96 -0
  562. package/packs/nrel/source.md +91 -0
  563. package/packs/nws/endpoints.json +272 -0
  564. package/packs/nws/policy.json +82 -0
  565. package/packs/nws/source.md +81 -0
  566. package/packs/nyfed/endpoints.json +80 -0
  567. package/packs/nyfed/policy.json +92 -0
  568. package/packs/nyfed/source.md +89 -0
  569. package/packs/omb/endpoints.json +460 -0
  570. package/packs/omb/policy.json +106 -0
  571. package/packs/omb/source.md +130 -0
  572. package/packs/open-payments/endpoints.json +298 -0
  573. package/packs/open-payments/policy.json +101 -0
  574. package/packs/open-payments/source.md +99 -0
  575. package/packs/opm/assets/download_opm.ps1 +125 -0
  576. package/packs/opm/assets/download_opm.sh +85 -0
  577. package/packs/opm/docs/fields.md +184 -0
  578. package/packs/opm/docs/opm_api.md +71 -0
  579. package/packs/opm/docs/report_patterns.md +83 -0
  580. package/packs/opm/lib/data_dictionary.csv +73 -0
  581. package/packs/opm/policy.json +142 -0
  582. package/packs/opm/source.md +137 -0
  583. package/packs/pbgc/endpoints.json +483 -0
  584. package/packs/pbgc/policy.json +94 -0
  585. package/packs/pbgc/source.md +95 -0
  586. package/packs/philfed/endpoints.json +85 -0
  587. package/packs/philfed/policy.json +94 -0
  588. package/packs/philfed/source.md +125 -0
  589. package/packs/port.json +287 -0
  590. package/packs/regulations/endpoints.json +288 -0
  591. package/packs/regulations/policy.json +86 -0
  592. package/packs/regulations/source.md +84 -0
  593. package/packs/sec/endpoints.json +237 -0
  594. package/packs/sec/policy.json +86 -0
  595. package/packs/sec/source.md +101 -0
  596. package/packs/senate-lobbying/endpoints.json +554 -0
  597. package/packs/senate-lobbying/policy.json +99 -0
  598. package/packs/senate-lobbying/source.md +100 -0
  599. package/packs/snapqc/endpoints.json +1426 -0
  600. package/packs/snapqc/policy.json +92 -0
  601. package/packs/snapqc/source.md +96 -0
  602. package/packs/treasury/catalog.json +2267 -0
  603. package/packs/treasury/docs/api_guide.md +245 -0
  604. package/packs/treasury/docs/datasets.md +56 -0
  605. package/packs/treasury/docs/report_patterns.md +126 -0
  606. package/packs/treasury/policy.json +159 -0
  607. package/packs/treasury/source.md +165 -0
  608. package/packs/treasury-yields/endpoints.json +226 -0
  609. package/packs/treasury-yields/policy.json +86 -0
  610. package/packs/treasury-yields/source.md +80 -0
  611. package/packs/usaspending/endpoints.json +1316 -0
  612. package/packs/usaspending/policy.json +109 -0
  613. package/packs/usaspending/source.md +103 -0
  614. package/packs/usda-fna/endpoints.json +427 -0
  615. package/packs/usda-fna/policy.json +133 -0
  616. package/packs/usda-fna/source.md +217 -0
  617. package/packs/usda-fooddata/endpoints.json +181 -0
  618. package/packs/usda-fooddata/policy.json +94 -0
  619. package/packs/usda-fooddata/source.md +105 -0
  620. package/packs/usda-nass/endpoints.json +324 -0
  621. package/packs/usda-nass/policy.json +95 -0
  622. package/packs/usda-nass/source.md +85 -0
  623. package/packs/usda-outlook/endpoints.json +86 -0
  624. package/packs/usda-outlook/policy.json +109 -0
  625. package/packs/usda-outlook/source.md +144 -0
  626. package/packs/usgs/endpoints.json +313 -0
  627. package/packs/usgs/policy.json +95 -0
  628. package/packs/usgs/source.md +96 -0
  629. package/packs/usitc/endpoints.json +74 -0
  630. package/packs/usitc/policy.json +111 -0
  631. package/packs/usitc/source.md +90 -0
  632. package/packs/uspto/endpoints.json +141 -0
  633. package/packs/uspto/policy.json +95 -0
  634. package/packs/uspto/source.md +101 -0
  635. package/packs/va/endpoints.json +414 -0
  636. package/packs/va/policy.json +101 -0
  637. package/packs/va/source.md +129 -0
  638. package/packs/world-bank/endpoints.json +122 -0
  639. package/packs/world-bank/policy.json +101 -0
  640. package/packs/world-bank/source.md +93 -0
@@ -0,0 +1,355 @@
1
+ /**
2
+ * The registry of every number cited in an answer or report: a port of scripts/analysis/facts.py.
3
+ * Written to <workspace>/facts.json (same schema); the report builder fills {{name}} / {{name|fmt}} placeholders
4
+ * from it and the audit traces each value to its cell.
5
+ *
6
+ * A table fact is read from the saved CSV (`table`: exhibits/<x>.csv, data/<id>/<t>.csv or published/<x>.csv): the
7
+ * registered value is the cell a reader finds there. A derived fact has a derivation; written as an expression
8
+ * over fact names ("a / b - 1"), it is computed here (and a given value must agree), so a rerun recomputes it;
9
+ * a prose derivation needs the value. NaN values and empty cells (gaps in the data) are refused.
10
+ *
11
+ * Each record holds label, fmt, value, source (the CSV path), where, column, derivation, se, the flags
12
+ * preliminary / vintage / projection (the audit requires the prose to state them) and headline (needs a
13
+ * benchmark.csv row), plus pack (the source pack(s): the pack argument; else the call's default; else the folder
14
+ * when the CSV lies under data/<id>/; else the exhibit's lineage; else the only data/<id>/ folder) and text (the
15
+ * value as the prose prints it, mode "answer").
16
+ */
17
+ import fs from "node:fs";
18
+ import path from "node:path";
19
+ import { pyStr } from "../core/py.js";
20
+ import { readCsv } from "../core/tabular.js";
21
+ import { inspect, parse, evaluate, ExprError } from "./expr.js";
22
+ import { lineagePacks, packList, PACK_ID, readLineage, relPath, ComputeError } from "./compute.js";
23
+ import { formatValue } from "./factfmt.js";
24
+ import { fixed, pyFloat, pyRepr } from "./pyfmt.js";
25
+ import { headerOf, newRow } from "./table.js";
26
+ export class FactError extends Error {
27
+ }
28
+ export const FACTS_FILE = "facts.json";
29
+ const NAME_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
30
+ export function readFacts(workspace) {
31
+ try {
32
+ const v = JSON.parse(fs.readFileSync(path.join(workspace, FACTS_FILE), "utf8"));
33
+ return v && typeof v === "object" && !Array.isArray(v) ? v : {};
34
+ }
35
+ catch {
36
+ return {};
37
+ }
38
+ }
39
+ const FLOAT = /^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/;
40
+ /** The one cell of `column` in the row matching `where`, as [number, text]; refuses with facts.py's messages. */
41
+ export function readCell(csvText, source, where, column, name) {
42
+ const cols = headerOf(csvText);
43
+ const base = path.basename(source);
44
+ const missing = [column, ...Object.keys(where)].filter((c) => !cols.includes(c));
45
+ if (missing.length)
46
+ throw new FactError(`fact ${name}: ${pyRepr(missing)} not in the table; its columns are ${pyRepr(cols)}`);
47
+ const rows = readCsv(csvText).filter((r) => Object.entries(where).every(([k, v]) => pyStr(r[k]) === pyStr(v)));
48
+ if (rows.length !== 1) {
49
+ throw new FactError(`fact ${name}: where=${pyRepr(where)} matches ${rows.length} rows of ${base} (need exactly 1; the CSV's ` +
50
+ "own text must equal each value, e.g. 2026 vs 2026.0)");
51
+ }
52
+ const text = String(rows[0][column] ?? "").trim();
53
+ if (!text || /^[+-]?nan$/i.test(text))
54
+ throw new FactError(`fact ${name}: the cell is empty (a gap or a flagged value); do not cite it`);
55
+ if (!FLOAT.test(text))
56
+ throw new FactError(`fact ${name}: ${column} = ${pyRepr(text)} in ${base} is not a number`);
57
+ return [Number(text), text];
58
+ }
59
+ /** Is a value stored exactly one digit past what its format prints, ending in 5 ("21.55" printed as "21.6")? */
60
+ export function roundingTie(cell, text, fmt) {
61
+ try {
62
+ const shown = /-?\d[\d,]*(?:\.(\d+))?/.exec(formatValue(cell, fmt));
63
+ if (!shown || !cell || !text.includes("."))
64
+ return false;
65
+ const disp = Number(shown[0].replace(/,/g, ""));
66
+ const k = disp ? Math.round(Math.log10(Math.abs(disp / cell))) : 0;
67
+ return text.split(".")[1].trimEnd().length - k === (shown[1] || "").length + 1 && text.trimEnd().endsWith("5");
68
+ }
69
+ catch {
70
+ return false;
71
+ }
72
+ }
73
+ function tieNote(name, cell, text, fmt) {
74
+ if (!roundingTie(cell, text, fmt))
75
+ return null;
76
+ return `note: fact ${name}: ${text} in the CSV sits on a rounding tie for its format: store full precision (no round() ` +
77
+ 'before saving), or, when the value is exact at that precision (published so), list it in decisions.json "rounding_ties_ok" ' +
78
+ "with why; the answer audit fails otherwise";
79
+ }
80
+ export function dataPacks(workspace) {
81
+ const d = path.join(workspace, "data");
82
+ if (!fs.existsSync(d))
83
+ return [];
84
+ return fs.readdirSync(d).filter((x) => fs.statSync(path.join(d, x)).isDirectory() && PACK_ID.test(x)).sort();
85
+ }
86
+ function packsOf(x) {
87
+ try {
88
+ return packList(x);
89
+ }
90
+ catch (e) {
91
+ if (e instanceof ComputeError)
92
+ throw new FactError(e.message);
93
+ throw e;
94
+ }
95
+ }
96
+ function inferPack(workspace, name, explicit, dflt, source) {
97
+ const given = explicit !== undefined && explicit !== null;
98
+ if (given && explicit !== "" && !packsOf(explicit).length)
99
+ return []; // pack: []: no pack, not the default
100
+ const rel = relPath(source || "");
101
+ if (!(given && packsOf(explicit).length) && rel.startsWith("published/"))
102
+ return [];
103
+ const got = given ? packsOf(explicit) : [];
104
+ if (got.length)
105
+ return got;
106
+ if (dflt.length)
107
+ return dflt;
108
+ if (source) {
109
+ const m = /^data\/([^/]+)\//.exec(rel);
110
+ if (m)
111
+ return [m[1]];
112
+ const lin = lineagePacks(readLineage(workspace)[rel]);
113
+ if (lin.length)
114
+ return packsOf(lin);
115
+ }
116
+ const present = dataPacks(workspace);
117
+ if (present.length === 1)
118
+ return present;
119
+ if (!present.length)
120
+ return [];
121
+ throw new FactError(`fact ${name}: the workspace holds data from ${pyRepr(present)}; say which source pack the number comes ` +
122
+ "from: pack: '<id>' (or a list), the call's pack default, or save the exhibit with compute's packs");
123
+ }
124
+ /** Units slips facts.py refuses (a value already scaled, a share given to pct, points given to pp ...). */
125
+ function unitChecks(name, v, fmt) {
126
+ const parts = (fmt || "").split(":");
127
+ const kind = parts.find((k) => ["usd_m", "usd_b", "usd_t", "dollars_m", "dollars_b", "dollars_t"].includes(k)) || "";
128
+ const scaled = ["from_thousands", "from_millions", "from_billions"].some((p) => (fmt || "").includes(p));
129
+ if (typeof v !== "number")
130
+ return;
131
+ if (kind && !scaled && v !== 0 && Math.abs(v) < 1e6) {
132
+ const word = { m: "million", b: "billion", t: "trillion" }[kind.slice(-1)];
133
+ throw new FactError(`fact ${name}: ${fmt} expects a value in dollars, but ${pyFloat(v)} looks already scaled (it would ` +
134
+ `print as $0.0 ${word}). Use usd_${kind.slice(-1)}_scaled for a value already in ${word}s of dollars, or a from_millions: / ` +
135
+ "from_billions: prefix.");
136
+ }
137
+ if (parts.includes("pct") && Math.abs(v) > 5) {
138
+ throw new FactError(`fact ${name}: fmt pct takes a share (0.274 = 27.4 percent); ${pyFloat(v)} would print as ` +
139
+ `${fixed(v * 100, 0)} percent: a value already in percent takes pct100`);
140
+ }
141
+ if (parts[0] === "pp" && Math.abs(v) > 1) {
142
+ throw new FactError(`fact ${name}: fmt pp takes a difference of shares (0.026 = 2.6 points); ${pyFloat(v)} is already ` +
143
+ "in points: use pp100");
144
+ }
145
+ if (fmt && v !== 0) {
146
+ let shown = "";
147
+ try {
148
+ shown = formatValue(v, fmt);
149
+ }
150
+ catch {
151
+ shown = "";
152
+ }
153
+ const digits = shown.split(" ")[0].replace(/[^\d]/g, "");
154
+ if ((shown.startsWith("$") || shown.startsWith("-$")) && digits && /^0+$/.test(digits) && !parts[parts.length - 1].includes("-")) {
155
+ throw new FactError(`fact ${name}: ${pyFloat(v)} prints as ${pyRepr(shown)} with fmt ${pyRepr(fmt)}: the value or the format ` +
156
+ "is in the wrong units (millions given to usd_b?); a real amount that small needs more digits (usd_b:2) or a smaller scale (usd_m)");
157
+ }
158
+ if (/[\d,]{8,}(\.\d+)? (?:million|billion|trillion)/.test(shown)) {
159
+ throw new FactError(`fact ${name}: ${pyFloat(v)} prints as ${pyRepr(shown)} with fmt ${pyRepr(fmt)}: a value already in dollars ` +
160
+ "given to a *_scaled or from_* format? Check the units.");
161
+ }
162
+ }
163
+ }
164
+ /** A derivation written as an expression over fact names, parsed; null when it is prose. */
165
+ export function derivationExpr(derivation, known) {
166
+ let n;
167
+ try {
168
+ n = parse(derivation);
169
+ }
170
+ catch {
171
+ return null;
172
+ }
173
+ let info;
174
+ try {
175
+ info = inspect(n);
176
+ }
177
+ catch {
178
+ return null;
179
+ }
180
+ if (info.agg || info.win || !info.cols.length || !info.cols.every(known))
181
+ return null;
182
+ return n;
183
+ }
184
+ /** The value of a derivation expression from fact values (numbers only). */
185
+ export function evalDerivation(n, values, name) {
186
+ const row = newRow();
187
+ for (const c of inspect(n).cols) {
188
+ const v = values(c);
189
+ if (typeof v !== "number")
190
+ throw new FactError(`fact ${name}: its derivation uses ${c}, whose value ${pyRepr(v)} is not a number`);
191
+ row[c] = v;
192
+ }
193
+ try {
194
+ return evaluate(n, { row, i: 0, group: [row], cache: new Map() });
195
+ }
196
+ catch (e) {
197
+ if (e instanceof ExprError)
198
+ throw new FactError(`fact ${name}: derivation: ${e.message}`);
199
+ throw e;
200
+ }
201
+ }
202
+ function common(rec, f, derivation) {
203
+ if (f.se !== undefined && f.se !== null)
204
+ rec.se = Number(f.se);
205
+ if (derivation)
206
+ rec.derivation = derivation;
207
+ if (f.preliminary)
208
+ rec.preliminary = true;
209
+ if (f.vintage && f.vintage !== "latest")
210
+ rec.vintage = String(f.vintage);
211
+ if (f.projection)
212
+ rec.projection = true;
213
+ if (f.headline)
214
+ rec.headline = true;
215
+ }
216
+ function checkName(name, fmt) {
217
+ if (!NAME_RE.test(String(name)))
218
+ throw new FactError(`fact name ${pyRepr(name)}: letters, digits and _ only`);
219
+ try {
220
+ formatValue(1.0, fmt); // an unknown format fails here, not in the report build
221
+ }
222
+ catch (e) {
223
+ throw new FactError(e.message);
224
+ }
225
+ }
226
+ /** Register a table fact: the value is the CSV cell. Returns the record and any note. */
227
+ export function tableFact(workspace, f, dflt = []) {
228
+ const fmt = f.fmt ?? "raw";
229
+ checkName(f.name, fmt);
230
+ if (!f.table || !f.where || typeof f.where !== "object" || !f.column) {
231
+ throw new FactError(`fact ${f.name}: table facts need table (the CSV path), where and column`);
232
+ }
233
+ const source = relPath(f.table);
234
+ if (!/^(data\/[^/]+\/|exhibits\/|published\/)[^/]+\.csv$/.test(source) || source.split("/").includes("..")) {
235
+ throw new FactError(`fact ${f.name}: table ${pyRepr(f.table)}: a workspace CSV (exhibits/<x>.csv, data/<id>/<t>.csv or published/<x>.csv)`);
236
+ }
237
+ const abs = path.join(workspace, source);
238
+ if (!fs.existsSync(abs))
239
+ throw new FactError(`fact ${f.name}: save ${source} before registering facts from it`);
240
+ const [cell, text] = readCell(fs.readFileSync(abs, "utf8"), source, f.where, f.column, f.name);
241
+ const rec = { label: f.label ?? "", fmt, value: cell, source,
242
+ where: Object.fromEntries(Object.entries(f.where).map(([k, v]) => [k, pyStr(v)])), column: f.column };
243
+ unitChecks(f.name, cell, fmt);
244
+ common(rec, f);
245
+ const packs = inferPack(workspace, f.name, f.pack, dflt, source);
246
+ if (packs.length)
247
+ rec.pack = packs.length === 1 ? packs[0] : packs;
248
+ return { rec, note: tieNote(f.name, cell, text, fmt) };
249
+ }
250
+ /** Register a derived fact; `known` gives the values of facts registered so far. */
251
+ export function derivedFact(workspace, f, known, dflt = []) {
252
+ const fmt = f.fmt ?? "raw";
253
+ checkName(f.name, fmt);
254
+ const derivation = String(f.derivation ?? "");
255
+ const node = derivation ? derivationExpr(derivation, (n) => Object.hasOwn(known, n) && n !== f.name) : null;
256
+ let value = f.value;
257
+ if (node) {
258
+ const got = evalDerivation(node, (n) => known[n]?.value, f.name);
259
+ if (got === null)
260
+ throw new FactError(`fact ${f.name}: its derivation ${derivation} gives no value (a gap or a division by zero); do not cite it`);
261
+ if (value !== undefined && value !== null) {
262
+ const a = Number(value), b = Number(got);
263
+ if (typeof got !== "number" || !(Math.abs(a - b) <= 1e-9 * Math.max(1, Math.abs(b)))) {
264
+ throw new FactError(`fact ${f.name}: value ${pyRepr(value)} differs from its derivation ${derivation} = ${pyRepr(got)}; ` +
265
+ "leave value out and the derivation is computed");
266
+ }
267
+ }
268
+ value = got;
269
+ }
270
+ if (value === undefined || value === null || !derivation) {
271
+ throw new FactError(`fact ${f.name}: give table/where/column, or value with derivation (a derivation written as an ` +
272
+ 'expression over fact names, e.g. "a / b - 1", is computed for you)');
273
+ }
274
+ if (typeof value === "number" && Number.isNaN(value))
275
+ throw new FactError(`fact ${f.name}: value is NaN (a gap in the data); do not cite it`);
276
+ const rec = { label: f.label ?? "", fmt, value, derivation };
277
+ unitChecks(f.name, value, fmt);
278
+ common(rec, f, derivation);
279
+ const packs = inferPack(workspace, f.name, f.pack, dflt);
280
+ if (packs.length)
281
+ rec.pack = packs.length === 1 ? packs[0] : packs;
282
+ return rec;
283
+ }
284
+ /** facts.json with "text" (the value as the prose prints it) on every numeric fact. */
285
+ export function saveFacts(workspace, items) {
286
+ for (const rec of Object.values(items)) {
287
+ if (typeof rec?.value === "number") {
288
+ try {
289
+ rec.text = formatValue(rec.value, rec.fmt ?? "raw", "answer");
290
+ }
291
+ catch {
292
+ /* an unformattable value keeps no text */
293
+ }
294
+ }
295
+ }
296
+ const p = path.join(workspace, FACTS_FILE);
297
+ fs.writeFileSync(p, JSON.stringify(items, null, 2));
298
+ return p;
299
+ }
300
+ /** Add (or replace) facts and save facts.json. Nothing is written when any fact is refused. */
301
+ export function registerFacts(workspace, a) {
302
+ const items = readFacts(workspace);
303
+ for (const n of a.remove || [])
304
+ delete items[n];
305
+ const dflt = packsOf(a.pack);
306
+ const added = {};
307
+ const notes = [];
308
+ const errors = [];
309
+ for (const f of a.add || []) {
310
+ try {
311
+ const { rec, note } = tableFact(workspace, f, dflt);
312
+ items[f.name] = added[f.name] = rec;
313
+ if (note)
314
+ notes.push(note);
315
+ }
316
+ catch (e) {
317
+ if (e instanceof FactError)
318
+ errors.push(e.message);
319
+ else
320
+ throw e;
321
+ }
322
+ }
323
+ for (const f of a.derived || []) {
324
+ try {
325
+ const rec = derivedFact(workspace, f, items, dflt);
326
+ items[f.name] = added[f.name] = rec;
327
+ }
328
+ catch (e) {
329
+ if (e instanceof FactError)
330
+ errors.push(e.message);
331
+ else
332
+ throw e;
333
+ }
334
+ }
335
+ if (errors.length) {
336
+ const err = new FactError(errors[0]);
337
+ err.problems = errors;
338
+ throw err;
339
+ }
340
+ for (const rec of Object.values(added)) {
341
+ if (typeof rec.value !== "number")
342
+ continue;
343
+ try {
344
+ formatValue(rec.value, rec.fmt, "answer", (m) => notes.push(`fact ${Object.keys(added).find((k) => added[k] === rec)}: ${m}`));
345
+ }
346
+ catch {
347
+ /* checked when registered */
348
+ }
349
+ }
350
+ const file = saveFacts(workspace, items);
351
+ for (const k of Object.keys(added))
352
+ added[k] = items[k];
353
+ return { file, added, notes, names: Object.keys(items) };
354
+ }
355
+ //# sourceMappingURL=facts.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"facts.js","sourceRoot":"","sources":["../../src/analysis/facts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAa,MAAM,WAAW,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACnG,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,OAAO,SAAU,SAAQ,KAAK;CAAG;AAEvC,MAAM,CAAC,MAAM,UAAU,GAAG,YAAY,CAAC;AACvC,MAAM,OAAO,GAAG,0BAA0B,CAAC;AAS3C,MAAM,UAAU,SAAS,CAAC,SAAiB;IACzC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QAChF,OAAO,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,KAAK,GAAG,yCAAyC,CAAC;AAExD,iHAAiH;AACjH,MAAM,UAAU,QAAQ,CAAC,OAAe,EAAE,MAAc,EAAE,KAA8B,EAAE,MAAc,EAAE,IAAY;IACpH,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IACjF,IAAI,OAAO,CAAC,MAAM;QAAE,MAAM,IAAI,SAAS,CAAC,QAAQ,IAAI,KAAK,MAAM,CAAC,OAAO,CAAC,sCAAsC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC9H,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/G,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,SAAS,CAAC,QAAQ,IAAI,WAAW,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,MAAM,YAAY,IAAI,8BAA8B;YAC3H,sDAAsD,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClD,IAAI,CAAC,IAAI,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,QAAQ,IAAI,gEAAgE,CAAC,CAAC;IACzI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,QAAQ,IAAI,KAAK,MAAM,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,kBAAkB,CAAC,CAAC;IACnH,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,gHAAgH;AAChH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,IAAY,EAAE,GAAW;IACjE,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,wBAAwB,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QACpE,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QACzD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACjH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,OAAO,CAAC,IAAY,EAAE,IAAY,EAAE,IAAY,EAAE,GAAW;IACpE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/C,OAAO,cAAc,IAAI,KAAK,IAAI,sFAAsF;QACtH,6HAA6H;QAC7H,4CAA4C,CAAC;AACjD,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,SAAiB;IACzC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,OAAO,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAC/G,CAAC;AAED,SAAS,OAAO,CAAC,CAAU;IACzB,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,YAAY;YAAE,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC9D,MAAM,CAAC,CAAC;IACV,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,SAAiB,EAAE,IAAY,EAAE,QAAiB,EAAE,IAAc,EAAE,MAAe;IACpG,MAAM,KAAK,GAAG,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,IAAI,CAAC;IAC1D,IAAI,KAAK,IAAI,QAAQ,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC,CAAU,qCAAqC;IACpH,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IAClC,IAAI,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,EAAE,CAAC;IACpF,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,IAAI,GAAG,CAAC,MAAM;QAAE,OAAO,GAAG,CAAC;IAC3B,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC7B,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,GAAG,GAAG,YAAY,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACtD,IAAI,GAAG,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;IACtC,CAAC;IACD,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IACrC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IACzC,IAAI,CAAC,OAAO,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC/B,MAAM,IAAI,SAAS,CAAC,QAAQ,IAAI,mCAAmC,MAAM,CAAC,OAAO,CAAC,2CAA2C;QAC3H,mGAAmG,CAAC,CAAC;AACzG,CAAC;AAED,2GAA2G;AAC3G,SAAS,UAAU,CAAC,IAAY,EAAE,CAAU,EAAE,GAAW;IACvD,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACrH,MAAM,MAAM,GAAG,CAAC,gBAAgB,EAAE,eAAe,EAAE,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IACzG,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO;IAClC,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC;QACpD,MAAM,IAAI,GAAG,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAoB,CAAC,CAAC;QAC9F,MAAM,IAAI,SAAS,CAAC,QAAQ,IAAI,KAAK,GAAG,oCAAoC,OAAO,CAAC,CAAC,CAAC,kCAAkC;YACtH,iBAAiB,IAAI,cAAc,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,kCAAkC,IAAI,sCAAsC;YAC7H,wBAAwB,CAAC,CAAC;IAC9B,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,SAAS,CAAC,QAAQ,IAAI,mDAAmD,OAAO,CAAC,CAAC,CAAC,kBAAkB;YAC7G,GAAG,KAAK,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,mDAAmD,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,SAAS,CAAC,QAAQ,IAAI,+DAA+D,OAAO,CAAC,CAAC,CAAC,cAAc;YACrH,sBAAsB,CAAC,CAAC;IAC5B,CAAC;IACD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACnB,IAAI,KAAK,GAAG,EAAE,CAAC;QACf,IAAI,CAAC;YACH,KAAK,GAAG,WAAW,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;QACD,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACjI,MAAM,IAAI,SAAS,CAAC,QAAQ,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,cAAc,MAAM,CAAC,KAAK,CAAC,aAAa,MAAM,CAAC,GAAG,CAAC,4BAA4B;gBAC5H,mIAAmI,CAAC,CAAC;QACzI,CAAC;QACD,IAAI,gDAAgD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,SAAS,CAAC,QAAQ,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,cAAc,MAAM,CAAC,KAAK,CAAC,aAAa,MAAM,CAAC,GAAG,CAAC,+BAA+B;gBAC/H,wDAAwD,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;AACH,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,cAAc,CAAC,UAAkB,EAAE,KAA6B;IAC9E,IAAI,CAAO,CAAC;IACZ,IAAI,CAAC;QACH,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,IAAI,CAAC;IACT,IAAI,CAAC;QACH,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtF,OAAO,CAAC,CAAC;AACX,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,cAAc,CAAC,CAAO,EAAE,MAAiC,EAAE,IAAY;IACrF,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;IACrB,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,MAAM,IAAI,SAAS,CAAC,QAAQ,IAAI,yBAAyB,CAAC,iBAAiB,MAAM,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC;QACnI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACb,CAAC;IACD,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;IACpE,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,SAAS;YAAE,MAAM,IAAI,SAAS,CAAC,QAAQ,IAAI,iBAAiB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1F,MAAM,CAAC,CAAC;IACV,CAAC;AACH,CAAC;AAED,SAAS,MAAM,CAAC,GAAwB,EAAE,CACE,EAAE,UAAmB;IAC/D,IAAI,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,EAAE,KAAK,IAAI;QAAE,GAAG,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC/D,IAAI,UAAU;QAAE,GAAG,CAAC,UAAU,GAAG,UAAU,CAAC;IAC5C,IAAI,CAAC,CAAC,WAAW;QAAE,GAAG,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1C,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,KAAK,QAAQ;QAAE,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACzE,IAAI,CAAC,CAAC,UAAU;QAAE,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC;IACxC,IAAI,CAAC,CAAC,QAAQ;QAAE,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC;AACtC,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,GAAW;IAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,aAAa,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC9G,IAAI,CAAC;QACH,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAuB,wDAAwD;IACvG,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QAChB,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,SAAS,CAAC,SAAiB,EAAE,CAAY,EAAE,OAAiB,EAAE;IAC5E,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,KAAK,CAAC;IAC3B,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvB,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QACrE,MAAM,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,2DAA2D,CAAC,CAAC;IACjG,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,CAAC,oDAAoD,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3G,MAAM,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,WAAW,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAC;IAC9I,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACzC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,UAAU,MAAM,mCAAmC,CAAC,CAAC;IAChH,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAC/F,MAAM,GAAG,GAAwB,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM;QAC/E,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;IACxG,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IAC9B,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACf,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACjE,IAAI,KAAK,CAAC,MAAM;QAAE,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACnE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;AACzD,CAAC;AAED,oFAAoF;AACpF,MAAM,UAAU,WAAW,CAAC,SAAiB,EAAE,CAAc,EAAE,KAA0B,EAAE,OAAiB,EAAE;IAC5G,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,KAAK,CAAC;IAC3B,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvB,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5G,IAAI,KAAK,GAAY,CAAC,CAAC,KAAK,CAAC;IAC7B,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QACjE,IAAI,GAAG,KAAK,IAAI;YAAE,MAAM,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,oBAAoB,UAAU,+DAA+D,CAAC,CAAC;QACnJ,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YAC1C,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YACzC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrF,MAAM,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,WAAW,MAAM,CAAC,KAAK,CAAC,gCAAgC,UAAU,MAAM,MAAM,CAAC,GAAG,CAAC,IAAI;oBACvH,gDAAgD,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QACD,KAAK,GAAG,GAAG,CAAC;IACd,CAAC;IACD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACzD,MAAM,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,kFAAkF;YAClH,oEAAoE,CAAC,CAAC;IAC1E,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,oDAAoD,CAAC,CAAC;IAC9I,MAAM,GAAG,GAAwB,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IAClF,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IAC/B,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC;IAC3B,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACzD,IAAI,KAAK,CAAC,MAAM;QAAE,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACnE,OAAO,GAAG,CAAC;AACb,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,SAAS,CAAC,SAAiB,EAAE,KAA0B;IACrE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACvC,IAAI,OAAO,GAAG,EAAE,KAAK,KAAK,QAAQ,EAAE,CAAC;YACnC,IAAI,CAAC;gBACH,GAAG,CAAC,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,IAAI,KAAK,EAAE,QAAQ,CAAC,CAAC;YAChE,CAAC;YAAC,MAAM,CAAC;gBACP,0CAA0C;YAC5C,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IAC3C,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACpD,OAAO,CAAC,CAAC;AACX,CAAC;AAID,+FAA+F;AAC/F,MAAM,UAAU,aAAa,CAAC,SAAiB,EAAE,CAAY;IAE3D,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,EAAE;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GAAwB,EAAE,CAAC;IACtC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;YACpD,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;YACpC,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,YAAY,SAAS;gBAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;;gBAC9C,MAAM,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,WAAW,CAAC,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YACnD,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;QACtC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,YAAY,SAAS;gBAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;;gBAC9C,MAAM,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,GAAW,CAAC,QAAQ,GAAG,MAAM,CAAC;QAC/B,MAAM,GAAG,CAAC;IACZ,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACvC,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ;YAAE,SAAS;QAC5C,IAAI,CAAC;YACH,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACjI,CAAC;QAAC,MAAM,CAAC;YACP,6BAA6B;QAC/B,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACxD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;AAC3D,CAAC"}
@@ -0,0 +1,58 @@
1
+ export declare const PREVIEW_ROWS = 4;
2
+ export declare const PREVIEW_COLS = 8;
3
+ export declare const CELL = 40;
4
+ export declare const MAX_TABLES = 6;
5
+ export declare const PROVENANCE: string[];
6
+ export declare const MAX_NOTES = 6;
7
+ export declare const CAVEAT: RegExp;
8
+ export type Preview = {
9
+ header: string[];
10
+ rows: string[][];
11
+ same: [string, string][];
12
+ hidden: number;
13
+ total: number;
14
+ columns: number;
15
+ };
16
+ /** The first n rows of a CSV, showing the columns that vary (a column with one value in the first `sample` rows is
17
+ * listed once under "same", so values are not crowded out by labels). */
18
+ export declare function preview(file: string, n?: number, sample?: number): Preview;
19
+ export type FileSummary = {
20
+ file: string;
21
+ rows?: number | null;
22
+ columns?: number;
23
+ preview?: Preview | null;
24
+ earlier?: string;
25
+ };
26
+ export type SourceSummary = {
27
+ source: string;
28
+ route: string;
29
+ retrieved: string;
30
+ files: FileSummary[];
31
+ notes: string[];
32
+ problems: string[];
33
+ readme: string | null;
34
+ microdata: boolean;
35
+ provenance: boolean;
36
+ provenance_file: string;
37
+ };
38
+ export declare function sourceSummary(out: string, d: string): SourceSummary;
39
+ export type PublishedSummary = {
40
+ source: "published";
41
+ route: string;
42
+ files: {
43
+ file: string;
44
+ rows: number;
45
+ source_url: string[];
46
+ accessed: string[];
47
+ preview: Preview;
48
+ }[];
49
+ };
50
+ export declare function publishedSummary(out: string): PublishedSummary | null;
51
+ export declare function summarize(workspace: string): {
52
+ status: "ok";
53
+ out: string;
54
+ sources: (SourceSummary | PublishedSummary)[];
55
+ unpackaged: string[];
56
+ text: string;
57
+ action: string;
58
+ };
@@ -0,0 +1,208 @@
1
+ /**
2
+ * The data hand-off: what each source delivered, read back from its provenance files (a port of
3
+ * scripts/core/handoff.py).
4
+ *
5
+ * For every <workspace>/data/<id>/ (request.json; Treasury's fetch_spec.json) and <workspace>/published/*.csv it
6
+ * lists the files with row counts, the route and retrieval date, the fetch notes and problems, and a short
7
+ * preview of each table, and writes the same as ready-to-send Markdown ("text"). Everything comes from the files
8
+ * written at fetch time, so the hand-off cannot drift from what was downloaded.
9
+ */
10
+ import fs from "node:fs";
11
+ import path from "node:path";
12
+ import { PACKS_DIR } from "../core/env.js";
13
+ import { localDay, readJson } from "../core/policy.js";
14
+ import { readCsv } from "../core/tabular.js";
15
+ import { fixed } from "./pyfmt.js";
16
+ import { readCsvGrid } from "./table.js";
17
+ export const PREVIEW_ROWS = 4;
18
+ export const PREVIEW_COLS = 8;
19
+ export const CELL = 40;
20
+ export const MAX_TABLES = 6;
21
+ export const PROVENANCE = ["request.json", "fetch_spec.json"];
22
+ export const MAX_NOTES = 6;
23
+ export const CAVEAT = new RegExp("missing|may be|upper bound|withheld|suppress|fewer|stopped at|partial|incomplete|not in|" +
24
+ "no rows|twice|restored|skipped|without --check|not fetched|not reported|dropped|lag", "i");
25
+ const comma = (n) => fixed(n, 0);
26
+ const posix = (p) => p.split(path.sep).join("/");
27
+ function cell(v) {
28
+ const s = String(v ?? "").split(/\s+/).filter(Boolean).join(" ").replace(/\|/g, "/");
29
+ return s.length <= CELL ? s : s.slice(0, CELL - 3) + "...";
30
+ }
31
+ /** The first n rows of a CSV, showing the columns that vary (a column with one value in the first `sample` rows is
32
+ * listed once under "same", so values are not crowded out by labels). */
33
+ export function preview(file, n = PREVIEW_ROWS, sample = 200) {
34
+ const grid = readCsvGrid(fs.readFileSync(file, "utf8"));
35
+ const head = grid[0] || [];
36
+ const rows = [];
37
+ let total = 0;
38
+ for (const row of grid.slice(1)) {
39
+ total++;
40
+ if (rows.length < sample)
41
+ rows.push([...row, ...Array(Math.max(0, head.length - row.length)).fill("")]);
42
+ }
43
+ const vary = head.map((_, i) => i).filter((i) => new Set(rows.map((r) => r[i])).size > 1);
44
+ const show = (vary.length ? vary : head.map((_, i) => i)).slice(0, PREVIEW_COLS);
45
+ const same = head.map((h, i) => [h, i])
46
+ .filter(([, i]) => !vary.includes(i) && rows.length && rows[0][i] !== "").map(([h, i]) => [h, rows[0][i]]);
47
+ return { header: show.map((i) => head[i]), rows: rows.slice(0, n).map((r) => show.map((i) => r[i])), same: same.slice(0, 6),
48
+ hidden: head.length - show.length - same.slice(0, 6).length, total, columns: head.length };
49
+ }
50
+ function tableMd(pv) {
51
+ if (!pv.header.length)
52
+ return ["(empty file)"];
53
+ const out = ["| " + pv.header.map(cell).join(" | ") + " |", "|" + "---|".repeat(pv.header.length)];
54
+ out.push(...pv.rows.map((r) => "| " + r.map(cell).join(" | ") + " |"));
55
+ if (pv.same.length)
56
+ out.push(`Same in the first ${comma(Math.min(pv.total, 200))} rows: ` + pv.same.map(([k, v]) => `${k} = ${cell(v)}`).join("; "));
57
+ if (pv.hidden > 0)
58
+ out.push(`(${pv.hidden} more columns: see the data dictionary)`);
59
+ return out;
60
+ }
61
+ function provenance(d) {
62
+ for (const name of PROVENANCE) {
63
+ const p = path.join(d, name);
64
+ if (fs.existsSync(p)) {
65
+ try {
66
+ return [JSON.parse(fs.readFileSync(p, "utf8")) || {}, name];
67
+ }
68
+ catch {
69
+ return [{}, name];
70
+ }
71
+ }
72
+ }
73
+ return [{}, ""];
74
+ }
75
+ function redistribution(source) {
76
+ try {
77
+ return readJson(path.join(PACKS_DIR, source), "policy.json")?.redistribution?.mode || "allowed";
78
+ }
79
+ catch {
80
+ return "allowed"; // a folder that is no pack: treat as shareable data
81
+ }
82
+ }
83
+ export function sourceSummary(out, d) {
84
+ const sid = path.basename(d);
85
+ const [rec, recFile] = provenance(d);
86
+ const tables = (Array.isArray(rec.tables) ? rec.tables : []).filter((t) => t && typeof t === "object" && t.file);
87
+ // an earlier fetch into the same folder whose tables are still there: they are part of what was delivered
88
+ const earlier = (Array.isArray(rec.earlier_fetches) ? rec.earlier_fetches : []).filter((e) => e && typeof e === "object");
89
+ const listed = new Set(tables.map((t) => t.file));
90
+ for (const e of earlier) {
91
+ for (const t of e.tables || []) {
92
+ if (t && typeof t === "object" && t.file && !listed.has(t.file) && fs.existsSync(path.join(d, t.file))) {
93
+ tables.push({ ...t, earlier: `route ${e.route || e.acquired || "unknown"}, retrieved ` + (e.retrieved ? localDay(e.retrieved) : "unknown") });
94
+ listed.add(t.file);
95
+ }
96
+ }
97
+ }
98
+ if (!tables.length) { // no list in the record: every CSV in the folder
99
+ for (const f of fs.readdirSync(d).filter((x) => x.toLowerCase().endsWith(".csv") && x !== "data_dictionary.csv").sort()) {
100
+ tables.push({ file: f });
101
+ }
102
+ }
103
+ const mode = redistribution(sid);
104
+ const micro = mode === "definition_only" || mode === "aggregates_only";
105
+ const files = [];
106
+ for (const t of tables) {
107
+ const p = path.join(d, t.file);
108
+ const tag = t.earlier ? { earlier: t.earlier } : {};
109
+ const rel = posix(path.relative(out, p));
110
+ if (!fs.existsSync(p) || !p.toLowerCase().endsWith(".csv")) {
111
+ files.push({ file: rel, rows: t.rows ?? null, ...tag });
112
+ continue;
113
+ }
114
+ const pv = preview(p);
115
+ files.push({ file: rel, rows: pv.total, columns: pv.columns, preview: micro ? null : pv, ...tag });
116
+ }
117
+ const retrieved = rec.retrieved || rec.downloaded || "";
118
+ const notes = (rec.notes || []).filter(Boolean).map(String);
119
+ const problems = (rec.problems || []).filter(Boolean).map(String);
120
+ for (const e of earlier)
121
+ for (const n of e.notes || [])
122
+ if (n)
123
+ notes.push(`(earlier fetch, table still in the folder) ${n}`);
124
+ for (const e of earlier)
125
+ for (const p of e.problems || [])
126
+ if (p)
127
+ problems.push(`(earlier fetch, table still in the folder) ${p}`);
128
+ return { source: sid, route: rec.route || rec.acquired || "unknown", retrieved: retrieved ? localDay(retrieved) : "unknown",
129
+ files, notes, problems, readme: fs.existsSync(path.join(d, "README.md")) ? posix(path.relative(out, path.join(d, "README.md"))) : null,
130
+ microdata: micro, provenance: Object.keys(rec).length > 0, provenance_file: recFile };
131
+ }
132
+ export function publishedSummary(out) {
133
+ const dir = path.join(out, "published");
134
+ if (!fs.existsSync(dir))
135
+ return null;
136
+ const paths = fs.readdirSync(dir).filter((f) => f.toLowerCase().endsWith(".csv")).sort().map((f) => path.join(dir, f));
137
+ if (!paths.length)
138
+ return null;
139
+ const files = paths.map((p) => {
140
+ const pv = preview(p);
141
+ const urls = new Set(), dates = new Set();
142
+ for (const r of readCsv(fs.readFileSync(p, "utf8"))) {
143
+ urls.add(String(r.source_url ?? ""));
144
+ dates.add(String(r.accessed ?? ""));
145
+ }
146
+ const sorted = (s) => [...s].filter(Boolean).sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
147
+ return { file: posix(path.relative(out, p)), rows: pv.total, source_url: sorted(urls), accessed: sorted(dates), preview: pv };
148
+ });
149
+ return { source: "published", route: "read from the agency's page", files };
150
+ }
151
+ /** Every caveat note and the first MAX_NOTES others; the rest are counted with where to read them. */
152
+ function notesMd(s) {
153
+ const shown = [], rest = [];
154
+ let others = 0;
155
+ for (const n of s.notes) {
156
+ const caveat = CAVEAT.test(n);
157
+ (caveat || others < MAX_NOTES ? shown : rest).push(n);
158
+ if (!caveat)
159
+ others++;
160
+ }
161
+ const md = shown.map((n) => `- Note: ${n}`);
162
+ if (rest.length)
163
+ md.push(`- (+${rest.length} more notes in data/${s.source}/${s.provenance_file || "request.json"})`);
164
+ return md;
165
+ }
166
+ export function summarize(workspace) {
167
+ const out = path.resolve(workspace);
168
+ const dataDir = path.join(out, "data");
169
+ const dirs = fs.existsSync(dataDir)
170
+ ? fs.readdirSync(dataDir).filter((x) => fs.statSync(path.join(dataDir, x)).isDirectory()).sort().map((x) => path.join(dataDir, x)) : [];
171
+ const sources = dirs.map((d) => sourceSummary(out, d));
172
+ const pub = publishedSummary(out);
173
+ const md = [];
174
+ for (const s of sources) {
175
+ md.push(`### ${s.source}`, `- Route: ${s.route}; retrieved ${s.retrieved}` +
176
+ (s.provenance ? "" : " (no request.json: not packaged by fetch; fetch it again or ingest it)"));
177
+ for (const f of s.files) {
178
+ md.push((typeof f.rows === "number" ? `- \`${f.file}\`: ${comma(f.rows)} rows` : `- \`${f.file}\``) +
179
+ (f.earlier ? ` (earlier fetch: ${f.earlier})` : ""));
180
+ }
181
+ md.push(...notesMd(s), ...s.problems.map((p) => `- **Problem**: ${p}`));
182
+ const pv = s.files.filter((f) => f.preview);
183
+ for (const f of pv.slice(0, MAX_TABLES)) {
184
+ md.push("", `\`${f.file}\` (first ${f.preview.rows.length} rows):`, "", ...tableMd(f.preview));
185
+ }
186
+ if (s.microdata)
187
+ md.push("- Microdata: no preview (the terms keep it in the analysis folder); see the data dictionary");
188
+ if (pv.length > MAX_TABLES)
189
+ md.push(`- (${pv.length - MAX_TABLES} more tables: listed above, no preview here)`);
190
+ md.push("");
191
+ }
192
+ if (pub) {
193
+ md.push("### Figures read from agency pages (`published/`)");
194
+ for (const f of pub.files) {
195
+ md.push(`- \`${f.file}\`: ${comma(f.rows)} rows; source ${f.source_url.join(", ") || "MISSING source_url"}; ` +
196
+ `accessed ${f.accessed.join(", ") || "MISSING accessed"}`);
197
+ }
198
+ md.push("");
199
+ }
200
+ const missing = sources.filter((s) => !s.provenance).map((s) => s.source);
201
+ return {
202
+ status: "ok", out, sources: [...sources, ...(pub ? [pub] : [])], unpackaged: missing, text: md.join("\n").trim(),
203
+ action: "Send `text` as the per-source part of the hand-off (it is read from the files fetched), then add each source's " +
204
+ "caveats from its README and any data that could not be fetched, with the route to get it." +
205
+ (missing.length ? ` Folders without provenance: ${missing.join(", ")}.` : ""),
206
+ };
207
+ }
208
+ //# sourceMappingURL=handoff.js.map