data-analyst-mcp 0.0.0-stage → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (640) hide show
  1. package/README.md +261 -2
  2. package/dist/adapters/_acf_tidy.d.ts +18 -0
  3. package/dist/adapters/_acf_tidy.js +432 -0
  4. package/dist/adapters/_acf_tidy.js.map +1 -0
  5. package/dist/adapters/_actuaries_read.d.ts +38 -0
  6. package/dist/adapters/_actuaries_read.js +728 -0
  7. package/dist/adapters/_actuaries_read.js.map +1 -0
  8. package/dist/adapters/_census_micro.d.ts +70 -0
  9. package/dist/adapters/_census_micro.js +622 -0
  10. package/dist/adapters/_census_micro.js.map +1 -0
  11. package/dist/adapters/_census_util.d.ts +40 -0
  12. package/dist/adapters/_census_util.js +275 -0
  13. package/dist/adapters/_census_util.js.map +1 -0
  14. package/dist/adapters/_dol_states.d.ts +6 -0
  15. package/dist/adapters/_dol_states.js +58 -0
  16. package/dist/adapters/_dol_states.js.map +1 -0
  17. package/dist/adapters/_govinfo_entities.d.ts +4 -0
  18. package/dist/adapters/_govinfo_entities.js +6 -0
  19. package/dist/adapters/_govinfo_entities.js.map +1 -0
  20. package/dist/adapters/_govinfo_json.d.ts +13 -0
  21. package/dist/adapters/_govinfo_json.js +107 -0
  22. package/dist/adapters/_govinfo_json.js.map +1 -0
  23. package/dist/adapters/_govinfo_py.d.ts +22 -0
  24. package/dist/adapters/_govinfo_py.js +272 -0
  25. package/dist/adapters/_govinfo_py.js.map +1 -0
  26. package/dist/adapters/_govinfo_xml.d.ts +22 -0
  27. package/dist/adapters/_govinfo_xml.js +233 -0
  28. package/dist/adapters/_govinfo_xml.js.map +1 -0
  29. package/dist/adapters/_hud_excel.d.ts +2 -0
  30. package/dist/adapters/_hud_excel.js +166 -0
  31. package/dist/adapters/_hud_excel.js.map +1 -0
  32. package/dist/adapters/_omb_grid.d.ts +17 -0
  33. package/dist/adapters/_omb_grid.js +132 -0
  34. package/dist/adapters/_omb_grid.js.map +1 -0
  35. package/dist/adapters/_omb_util.d.ts +40 -0
  36. package/dist/adapters/_omb_util.js +192 -0
  37. package/dist/adapters/_omb_util.js.map +1 -0
  38. package/dist/adapters/_usaspending_state.d.ts +6 -0
  39. package/dist/adapters/_usaspending_state.js +54 -0
  40. package/dist/adapters/_usaspending_state.js.map +1 -0
  41. package/dist/adapters/_usda-fna_book.d.ts +59 -0
  42. package/dist/adapters/_usda-fna_book.js +869 -0
  43. package/dist/adapters/_usda-fna_book.js.map +1 -0
  44. package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
  45. package/dist/adapters/_usda-fna_pdf.js +361 -0
  46. package/dist/adapters/_usda-fna_pdf.js.map +1 -0
  47. package/dist/adapters/_usda-outlook_read.d.ts +70 -0
  48. package/dist/adapters/_usda-outlook_read.js +362 -0
  49. package/dist/adapters/_usda-outlook_read.js.map +1 -0
  50. package/dist/adapters/acf.d.ts +3 -0
  51. package/dist/adapters/acf.js +187 -0
  52. package/dist/adapters/acf.js.map +1 -0
  53. package/dist/adapters/actuaries.d.ts +3 -0
  54. package/dist/adapters/actuaries.js +293 -0
  55. package/dist/adapters/actuaries.js.map +1 -0
  56. package/dist/adapters/bts.d.ts +3 -0
  57. package/dist/adapters/bts.js +26 -0
  58. package/dist/adapters/bts.js.map +1 -0
  59. package/dist/adapters/census-geocoder.d.ts +10 -0
  60. package/dist/adapters/census-geocoder.js +47 -0
  61. package/dist/adapters/census-geocoder.js.map +1 -0
  62. package/dist/adapters/census.d.ts +17 -0
  63. package/dist/adapters/census.js +983 -0
  64. package/dist/adapters/census.js.map +1 -0
  65. package/dist/adapters/cfpb.d.ts +7 -0
  66. package/dist/adapters/cfpb.js +348 -0
  67. package/dist/adapters/cfpb.js.map +1 -0
  68. package/dist/adapters/clinical-trials.d.ts +3 -0
  69. package/dist/adapters/clinical-trials.js +135 -0
  70. package/dist/adapters/clinical-trials.js.map +1 -0
  71. package/dist/adapters/cms.d.ts +3 -0
  72. package/dist/adapters/cms.js +292 -0
  73. package/dist/adapters/cms.js.map +1 -0
  74. package/dist/adapters/college-scorecard.d.ts +3 -0
  75. package/dist/adapters/college-scorecard.js +122 -0
  76. package/dist/adapters/college-scorecard.js.map +1 -0
  77. package/dist/adapters/congress.d.ts +33 -0
  78. package/dist/adapters/congress.js +589 -0
  79. package/dist/adapters/congress.js.map +1 -0
  80. package/dist/adapters/dhs-ohss.d.ts +13 -0
  81. package/dist/adapters/dhs-ohss.js +598 -0
  82. package/dist/adapters/dhs-ohss.js.map +1 -0
  83. package/dist/adapters/doj-news.d.ts +7 -0
  84. package/dist/adapters/doj-news.js +286 -0
  85. package/dist/adapters/doj-news.js.map +1 -0
  86. package/dist/adapters/dol.d.ts +4 -0
  87. package/dist/adapters/dol.js +634 -0
  88. package/dist/adapters/dol.js.map +1 -0
  89. package/dist/adapters/eia.d.ts +15 -0
  90. package/dist/adapters/eia.js +456 -0
  91. package/dist/adapters/eia.js.map +1 -0
  92. package/dist/adapters/epa-aqs.d.ts +3 -0
  93. package/dist/adapters/epa-aqs.js +357 -0
  94. package/dist/adapters/epa-aqs.js.map +1 -0
  95. package/dist/adapters/epa.d.ts +3 -0
  96. package/dist/adapters/epa.js +298 -0
  97. package/dist/adapters/epa.js.map +1 -0
  98. package/dist/adapters/fbi.d.ts +3 -0
  99. package/dist/adapters/fbi.js +192 -0
  100. package/dist/adapters/fbi.js.map +1 -0
  101. package/dist/adapters/fda.d.ts +5 -0
  102. package/dist/adapters/fda.js +184 -0
  103. package/dist/adapters/fda.js.map +1 -0
  104. package/dist/adapters/fdic.d.ts +4 -0
  105. package/dist/adapters/fdic.js +95 -0
  106. package/dist/adapters/fdic.js.map +1 -0
  107. package/dist/adapters/fec.d.ts +3 -0
  108. package/dist/adapters/fec.js +142 -0
  109. package/dist/adapters/fec.js.map +1 -0
  110. package/dist/adapters/fed-board.d.ts +12 -0
  111. package/dist/adapters/fed-board.js +581 -0
  112. package/dist/adapters/fed-board.js.map +1 -0
  113. package/dist/adapters/federal-register.d.ts +11 -0
  114. package/dist/adapters/federal-register.js +43 -0
  115. package/dist/adapters/federal-register.js.map +1 -0
  116. package/dist/adapters/fhfa.d.ts +3 -0
  117. package/dist/adapters/fhfa.js +253 -0
  118. package/dist/adapters/fhfa.js.map +1 -0
  119. package/dist/adapters/fsa.d.ts +9 -0
  120. package/dist/adapters/fsa.js +335 -0
  121. package/dist/adapters/fsa.js.map +1 -0
  122. package/dist/adapters/govinfo.d.ts +30 -0
  123. package/dist/adapters/govinfo.js +636 -0
  124. package/dist/adapters/govinfo.js.map +1 -0
  125. package/dist/adapters/gsa-calc.d.ts +10 -0
  126. package/dist/adapters/gsa-calc.js +318 -0
  127. package/dist/adapters/gsa-calc.js.map +1 -0
  128. package/dist/adapters/hud.d.ts +3 -0
  129. package/dist/adapters/hud.js +200 -0
  130. package/dist/adapters/hud.js.map +1 -0
  131. package/dist/adapters/index.d.ts +3 -0
  132. package/dist/adapters/index.js +25 -0
  133. package/dist/adapters/index.js.map +1 -0
  134. package/dist/adapters/irs-soi.d.ts +6 -0
  135. package/dist/adapters/irs-soi.js +576 -0
  136. package/dist/adapters/irs-soi.js.map +1 -0
  137. package/dist/adapters/naep.d.ts +15 -0
  138. package/dist/adapters/naep.js +341 -0
  139. package/dist/adapters/naep.js.map +1 -0
  140. package/dist/adapters/nhtsa.d.ts +10 -0
  141. package/dist/adapters/nhtsa.js +239 -0
  142. package/dist/adapters/nhtsa.js.map +1 -0
  143. package/dist/adapters/nih.d.ts +5 -0
  144. package/dist/adapters/nih.js +196 -0
  145. package/dist/adapters/nih.js.map +1 -0
  146. package/dist/adapters/noaa.d.ts +3 -0
  147. package/dist/adapters/noaa.js +148 -0
  148. package/dist/adapters/noaa.js.map +1 -0
  149. package/dist/adapters/nrel.d.ts +3 -0
  150. package/dist/adapters/nrel.js +186 -0
  151. package/dist/adapters/nrel.js.map +1 -0
  152. package/dist/adapters/nws.d.ts +3 -0
  153. package/dist/adapters/nws.js +119 -0
  154. package/dist/adapters/nws.js.map +1 -0
  155. package/dist/adapters/omb.d.ts +15 -0
  156. package/dist/adapters/omb.js +517 -0
  157. package/dist/adapters/omb.js.map +1 -0
  158. package/dist/adapters/open-payments.d.ts +5 -0
  159. package/dist/adapters/open-payments.js +149 -0
  160. package/dist/adapters/open-payments.js.map +1 -0
  161. package/dist/adapters/pbgc.d.ts +13 -0
  162. package/dist/adapters/pbgc.js +283 -0
  163. package/dist/adapters/pbgc.js.map +1 -0
  164. package/dist/adapters/philfed.d.ts +12 -0
  165. package/dist/adapters/philfed.js +59 -0
  166. package/dist/adapters/philfed.js.map +1 -0
  167. package/dist/adapters/regulations.d.ts +6 -0
  168. package/dist/adapters/regulations.js +58 -0
  169. package/dist/adapters/regulations.js.map +1 -0
  170. package/dist/adapters/sec.d.ts +13 -0
  171. package/dist/adapters/sec.js +504 -0
  172. package/dist/adapters/sec.js.map +1 -0
  173. package/dist/adapters/senate-lobbying.d.ts +3 -0
  174. package/dist/adapters/senate-lobbying.js +181 -0
  175. package/dist/adapters/senate-lobbying.js.map +1 -0
  176. package/dist/adapters/snapqc.d.ts +32 -0
  177. package/dist/adapters/snapqc.js +701 -0
  178. package/dist/adapters/snapqc.js.map +1 -0
  179. package/dist/adapters/treasury-yields.d.ts +3 -0
  180. package/dist/adapters/treasury-yields.js +170 -0
  181. package/dist/adapters/treasury-yields.js.map +1 -0
  182. package/dist/adapters/usaspending.d.ts +3 -0
  183. package/dist/adapters/usaspending.js +340 -0
  184. package/dist/adapters/usaspending.js.map +1 -0
  185. package/dist/adapters/usda-fna.d.ts +7 -0
  186. package/dist/adapters/usda-fna.js +707 -0
  187. package/dist/adapters/usda-fna.js.map +1 -0
  188. package/dist/adapters/usda-fooddata.d.ts +3 -0
  189. package/dist/adapters/usda-fooddata.js +101 -0
  190. package/dist/adapters/usda-fooddata.js.map +1 -0
  191. package/dist/adapters/usda-nass.d.ts +3 -0
  192. package/dist/adapters/usda-nass.js +108 -0
  193. package/dist/adapters/usda-nass.js.map +1 -0
  194. package/dist/adapters/usda-outlook.d.ts +12 -0
  195. package/dist/adapters/usda-outlook.js +696 -0
  196. package/dist/adapters/usda-outlook.js.map +1 -0
  197. package/dist/adapters/usgs.d.ts +9 -0
  198. package/dist/adapters/usgs.js +265 -0
  199. package/dist/adapters/usgs.js.map +1 -0
  200. package/dist/adapters/usitc.d.ts +18 -0
  201. package/dist/adapters/usitc.js +170 -0
  202. package/dist/adapters/usitc.js.map +1 -0
  203. package/dist/adapters/uspto.d.ts +3 -0
  204. package/dist/adapters/uspto.js +47 -0
  205. package/dist/adapters/uspto.js.map +1 -0
  206. package/dist/adapters/world-bank.d.ts +3 -0
  207. package/dist/adapters/world-bank.js +163 -0
  208. package/dist/adapters/world-bank.js.map +1 -0
  209. package/dist/analysis/cite.d.ts +31 -0
  210. package/dist/analysis/cite.js +229 -0
  211. package/dist/analysis/cite.js.map +1 -0
  212. package/dist/analysis/combine.d.ts +27 -0
  213. package/dist/analysis/combine.js +198 -0
  214. package/dist/analysis/combine.js.map +1 -0
  215. package/dist/analysis/compute.d.ts +62 -0
  216. package/dist/analysis/compute.js +583 -0
  217. package/dist/analysis/compute.js.map +1 -0
  218. package/dist/analysis/expr.d.ts +69 -0
  219. package/dist/analysis/expr.js +780 -0
  220. package/dist/analysis/expr.js.map +1 -0
  221. package/dist/analysis/factfmt.d.ts +16 -0
  222. package/dist/analysis/factfmt.js +187 -0
  223. package/dist/analysis/factfmt.js.map +1 -0
  224. package/dist/analysis/facts.d.ts +63 -0
  225. package/dist/analysis/facts.js +355 -0
  226. package/dist/analysis/facts.js.map +1 -0
  227. package/dist/analysis/handoff.d.ts +58 -0
  228. package/dist/analysis/handoff.js +208 -0
  229. package/dist/analysis/handoff.js.map +1 -0
  230. package/dist/analysis/pyfmt.d.ts +16 -0
  231. package/dist/analysis/pyfmt.js +156 -0
  232. package/dist/analysis/pyfmt.js.map +1 -0
  233. package/dist/analysis/rerun.d.ts +13 -0
  234. package/dist/analysis/rerun.js +125 -0
  235. package/dist/analysis/rerun.js.map +1 -0
  236. package/dist/analysis/table.d.ts +18 -0
  237. package/dist/analysis/table.js +109 -0
  238. package/dist/analysis/table.js.map +1 -0
  239. package/dist/analysis/tools.d.ts +67 -0
  240. package/dist/analysis/tools.js +217 -0
  241. package/dist/analysis/tools.js.map +1 -0
  242. package/dist/analysis/write_table.d.ts +13 -0
  243. package/dist/analysis/write_table.js +59 -0
  244. package/dist/analysis/write_table.js.map +1 -0
  245. package/dist/bin/cli.d.ts +10 -0
  246. package/dist/bin/cli.js +177 -0
  247. package/dist/bin/cli.js.map +1 -0
  248. package/dist/core/cache.d.ts +14 -0
  249. package/dist/core/cache.js +90 -0
  250. package/dist/core/cache.js.map +1 -0
  251. package/dist/core/context.d.ts +12 -0
  252. package/dist/core/context.js +15 -0
  253. package/dist/core/context.js.map +1 -0
  254. package/dist/core/env.d.ts +14 -0
  255. package/dist/core/env.js +52 -0
  256. package/dist/core/env.js.map +1 -0
  257. package/dist/core/generic-rest.d.ts +61 -0
  258. package/dist/core/generic-rest.js +639 -0
  259. package/dist/core/generic-rest.js.map +1 -0
  260. package/dist/core/http.d.ts +120 -0
  261. package/dist/core/http.js +514 -0
  262. package/dist/core/http.js.map +1 -0
  263. package/dist/core/keys.d.ts +43 -0
  264. package/dist/core/keys.js +174 -0
  265. package/dist/core/keys.js.map +1 -0
  266. package/dist/core/manual.d.ts +41 -0
  267. package/dist/core/manual.js +218 -0
  268. package/dist/core/manual.js.map +1 -0
  269. package/dist/core/package.d.ts +58 -0
  270. package/dist/core/package.js +228 -0
  271. package/dist/core/package.js.map +1 -0
  272. package/dist/core/packs.d.ts +139 -0
  273. package/dist/core/packs.js +173 -0
  274. package/dist/core/packs.js.map +1 -0
  275. package/dist/core/policy.d.ts +69 -0
  276. package/dist/core/policy.js +189 -0
  277. package/dist/core/policy.js.map +1 -0
  278. package/dist/core/py.d.ts +40 -0
  279. package/dist/core/py.js +171 -0
  280. package/dist/core/py.js.map +1 -0
  281. package/dist/core/routing.d.ts +10 -0
  282. package/dist/core/routing.js +39 -0
  283. package/dist/core/routing.js.map +1 -0
  284. package/dist/core/tabular.d.ts +21 -0
  285. package/dist/core/tabular.js +229 -0
  286. package/dist/core/tabular.js.map +1 -0
  287. package/dist/deep/_bea_api.d.ts +102 -0
  288. package/dist/deep/_bea_api.js +457 -0
  289. package/dist/deep/_bea_api.js.map +1 -0
  290. package/dist/deep/_bea_fetch.d.ts +151 -0
  291. package/dist/deep/_bea_fetch.js +1274 -0
  292. package/dist/deep/_bea_fetch.js.map +1 -0
  293. package/dist/deep/_bea_util.d.ts +14 -0
  294. package/dist/deep/_bea_util.js +156 -0
  295. package/dist/deep/_bea_util.js.map +1 -0
  296. package/dist/deep/_bls_api.d.ts +84 -0
  297. package/dist/deep/_bls_api.js +231 -0
  298. package/dist/deep/_bls_api.js.map +1 -0
  299. package/dist/deep/_bls_ids.d.ts +43 -0
  300. package/dist/deep/_bls_ids.js +188 -0
  301. package/dist/deep/_bls_ids.js.map +1 -0
  302. package/dist/deep/_bls_lib.d.ts +61 -0
  303. package/dist/deep/_bls_lib.js +442 -0
  304. package/dist/deep/_bls_lib.js.map +1 -0
  305. package/dist/deep/_fred_api.d.ts +148 -0
  306. package/dist/deep/_fred_api.js +545 -0
  307. package/dist/deep/_fred_api.js.map +1 -0
  308. package/dist/deep/_fred_pack.d.ts +162 -0
  309. package/dist/deep/_fred_pack.js +1027 -0
  310. package/dist/deep/_fred_pack.js.map +1 -0
  311. package/dist/deep/_fred_py.d.ts +40 -0
  312. package/dist/deep/_fred_py.js +199 -0
  313. package/dist/deep/_fred_py.js.map +1 -0
  314. package/dist/deep/_ipums_bundle.d.ts +29 -0
  315. package/dist/deep/_ipums_bundle.js +194 -0
  316. package/dist/deep/_ipums_bundle.js.map +1 -0
  317. package/dist/deep/_ipums_ddi.d.ts +23 -0
  318. package/dist/deep/_ipums_ddi.js +162 -0
  319. package/dist/deep/_ipums_ddi.js.map +1 -0
  320. package/dist/deep/_opm_io.d.ts +26 -0
  321. package/dist/deep/_opm_io.js +383 -0
  322. package/dist/deep/_opm_io.js.map +1 -0
  323. package/dist/deep/_opm_parquet.d.ts +31 -0
  324. package/dist/deep/_opm_parquet.js +744 -0
  325. package/dist/deep/_opm_parquet.js.map +1 -0
  326. package/dist/deep/_opm_py.d.ts +6 -0
  327. package/dist/deep/_opm_py.js +74 -0
  328. package/dist/deep/_opm_py.js.map +1 -0
  329. package/dist/deep/_treasury_api.d.ts +48 -0
  330. package/dist/deep/_treasury_api.js +221 -0
  331. package/dist/deep/_treasury_api.js.map +1 -0
  332. package/dist/deep/_treasury_context.d.ts +34 -0
  333. package/dist/deep/_treasury_context.js +142 -0
  334. package/dist/deep/_treasury_context.js.map +1 -0
  335. package/dist/deep/_treasury_lib.d.ts +109 -0
  336. package/dist/deep/_treasury_lib.js +686 -0
  337. package/dist/deep/_treasury_lib.js.map +1 -0
  338. package/dist/deep/bea.d.ts +3 -0
  339. package/dist/deep/bea.js +248 -0
  340. package/dist/deep/bea.js.map +1 -0
  341. package/dist/deep/bls.d.ts +3 -0
  342. package/dist/deep/bls.js +220 -0
  343. package/dist/deep/bls.js.map +1 -0
  344. package/dist/deep/fred.d.ts +3 -0
  345. package/dist/deep/fred.js +325 -0
  346. package/dist/deep/fred.js.map +1 -0
  347. package/dist/deep/ipums.d.ts +59 -0
  348. package/dist/deep/ipums.js +744 -0
  349. package/dist/deep/ipums.js.map +1 -0
  350. package/dist/deep/opm.d.ts +56 -0
  351. package/dist/deep/opm.js +1033 -0
  352. package/dist/deep/opm.js.map +1 -0
  353. package/dist/deep/treasury.d.ts +3 -0
  354. package/dist/deep/treasury.js +218 -0
  355. package/dist/deep/treasury.js.map +1 -0
  356. package/dist/envelope.d.ts +47 -0
  357. package/dist/envelope.js +88 -0
  358. package/dist/envelope.js.map +1 -0
  359. package/dist/index.d.ts +277 -0
  360. package/dist/index.js +106 -0
  361. package/dist/index.js.map +1 -0
  362. package/dist/remote/app.d.ts +21 -0
  363. package/dist/remote/app.js +218 -0
  364. package/dist/remote/app.js.map +1 -0
  365. package/dist/remote/oauth.d.ts +14 -0
  366. package/dist/remote/oauth.js +90 -0
  367. package/dist/remote/oauth.js.map +1 -0
  368. package/dist/remote/pages.d.ts +8 -0
  369. package/dist/remote/pages.js +56 -0
  370. package/dist/remote/pages.js.map +1 -0
  371. package/dist/remote/sign.d.ts +4 -0
  372. package/dist/remote/sign.js +40 -0
  373. package/dist/remote/sign.js.map +1 -0
  374. package/dist/remote/store.d.ts +31 -0
  375. package/dist/remote/store.js +129 -0
  376. package/dist/remote/store.js.map +1 -0
  377. package/dist/report/audit_answer.d.ts +94 -0
  378. package/dist/report/audit_answer.js +1349 -0
  379. package/dist/report/audit_answer.js.map +1 -0
  380. package/dist/report/audit_report.d.ts +23 -0
  381. package/dist/report/audit_report.js +637 -0
  382. package/dist/report/audit_report.js.map +1 -0
  383. package/dist/report/build_report.d.ts +27 -0
  384. package/dist/report/build_report.js +1168 -0
  385. package/dist/report/build_report.js.map +1 -0
  386. package/dist/report/method_audit.d.ts +47 -0
  387. package/dist/report/method_audit.js +716 -0
  388. package/dist/report/method_audit.js.map +1 -0
  389. package/dist/report/policyset.d.ts +50 -0
  390. package/dist/report/policyset.js +421 -0
  391. package/dist/report/policyset.js.map +1 -0
  392. package/dist/report/py.d.ts +58 -0
  393. package/dist/report/py.js +311 -0
  394. package/dist/report/py.js.map +1 -0
  395. package/dist/report/scan_secrets.d.ts +24 -0
  396. package/dist/report/scan_secrets.js +265 -0
  397. package/dist/report/scan_secrets.js.map +1 -0
  398. package/dist/report/style_check.d.ts +29 -0
  399. package/dist/report/style_check.js +173 -0
  400. package/dist/report/style_check.js.map +1 -0
  401. package/dist/report/tools.d.ts +87 -0
  402. package/dist/report/tools.js +167 -0
  403. package/dist/report/tools.js.map +1 -0
  404. package/dist/server.d.ts +5 -0
  405. package/dist/server.js +247 -0
  406. package/dist/server.js.map +1 -0
  407. package/dist/session.d.ts +47 -0
  408. package/dist/session.js +107 -0
  409. package/dist/session.js.map +1 -0
  410. package/dist/toolkit.d.ts +21 -0
  411. package/dist/toolkit.js +2 -0
  412. package/dist/toolkit.js.map +1 -0
  413. package/dist/tools.d.ts +299 -0
  414. package/dist/tools.js +491 -0
  415. package/dist/tools.js.map +1 -0
  416. package/guide/WORKFLOW.md +134 -0
  417. package/guide/assets/disclaimer.txt +1 -0
  418. package/guide/assets/download_data.ps1 +162 -0
  419. package/guide/assets/download_data.sh +143 -0
  420. package/guide/assets/report_template.html +597 -0
  421. package/guide/references/report_patterns.md +199 -0
  422. package/guide/references/routing.md +171 -0
  423. package/guide/references/style_guide.md +398 -0
  424. package/package.json +54 -4
  425. package/packs/acf/endpoints.json +114 -0
  426. package/packs/acf/policy.json +104 -0
  427. package/packs/acf/source.md +138 -0
  428. package/packs/actuaries/endpoints.json +82 -0
  429. package/packs/actuaries/policy.json +109 -0
  430. package/packs/actuaries/source.md +137 -0
  431. package/packs/bea/assets/download_bea.ps1 +182 -0
  432. package/packs/bea/assets/download_bea.sh +144 -0
  433. package/packs/bea/docs/bea_api.md +461 -0
  434. package/packs/bea/docs/datasets.md +157 -0
  435. package/packs/bea/docs/report_patterns.md +112 -0
  436. package/packs/bea/policy.json +170 -0
  437. package/packs/bea/source.md +152 -0
  438. package/packs/bls/assets/download_bls.ps1 +152 -0
  439. package/packs/bls/assets/download_bls.sh +167 -0
  440. package/packs/bls/docs/analysis_patterns.md +108 -0
  441. package/packs/bls/docs/bls_api.md +73 -0
  442. package/packs/bls/docs/series_ids.md +149 -0
  443. package/packs/bls/policy.json +238 -0
  444. package/packs/bls/series_ids.json +1375 -0
  445. package/packs/bls/source.md +156 -0
  446. package/packs/bts/endpoints.json +347 -0
  447. package/packs/bts/policy.json +94 -0
  448. package/packs/bts/source.md +97 -0
  449. package/packs/cdc/endpoints.json +197 -0
  450. package/packs/cdc/policy.json +83 -0
  451. package/packs/cdc/source.md +74 -0
  452. package/packs/census/endpoints.json +99 -0
  453. package/packs/census/policy.json +238 -0
  454. package/packs/census/source.md +304 -0
  455. package/packs/census-geocoder/endpoints.json +70 -0
  456. package/packs/census-geocoder/policy.json +80 -0
  457. package/packs/census-geocoder/source.md +53 -0
  458. package/packs/cfpb/endpoints.json +327 -0
  459. package/packs/cfpb/policy.json +119 -0
  460. package/packs/cfpb/source.md +96 -0
  461. package/packs/clinical-trials/endpoints.json +167 -0
  462. package/packs/clinical-trials/policy.json +84 -0
  463. package/packs/clinical-trials/source.md +82 -0
  464. package/packs/cms/endpoints.json +69 -0
  465. package/packs/cms/policy.json +96 -0
  466. package/packs/cms/source.md +112 -0
  467. package/packs/college-scorecard/endpoints.json +253 -0
  468. package/packs/college-scorecard/policy.json +106 -0
  469. package/packs/college-scorecard/source.md +92 -0
  470. package/packs/congress/endpoints.json +1701 -0
  471. package/packs/congress/policy.json +117 -0
  472. package/packs/congress/source.md +190 -0
  473. package/packs/dhs-ohss/endpoints.json +66 -0
  474. package/packs/dhs-ohss/policy.json +107 -0
  475. package/packs/dhs-ohss/source.md +120 -0
  476. package/packs/doj-news/endpoints.json +59 -0
  477. package/packs/doj-news/policy.json +96 -0
  478. package/packs/doj-news/source.md +99 -0
  479. package/packs/dol/endpoints.json +818 -0
  480. package/packs/dol/policy.json +135 -0
  481. package/packs/dol/source.md +181 -0
  482. package/packs/eia/endpoints.json +718 -0
  483. package/packs/eia/policy.json +112 -0
  484. package/packs/eia/source.md +112 -0
  485. package/packs/epa/endpoints.json +292 -0
  486. package/packs/epa/policy.json +99 -0
  487. package/packs/epa/source.md +99 -0
  488. package/packs/epa-aqs/endpoints.json +174 -0
  489. package/packs/epa-aqs/policy.json +96 -0
  490. package/packs/epa-aqs/source.md +97 -0
  491. package/packs/fbi/endpoints.json +1842 -0
  492. package/packs/fbi/policy.json +118 -0
  493. package/packs/fbi/source.md +108 -0
  494. package/packs/fda/endpoints.json +238 -0
  495. package/packs/fda/policy.json +121 -0
  496. package/packs/fda/source.md +117 -0
  497. package/packs/fdic/endpoints.json +417 -0
  498. package/packs/fdic/policy.json +83 -0
  499. package/packs/fdic/source.md +92 -0
  500. package/packs/fec/endpoints.json +708 -0
  501. package/packs/fec/policy.json +93 -0
  502. package/packs/fec/source.md +90 -0
  503. package/packs/fed-board/endpoints.json +89 -0
  504. package/packs/fed-board/policy.json +136 -0
  505. package/packs/fed-board/source.md +118 -0
  506. package/packs/federal-register/endpoints.json +143 -0
  507. package/packs/federal-register/policy.json +86 -0
  508. package/packs/federal-register/source.md +98 -0
  509. package/packs/fema/endpoints.json +646 -0
  510. package/packs/fema/policy.json +103 -0
  511. package/packs/fema/source.md +103 -0
  512. package/packs/fhfa/endpoints.json +105 -0
  513. package/packs/fhfa/policy.json +109 -0
  514. package/packs/fhfa/source.md +108 -0
  515. package/packs/fred/assets/download_fred.ps1 +122 -0
  516. package/packs/fred/assets/download_fred.sh +123 -0
  517. package/packs/fred/docs/analysis_patterns.md +124 -0
  518. package/packs/fred/docs/fred_api.md +94 -0
  519. package/packs/fred/docs/series.md +188 -0
  520. package/packs/fred/policy.json +253 -0
  521. package/packs/fred/source.md +142 -0
  522. package/packs/fsa/endpoints.json +150 -0
  523. package/packs/fsa/policy.json +99 -0
  524. package/packs/fsa/source.md +102 -0
  525. package/packs/govinfo/endpoints.json +520 -0
  526. package/packs/govinfo/policy.json +116 -0
  527. package/packs/govinfo/source.md +241 -0
  528. package/packs/gsa-calc/endpoints.json +93 -0
  529. package/packs/gsa-calc/policy.json +97 -0
  530. package/packs/gsa-calc/source.md +108 -0
  531. package/packs/hud/endpoints.json +147 -0
  532. package/packs/hud/policy.json +84 -0
  533. package/packs/hud/source.md +97 -0
  534. package/packs/ipums/assets/download_ipums.ps1 +215 -0
  535. package/packs/ipums/assets/download_ipums.sh +192 -0
  536. package/packs/ipums/docs/ipums_api.md +67 -0
  537. package/packs/ipums/docs/report_patterns.md +117 -0
  538. package/packs/ipums/docs/variables.md +284 -0
  539. package/packs/ipums/lib/fetch_text.json +112 -0
  540. package/packs/ipums/policy.json +215 -0
  541. package/packs/ipums/source.md +169 -0
  542. package/packs/irs-soi/endpoints.json +114 -0
  543. package/packs/irs-soi/policy.json +107 -0
  544. package/packs/irs-soi/source.md +124 -0
  545. package/packs/medicaid/endpoints.json +412 -0
  546. package/packs/medicaid/policy.json +108 -0
  547. package/packs/medicaid/source.md +136 -0
  548. package/packs/naep/endpoints.json +217 -0
  549. package/packs/naep/policy.json +100 -0
  550. package/packs/naep/source.md +120 -0
  551. package/packs/nhtsa/endpoints.json +132 -0
  552. package/packs/nhtsa/policy.json +93 -0
  553. package/packs/nhtsa/source.md +109 -0
  554. package/packs/nih/endpoints.json +453 -0
  555. package/packs/nih/policy.json +86 -0
  556. package/packs/nih/source.md +108 -0
  557. package/packs/noaa/endpoints.json +432 -0
  558. package/packs/noaa/policy.json +95 -0
  559. package/packs/noaa/source.md +88 -0
  560. package/packs/nrel/endpoints.json +328 -0
  561. package/packs/nrel/policy.json +96 -0
  562. package/packs/nrel/source.md +91 -0
  563. package/packs/nws/endpoints.json +272 -0
  564. package/packs/nws/policy.json +82 -0
  565. package/packs/nws/source.md +81 -0
  566. package/packs/nyfed/endpoints.json +80 -0
  567. package/packs/nyfed/policy.json +92 -0
  568. package/packs/nyfed/source.md +89 -0
  569. package/packs/omb/endpoints.json +460 -0
  570. package/packs/omb/policy.json +106 -0
  571. package/packs/omb/source.md +130 -0
  572. package/packs/open-payments/endpoints.json +298 -0
  573. package/packs/open-payments/policy.json +101 -0
  574. package/packs/open-payments/source.md +99 -0
  575. package/packs/opm/assets/download_opm.ps1 +125 -0
  576. package/packs/opm/assets/download_opm.sh +85 -0
  577. package/packs/opm/docs/fields.md +184 -0
  578. package/packs/opm/docs/opm_api.md +71 -0
  579. package/packs/opm/docs/report_patterns.md +83 -0
  580. package/packs/opm/lib/data_dictionary.csv +73 -0
  581. package/packs/opm/policy.json +142 -0
  582. package/packs/opm/source.md +137 -0
  583. package/packs/pbgc/endpoints.json +483 -0
  584. package/packs/pbgc/policy.json +94 -0
  585. package/packs/pbgc/source.md +95 -0
  586. package/packs/philfed/endpoints.json +85 -0
  587. package/packs/philfed/policy.json +94 -0
  588. package/packs/philfed/source.md +125 -0
  589. package/packs/port.json +287 -0
  590. package/packs/regulations/endpoints.json +288 -0
  591. package/packs/regulations/policy.json +86 -0
  592. package/packs/regulations/source.md +84 -0
  593. package/packs/sec/endpoints.json +237 -0
  594. package/packs/sec/policy.json +86 -0
  595. package/packs/sec/source.md +101 -0
  596. package/packs/senate-lobbying/endpoints.json +554 -0
  597. package/packs/senate-lobbying/policy.json +99 -0
  598. package/packs/senate-lobbying/source.md +100 -0
  599. package/packs/snapqc/endpoints.json +1426 -0
  600. package/packs/snapqc/policy.json +92 -0
  601. package/packs/snapqc/source.md +96 -0
  602. package/packs/treasury/catalog.json +2267 -0
  603. package/packs/treasury/docs/api_guide.md +245 -0
  604. package/packs/treasury/docs/datasets.md +56 -0
  605. package/packs/treasury/docs/report_patterns.md +126 -0
  606. package/packs/treasury/policy.json +159 -0
  607. package/packs/treasury/source.md +170 -0
  608. package/packs/treasury-yields/endpoints.json +226 -0
  609. package/packs/treasury-yields/policy.json +86 -0
  610. package/packs/treasury-yields/source.md +80 -0
  611. package/packs/usaspending/endpoints.json +1316 -0
  612. package/packs/usaspending/policy.json +109 -0
  613. package/packs/usaspending/source.md +103 -0
  614. package/packs/usda-fna/endpoints.json +427 -0
  615. package/packs/usda-fna/policy.json +133 -0
  616. package/packs/usda-fna/source.md +217 -0
  617. package/packs/usda-fooddata/endpoints.json +181 -0
  618. package/packs/usda-fooddata/policy.json +94 -0
  619. package/packs/usda-fooddata/source.md +105 -0
  620. package/packs/usda-nass/endpoints.json +324 -0
  621. package/packs/usda-nass/policy.json +95 -0
  622. package/packs/usda-nass/source.md +85 -0
  623. package/packs/usda-outlook/endpoints.json +86 -0
  624. package/packs/usda-outlook/policy.json +109 -0
  625. package/packs/usda-outlook/source.md +144 -0
  626. package/packs/usgs/endpoints.json +313 -0
  627. package/packs/usgs/policy.json +95 -0
  628. package/packs/usgs/source.md +96 -0
  629. package/packs/usitc/endpoints.json +74 -0
  630. package/packs/usitc/policy.json +111 -0
  631. package/packs/usitc/source.md +90 -0
  632. package/packs/uspto/endpoints.json +141 -0
  633. package/packs/uspto/policy.json +95 -0
  634. package/packs/uspto/source.md +101 -0
  635. package/packs/va/endpoints.json +414 -0
  636. package/packs/va/policy.json +101 -0
  637. package/packs/va/source.md +129 -0
  638. package/packs/world-bank/endpoints.json +122 -0
  639. package/packs/world-bank/policy.json +101 -0
  640. package/packs/world-bank/source.md +93 -0
@@ -0,0 +1,622 @@
1
+ /**
2
+ * Census microdata on the API: ACS PUMS (acs/acs1/pums, acs/acs5/pums, acs/acs1/pumspr) and the CPS (cps/asec/mar,
3
+ * cps/basic/<month>). Every request gets the person keys and the weights; columns past the 50-variable limit
4
+ * (replicate weights) are fetched in further calls and joined on the keys; a national ACS request is fetched state
5
+ * by state. Rows stream to a CSV on disk, values exactly as published, and a <request>_codebook table gives every
6
+ * code's label. ASEC replicate weights are joined from Census's replicate weight file, through gates.
7
+ */
8
+ import fs from "node:fs";
9
+ import os from "node:os";
10
+ import path from "node:path";
11
+ import zlib from "node:zlib";
12
+ import { parse } from "csv-parse/sync";
13
+ import { Cache } from "../core/cache.js";
14
+ import { DataError } from "../core/http.js";
15
+ import { FileTable } from "../core/package.js";
16
+ import { parsePyJson, pyStr } from "../core/py.js";
17
+ import { censusGet, csvLine, ensureList, given, pyFloat, pyInt, pyRepr, pyTuple, STATES, stateOf, startsWithBrace, ValueErr } from "./_census_util.js";
18
+ export const MAX_VARS = 50;
19
+ const range = (pre, n) => Array.from({ length: n }, (_, i) => `${pre}${i + 1}`);
20
+ export const MICRO = {
21
+ acs_pums: { keys: ["SERIALNO", "SPORDER"], weights: ["PWGTP", "WGTP"],
22
+ replicates: { person: range("PWGTP", 80), household: range("WGTP", 80) } },
23
+ cps_asec: { keys: ["H_SEQ", "A_LINENO", "PERIDNUM"], weights: ["MARSUPWT", "HSUP_WGT"] }, // PERIDNUM alone repeats within a household
24
+ cps_basic: { keys: ["HRHHID", "HRHHID2", "PULINENO"], weights: ["PWSSWGT", "PWCMPWGT"] },
25
+ };
26
+ const MICRO_NOT_PARAMS = ["dataset", "get", "state", "replicates", "moe", "county", "place", "zcta"];
27
+ export const ASEC_RW_DIR = (year) => `https://www2.census.gov/programs-surveys/cps/datasets/${year}/march/`;
28
+ export const ASEC_RW_FIRST = 2005;
29
+ export const ASEC_REPS = range("PWWGT", 160);
30
+ /** 'acs_pums', 'cps_asec' or 'cps_basic' for a microdata dataset path (with or without the year). */
31
+ export function microKind(ds) {
32
+ const d = pyStr(ds).trim().replace(/^\/+|\/+$/g, "");
33
+ if (/(^|\/)acs\/acs[15]\/pums(pr)?$/.test(d))
34
+ return "acs_pums";
35
+ if (/(^|\/)cps\/asec\/mar$/.test(d))
36
+ return "cps_asec";
37
+ if (/(^|\/)cps\/basic\/[a-z]{3}$/.test(d))
38
+ return "cps_basic";
39
+ return null;
40
+ }
41
+ export function datasetYear(ds) {
42
+ const m = /^(\d{4})\//.exec(ds);
43
+ return m ? Number(m[1]) : null;
44
+ }
45
+ const off = (v) => v === undefined || v === null || v === false || v === 0 || v === "" ||
46
+ v === "false" || v === "False" || v === "none";
47
+ /** replicates: true / "person" -> person replicate weights; "household"; "both". */
48
+ export function replicateSets(kind, value) {
49
+ if (off(value))
50
+ return [];
51
+ const sets = MICRO[kind].replicates;
52
+ const v = value === true || pyStr(value).toLowerCase() === "true" ? "person" : pyStr(value).toLowerCase();
53
+ if (kind === "cps_asec") { // not API columns: joined from Census's replicate weight file
54
+ if (!["person", "household", "both"].includes(v))
55
+ throw new ValueErr(`replicates must be true for the ASEC, not ${pyRepr(value)}`);
56
+ return [];
57
+ }
58
+ if (!sets) {
59
+ throw new ValueErr("the CPS basic monthly files have no replicate weights: drop replicates and give an " +
60
+ "approximate standard error, saying so");
61
+ }
62
+ if (!["person", "household", "both"].includes(v)) {
63
+ throw new ValueErr(`replicates must be true, "person", "household" or "both", not ${pyRepr(value)}`);
64
+ }
65
+ return [...(["person", "both"].includes(v) ? sets.person : []), ...(["household", "both"].includes(v) ? sets.household : [])];
66
+ }
67
+ /** The geography parameters of each call: one state, a PUMA list, or (national ACS) one call per state. */
68
+ export function microGeos(ds, kind, p) {
69
+ const st = p.state;
70
+ const notes = [];
71
+ for (const k of ["county", "place", "zcta"]) {
72
+ if (given(p[k])) {
73
+ throw new ValueErr("microdata have no county, place or ZIP geography: the smallest area is the PUMA " +
74
+ '(ACS: for "public use microdata area:*" with a state); CPS has states only');
75
+ }
76
+ }
77
+ const fips = given(st) && st !== "*" ? stateOf(st, "fips") : null;
78
+ const year = datasetYear(pyStr(ds).replace(/^\/+|\/+$/g, ""));
79
+ if (kind === "cps_asec" && year && year < 2001 && (fips || p.for || p.ucgid)) {
80
+ throw new ValueErr("the API's ASEC files before 2001 take no state geography (HTTP 500; checked 2026-11-04): " +
81
+ `fetch the nation (no state) and filter on the file's state variable (request_schema search ` +
82
+ `"${year}/cps/asec/mar: state"), or use the ipums pack (STATEFIP)`);
83
+ }
84
+ if (p.ucgid)
85
+ return [[{ ucgid: p.ucgid }], notes];
86
+ if (p.for) {
87
+ const g = { for: p.for };
88
+ if (p.in || fips)
89
+ g.in = p.in || `state:${fips}`;
90
+ return [[g], notes];
91
+ }
92
+ if (fips)
93
+ return [[{ for: `state:${fips}` }], notes];
94
+ if (kind === "acs_pums" && !pyStr(ds).replace(/\/+$/, "").endsWith("pumspr")) {
95
+ const states = STATES.map(([, f]) => f).filter((f) => parseInt(f, 10) <= 56);
96
+ notes.push(`national ACS PUMS fetched state by state (${states.length} calls: 50 states and DC; Puerto ` +
97
+ "Rico is the separate pumspr dataset)");
98
+ return [states.map((f) => ({ for: `state:${f}` })), notes];
99
+ }
100
+ return [[{}], notes]; // the nation: CPS files answer without a geography (any year)
101
+ }
102
+ /** The columns asked for, keys and weights added, split into calls of at most MAX_VARS variables (each later call
103
+ * repeats the join keys), with the predicates (AGEP=25:54) every call repeats so all chunks cover the same persons. */
104
+ export function microPlan(ds, params, meta = null, absent = new Set()) {
105
+ const kind = microKind(ds);
106
+ const spec = MICRO[kind];
107
+ const p = { ...params };
108
+ const user = [];
109
+ for (const v of ensureList(p.get)) {
110
+ if (v.toUpperCase() !== v)
111
+ throw new ValueErr(`microdata variable names are upper case: ${pyRepr(v)}`);
112
+ if (!user.includes(v))
113
+ user.push(v);
114
+ }
115
+ if (!user.length)
116
+ throw new ValueErr("'get' needs at least one variable (AGEP,HICOV)");
117
+ const notes = [];
118
+ const adj = [];
119
+ for (const v of user) { // dollar amounts need the adjustment factor beside them
120
+ const label = pyStr((meta || {})[v]?.label ?? "");
121
+ for (const f of ["ADJINC", "ADJHSG"])
122
+ if (label.includes(f) && !user.includes(f) && !adj.includes(f))
123
+ adj.push(f);
124
+ }
125
+ if (adj.length) {
126
+ notes.push(`added ${adj.join(", ")}: the dollar variables say to multiply by it (value * ${adj[0]} / ` +
127
+ "1,000,000 gives the survey year's dollars)");
128
+ }
129
+ const reps = replicateSets(kind, p.replicates);
130
+ const asecReps = kind === "cps_asec" && !off(p.replicates);
131
+ const year = datasetYear(pyStr(ds).replace(/^\/+|\/+$/g, ""));
132
+ if (asecReps && year && year < ASEC_RW_FIRST) {
133
+ throw new ValueErr(`Census's ASEC replicate weight files start with ${ASEC_RW_FIRST}: use the ipums pack ` +
134
+ `(REPWTP) for ${year}, or drop replicates`);
135
+ }
136
+ const keys = [...spec.keys, ...(asecReps ? ["PH_SEQ", "PPPOS"] : [])];
137
+ const added = [...keys, ...spec.weights, ...adj].filter((v) => !user.includes(v) && !absent.has(v));
138
+ if (absent.size) {
139
+ notes.push(`not in this file, so not added: ${[...absent].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)).join(", ")} ` +
140
+ "(older years lack some keys and weights)");
141
+ }
142
+ notes.push(`microdata keys and weights added: ${added.join(", ")} (weight every estimate; never count rows)`);
143
+ if (reps.length) {
144
+ notes.push(`${reps.length} replicate weights added (${reps[0]}-${reps[reps.length - 1]}): SE = sqrt(4/80 * sum over r of ` +
145
+ "(estimate_r - estimate)^2), Census's method for ACS PUMS");
146
+ }
147
+ if (asecReps) {
148
+ notes.push("160 replicate weights PWWGT1-PWWGT160 joined from Census's ASEC replicate weight file " +
149
+ "(CPS_ASEC_ASCII_REPWGT_<year>) on PH_SEQ and PPPOS: SE = sqrt(4/160 * sum over r of " +
150
+ "(estimate_r - estimate)^2); household estimates: the reference person's row (A_EXPRRP 1-2) " +
151
+ "with HSUP_WGT and the same replicates");
152
+ }
153
+ const cols = [...user, ...added, ...reps.filter((r) => !user.includes(r))];
154
+ const join = (spec.join || spec.keys).filter((k) => !absent.has(k));
155
+ const first = cols.filter((c) => keys.includes(c) || spec.weights.includes(c));
156
+ let rest = cols.filter((c) => !first.includes(c));
157
+ const room = MAX_VARS - first.length;
158
+ const chunks = [[...rest.slice(0, room), ...first]];
159
+ rest = rest.slice(room);
160
+ while (rest.length) {
161
+ const n = MAX_VARS - join.length;
162
+ chunks.push([...rest.slice(0, n), ...join]);
163
+ rest = rest.slice(n);
164
+ }
165
+ if (chunks.length > 1) {
166
+ notes.push(`${cols.length} columns in ${chunks.length} calls of at most ${MAX_VARS} variables, joined on ${join.join(", ")}`);
167
+ }
168
+ const preds = {};
169
+ for (const [k, v] of Object.entries(p)) {
170
+ if ([...MICRO_NOT_PARAMS, "for", "in", "ucgid"].includes(k) || !given(v))
171
+ continue;
172
+ preds[k] = Array.isArray(v) ? v.map(pyStr).join(",") : v;
173
+ }
174
+ for (const k of Object.keys(preds)) {
175
+ if (k.toUpperCase() !== k) {
176
+ throw new ValueErr(`unknown parameter ${pyRepr(k)} for microdata: filters are variable names in upper case ` +
177
+ '(AGEP: "25:54", HICOV: 2)');
178
+ }
179
+ }
180
+ const [geos, gnotes] = microGeos(ds, kind, p);
181
+ return { kind, cols, chunks, join, geos, preds, asec_reps: asecReps, notes: [...notes, ...gnotes] };
182
+ }
183
+ // ---------------------------------------------------------------------- rows of the API's JSON arrays
184
+ /** Rows of a Census API JSON array, one line at a time (the API writes one row per line), so a large body is
185
+ * never parsed whole; a body on one line (hand-written fixtures) is parsed at once. */
186
+ export function* iterRows(body) {
187
+ const text = body.toString("utf8").replace(/^\uFEFF/, "");
188
+ if ((text.match(/\n/g) || []).length < 2) {
189
+ yield* (parsePyJson(text) || []);
190
+ return;
191
+ }
192
+ let pos = 0;
193
+ while (pos <= text.length) {
194
+ let end = text.indexOf("\n", pos);
195
+ if (end < 0)
196
+ end = text.length;
197
+ let s = text.slice(pos, end).trim();
198
+ pos = end + 1;
199
+ if (!s)
200
+ continue;
201
+ if (s.startsWith("[["))
202
+ s = s.slice(1);
203
+ if (s.endsWith("]]"))
204
+ s = s.slice(0, -1);
205
+ s = s.replace(/,+$/, "");
206
+ if (s)
207
+ yield parsePyJson(s);
208
+ }
209
+ }
210
+ const cell = (v) => (v === null || v === undefined ? "" : v);
211
+ /** [(header, row iterator)] of the calls for one geography -> (columns, joined row iterator). The calls return the
212
+ * persons in the same order, so rows are joined in step and every row's keys are checked; from the first row out of
213
+ * step on, the rest is joined through a key index; a person missing from a call is an error, never a silent blank. */
214
+ export function mergeChunks(parts, join) {
215
+ const cols = [];
216
+ const picks = [];
217
+ for (const [head] of parts) {
218
+ const pick = [];
219
+ head.forEach((c, i) => {
220
+ if (!cols.includes(c)) {
221
+ cols.push(c);
222
+ pick.push(i);
223
+ }
224
+ });
225
+ picks.push(pick);
226
+ }
227
+ if (parts.length === 1) {
228
+ const [, rows] = parts[0];
229
+ return [cols, (function* () {
230
+ for (const row of rows)
231
+ yield picks[0].map((i) => cell(row[i]));
232
+ })()];
233
+ }
234
+ const kidx = parts.map(([head]) => join.map((k) => head.indexOf(k)));
235
+ const its = parts.map(([, rows]) => rows[Symbol.iterator]());
236
+ const take = (j, row) => picks[j].map((i) => cell(row[i]));
237
+ const keyv = (j, row) => kidx[j].map((i) => row[i]);
238
+ const key = (j, row) => JSON.stringify(keyv(j, row));
239
+ const next = (it) => {
240
+ const r = it.next();
241
+ return r.done ? null : r.value;
242
+ };
243
+ function* gen() {
244
+ let pending = null;
245
+ for (;;) {
246
+ const row0 = next(its[0]);
247
+ if (row0 === null)
248
+ break;
249
+ const others = its.slice(1).map(next);
250
+ if (others.every((o, j) => o !== null && key(j + 1, o) === key(0, row0))) {
251
+ yield [...take(0, row0), ...others.flatMap((o, j) => take(j + 1, o))];
252
+ continue;
253
+ }
254
+ pending = [row0, others];
255
+ break;
256
+ }
257
+ if (pending === null) {
258
+ if (its.slice(1).some((it) => next(it) !== null)) {
259
+ throw new DataError(`census microdata: a later call returned persons the first did not (${join.join(", ")})`, "api");
260
+ }
261
+ return;
262
+ }
263
+ const [row0, others] = pending;
264
+ const idx = [];
265
+ its.slice(1).forEach((it, j0) => {
266
+ const m = new Map();
267
+ const o = others[j0];
268
+ if (o !== null)
269
+ m.set(key(j0 + 1, o), o);
270
+ for (let x = next(it); x !== null; x = next(it))
271
+ m.set(key(j0 + 1, x), x);
272
+ idx.push(m);
273
+ });
274
+ const rows0 = [row0];
275
+ for (let x = next(its[0]); x !== null; x = next(its[0]))
276
+ rows0.push(x);
277
+ for (const r0 of rows0) {
278
+ const k0 = key(0, r0);
279
+ const got = idx.map((m) => {
280
+ const g = m.get(k0) ?? null;
281
+ m.delete(k0);
282
+ return g;
283
+ });
284
+ if (got.some((g) => g === null)) {
285
+ throw new DataError(`census microdata: person ${pyTuple(keyv(0, r0))} is missing from a later call; fetch again`, "api");
286
+ }
287
+ yield [...take(0, r0), ...got.flatMap((g, j) => take(j + 1, g))];
288
+ }
289
+ }
290
+ return [cols, gen()];
291
+ }
292
+ // ---------------------------------------------------------------------- ASEC replicate weights
293
+ /** [(data url, SAS layout url)] of the year's replicate weight files, from the folder listing: the main file first,
294
+ * then variants (2020: ..._2020BASE, weights controlled to the 2020 Census). */
295
+ export async function asecRwCandidates(client, year) {
296
+ const base = ASEC_RW_DIR(year);
297
+ const text = (await censusGet(client, base)).text();
298
+ const hrefs = [...text.matchAll(/href="([^"?/][^"]*)"/g)].map((m) => m[1]);
299
+ const re = new RegExp(`^CPS_ASEC_ASCII_REPWGT_${year}(_\\w+)?\\.(dat\\.gz|gz)$`, "i");
300
+ const dats = hrefs.filter((h) => re.test(h));
301
+ const out = [];
302
+ const sorted = [...dats].sort((a, b) => (a.length !== b.length ? a.length - b.length : a < b ? -1 : a > b ? 1 : 0));
303
+ for (const d of sorted) {
304
+ const stem = d.replace(/(\.dat)?\.gz$/i, "");
305
+ const sas = hrefs.filter((h) => h.toLowerCase() === `${stem.toLowerCase()}.sas`);
306
+ if (sas.length)
307
+ out.push([base + d, base + sas[0]]);
308
+ }
309
+ if (!out.length) {
310
+ throw new DataError(`census: no ASEC replicate weight file (CPS_ASEC_ASCII_REPWGT_${year}) in ${base}`, "not_found");
311
+ }
312
+ return out;
313
+ }
314
+ const latin = (b, s, e) => b.toString("latin1", s, Math.min(e, b.length));
315
+ /** The layout and offsets {(h_seq, pppos): start of line} of a replicate weight file (fixed width, layout and weight
316
+ * totals from the year's SAS file). Gate: the sums of PWWGT0, PWWGT1 and PWWGT160 over every record must equal the
317
+ * totals the SAS file states, so a misread layout or a cut download never reaches an estimate. */
318
+ export function asecRwIndex(blob, sas) {
319
+ const lay = {};
320
+ for (const m of sas.matchAll(/@\s*(\d+)\s+(\w+)\s+(\$char|f)?(\d+)\.(\d*)/gi)) {
321
+ lay[m[2].toUpperCase()] = [Number(m[1]) - 1, Number(m[4]), Number(m[5] || 0)];
322
+ }
323
+ const totals = {};
324
+ for (const m of sas.matchAll(/(PWWGT\d+)\s*=\s*(-?[\d.]+)/gi))
325
+ totals[m[1].toUpperCase()] = Number(m[2]);
326
+ const need = ["PWWGT0", ...ASEC_REPS, "H_SEQ", "PPPOS"];
327
+ if (need.some((k) => !(k in lay)) || !["PWWGT0", "PWWGT1", "PWWGT160"].every((k) => k in totals)) {
328
+ throw new DataError("census: the ASEC replicate weight SAS file has no layout or totals for PWWGT0-160, H_SEQ, " +
329
+ "PPPOS: the file format changed; use the ipums pack for ASEC standard errors", "api");
330
+ }
331
+ const check = ["PWWGT0", "PWWGT1", "PWWGT160"];
332
+ const sums = { PWWGT0: 0.0, PWWGT1: 0.0, PWWGT160: 0.0 };
333
+ const [hs, hw] = lay.H_SEQ, [ps, pw] = lay.PPPOS;
334
+ const offsets = new Map();
335
+ let pos = 0;
336
+ const n = blob.length;
337
+ while (pos < n) {
338
+ let end = blob.indexOf(0x0a, pos);
339
+ end = end < 0 ? n : end;
340
+ let lend = end;
341
+ while (lend > pos && blob[lend - 1] === 0x0d)
342
+ lend--;
343
+ const line = latin(blob, pos, lend);
344
+ if (line.trim()) {
345
+ for (const k of check) {
346
+ const [st, w, dec] = lay[k];
347
+ sums[k] += pyInt(line.slice(st, st + w)) / 10 ** dec;
348
+ }
349
+ offsets.set(`${pyInt(line.slice(hs, hs + hw))}|${pyInt(line.slice(ps, ps + pw))}`, pos);
350
+ }
351
+ pos = end + 1;
352
+ }
353
+ const bad = check.filter((k) => Math.abs(sums[k] - totals[k]) > 0.05);
354
+ if (bad.length) {
355
+ const diff = "{" + bad.map((k) => `'${k}': ${pyRound2(sums[k] - totals[k])}`).join(", ") + "}";
356
+ throw new DataError(`census: the ASEC replicate weight file does not add up to its stated totals (difference ` +
357
+ `${diff}): a cut download or a changed layout; fetch again`, "api");
358
+ }
359
+ return { layout: lay, offsets, records: offsets.size };
360
+ }
361
+ function pyRound2(x) {
362
+ const r = Math.round(x * 100) / 100;
363
+ return Number.isInteger(r) ? `${r}.0` : String(r);
364
+ }
365
+ /** (PWWGT0, [PWWGT1..160 as text with 4 decimals]) of one person, or null when the person is not in the file. */
366
+ export function asecRwValues(blob, idx, key) {
367
+ const pos = idx.offsets.get(`${key[0]}|${key[1]}`);
368
+ if (pos === undefined)
369
+ return null;
370
+ const val = (k) => {
371
+ const [st, w, dec] = idx.layout[k];
372
+ return pyInt(latin(blob, pos + st, pos + st + w)) / 10 ** dec;
373
+ };
374
+ return [val("PWWGT0"), ASEC_REPS.map((k) => val(k).toFixed(4))];
375
+ }
376
+ /** Copy the streamed CSV `src` to `dst` with PWWGT1-160 appended. Gate: every person must be in the file and its
377
+ * full-sample weight PWWGT0 must equal the API's MARSUPWT; otherwise DataError (the caller tries the next candidate
378
+ * file). Returns [columns, rows, first 500 rows]. */
379
+ export function asecJoin(src, dst, cols, blob, idx) {
380
+ const iH = cols.indexOf("PH_SEQ"), iP = cols.indexOf("PPPOS"), iW = cols.indexOf("MARSUPWT");
381
+ const outCols = [...cols, ...ASEC_REPS];
382
+ let n = 0, missing = 0, offw = 0;
383
+ const sample = [];
384
+ const grid = parse(fs.readFileSync(src, "utf8"), { relax_column_count: true });
385
+ const fd = fs.openSync(dst, "w");
386
+ try {
387
+ let buf = csvLine(outCols);
388
+ for (const row0 of grid.slice(1)) {
389
+ const got = asecRwValues(blob, idx, [pyInt(row0[iH]), pyInt(row0[iP])]);
390
+ if (got === null) {
391
+ missing++;
392
+ continue;
393
+ }
394
+ const w = row0[iW] ? pyFloat(row0[iW]) : 0;
395
+ if (w === null)
396
+ throw new Error(`could not convert string to float: ${pyRepr(row0[iW])}`);
397
+ if (Math.abs(got[0] - w) > 0.01)
398
+ offw++;
399
+ const row = [...row0, ...got[1]];
400
+ buf += csvLine(row);
401
+ if (buf.length > 1 << 20) {
402
+ fs.writeSync(fd, buf);
403
+ buf = "";
404
+ }
405
+ n++;
406
+ if (sample.length < 500)
407
+ sample.push(Object.fromEntries(outCols.map((c, i) => [c, row[i]])));
408
+ }
409
+ fs.writeSync(fd, buf);
410
+ }
411
+ finally {
412
+ fs.closeSync(fd);
413
+ }
414
+ if (missing || offw) {
415
+ throw new DataError(`census: ${missing} persons not in the ASEC replicate weight file and ${offw} whose PWWGT0 ` +
416
+ "differs from MARSUPWT", "api");
417
+ }
418
+ return [outCols, n, sample];
419
+ }
420
+ // ---------------------------------------------------------------------- variable metadata and codebook
421
+ /** {variable: metadata} from /variables/<VAR>.json (cached 30 days); an unknown name is a spec error (strict), else
422
+ * left out of the result. */
423
+ export async function microMeta(client, pack, ds, vars, strict = true) {
424
+ const out = {};
425
+ const bad = [];
426
+ const useCache = !(process.env.DATA_ANALYST_FIXTURES || process.env.DATA_ANALYST_RECORD);
427
+ const cache = useCache ? new Cache("census", pack.policy) : null;
428
+ for (const v of vars) {
429
+ const url = `${client.base}/${ds}/variables/${v}.json`;
430
+ const hit = cache ? cache.get(url) : null;
431
+ if (hit) {
432
+ out[v] = parsePyJson(hit[0].toString("utf8"));
433
+ continue;
434
+ }
435
+ const r = await censusGet(client, `/${ds}/variables/${v}.json`, null, true);
436
+ if (r.status === 404 || !startsWithBrace(r.body)) {
437
+ bad.push(v);
438
+ continue;
439
+ }
440
+ out[v] = r.json();
441
+ if (cache)
442
+ cache.put(url, r.body, { dataset: ds });
443
+ }
444
+ if (bad.length && strict) {
445
+ throw new DataError(`not variables of ${ds}: ${bad.join(", ")}; find names with the request_schema search "${ds}: <words>"`, "bad_request");
446
+ }
447
+ return out;
448
+ }
449
+ /** One row per code (variable, label, value, value_label); ranges as "min..max"; weights and keys one row. */
450
+ export function microCodebook(meta) {
451
+ const order = (x) => {
452
+ const f = pyFloat(x);
453
+ return f === null ? [1, 0.0, x] : [0, f, ""];
454
+ };
455
+ const cmp = (a, b) => a[0] - b[0] || (a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0) || (a[2] < b[2] ? -1 : a[2] > b[2] ? 1 : 0);
456
+ const rows = [];
457
+ for (const [v, m] of Object.entries(meta)) {
458
+ const base = { variable: v, label: m.label ?? "", predicate_type: m.predicateType ?? "",
459
+ suggested_weight: m["suggested-weight"] ?? "" };
460
+ const vals = m.values || {};
461
+ const items = Object.entries(vals.item || {}).sort((a, b) => cmp(order(a[0]), order(b[0])));
462
+ const rng = vals.range || [];
463
+ for (const [code, lab] of items)
464
+ rows.push({ ...base, value: code, value_label: lab });
465
+ for (const r of rng)
466
+ rows.push({ ...base, value: `${pyStr(r.min ?? "")}..${pyStr(r.max ?? "")}`, value_label: r.description ?? "" });
467
+ if (!items.length && !rng.length)
468
+ rows.push({ ...base, value: "", value_label: "" });
469
+ }
470
+ return rows;
471
+ }
472
+ // ---------------------------------------------------------------------- fetch
473
+ function tmpFile(prefix) {
474
+ const p = path.join(os.tmpdir(), `${prefix}${process.pid}_${Date.now()}_${Math.random().toString(36).slice(2)}.csv`);
475
+ fs.writeFileSync(p, "");
476
+ return p;
477
+ }
478
+ /** Join PWWGT1-160 to the streamed ASEC CSV; tries each candidate file of the year until one passes the gates. */
479
+ async function withAsecReps(client, ds, tmp, cols, nonmiss) {
480
+ const year = datasetYear(ds);
481
+ const errors = [];
482
+ for (const [dat, sasUrl] of await asecRwCandidates(client, year)) {
483
+ const old = client.timeout;
484
+ client.timeout = Math.max(old, 300); // about 70 MB
485
+ let blob, sas;
486
+ try {
487
+ blob = zlib.gunzipSync((await censusGet(client, dat)).body);
488
+ sas = (await censusGet(client, sasUrl)).text();
489
+ }
490
+ finally {
491
+ client.timeout = old;
492
+ }
493
+ let idx, out, res;
494
+ try {
495
+ idx = asecRwIndex(blob, sas);
496
+ out = tmpFile("census_asec_rw_");
497
+ try {
498
+ res = asecJoin(tmp, out, cols, blob, idx);
499
+ }
500
+ catch (e) {
501
+ fs.rmSync(out, { force: true });
502
+ throw e;
503
+ }
504
+ }
505
+ catch (e) {
506
+ if (!(e instanceof DataError))
507
+ throw e;
508
+ errors.push(`${dat.split("/").pop()}: ${e.message}`);
509
+ continue;
510
+ }
511
+ fs.rmSync(tmp, { force: true });
512
+ const [outCols, m, sample] = res;
513
+ const nm = { ...nonmiss, ...Object.fromEntries(ASEC_REPS.map((c) => [c, m])) };
514
+ return [out, outCols, m, sample, nm,
515
+ `replicate weights from ${dat} (${idx.records.toLocaleString("en-US")} records; their sums match the totals in ` +
516
+ `${sasUrl.split("/").pop()}, and every person's PWWGT0 equals MARSUPWT)`];
517
+ }
518
+ fs.rmSync(tmp, { force: true });
519
+ throw new DataError(`census: no ASEC replicate weight file of ${year} matched the API's records: ` + errors.join("; ") +
520
+ ". Use the ipums pack (REPWTP) for ASEC standard errors", "api");
521
+ }
522
+ /** {name: FileTable, name_codebook: rows}, notes. Streams each geography's joined rows to a temp CSV. */
523
+ export async function fetchMicro(client, pack, r, ds, params) {
524
+ const name = r.name;
525
+ const kind = microKind(ds);
526
+ const asked = ensureList(params.get);
527
+ const meta = await microMeta(client, pack, ds, asked.filter((v) => v === v.toUpperCase()));
528
+ let plan;
529
+ try {
530
+ plan = microPlan(ds, params, meta);
531
+ }
532
+ catch (e) {
533
+ if (e instanceof ValueErr)
534
+ throw new DataError(`request ${pyRepr(name)}: ${e.message}`, "bad_request");
535
+ throw e;
536
+ }
537
+ const reps = new Set(replicateSets(kind, params.replicates));
538
+ const extra = plan.cols.filter((c) => !(c in meta) && !reps.has(c));
539
+ const found = await microMeta(client, pack, ds, extra, false);
540
+ Object.assign(meta, found);
541
+ const missing = new Set(extra.filter((c) => !(c in found)));
542
+ if (missing.size) { // an older file without HRHHID2 or PWCMPWGT: plan again without them
543
+ plan = microPlan(ds, params, meta, missing);
544
+ if (plan.chunks.length > 1 && !plan.join.length) {
545
+ throw new DataError(`request ${pyRepr(name)}: ${ds} has none of the join keys, so its columns cannot be split ` +
546
+ `across calls: ask for at most ${MAX_VARS} variables`, "bad_request");
547
+ }
548
+ }
549
+ let tmp = tmpFile(`census_${name}_`);
550
+ let cols = null;
551
+ let n = 0, emptyGeos = 0;
552
+ let nonmiss = {};
553
+ let sample = [];
554
+ try {
555
+ const fd = fs.openSync(tmp, "w");
556
+ try {
557
+ for (const g of plan.geos) {
558
+ let parts = [];
559
+ for (const chunk of plan.chunks) {
560
+ const resp = await censusGet(client, `/${ds}`, { get: chunk.join(","), ...plan.preds, ...g });
561
+ if (resp.status === 204 || !resp.body.toString("latin1").trim()) {
562
+ parts = null;
563
+ break;
564
+ }
565
+ const it = iterRows(resp.body);
566
+ const head = it.next();
567
+ parts.push([head.done ? [] : head.value, { [Symbol.iterator]: () => it }]);
568
+ }
569
+ if (!parts || !parts.length) {
570
+ emptyGeos++;
571
+ continue;
572
+ }
573
+ const [gcols, rows] = mergeChunks(parts, plan.join);
574
+ if (cols === null) { // asked order: variables, keys, weights, replicates, then geography
575
+ cols = [...plan.cols.filter((c) => gcols.includes(c)), ...gcols.filter((c) => !plan.cols.includes(c))];
576
+ fs.writeSync(fd, csvLine(cols));
577
+ nonmiss = Object.fromEntries(cols.map((c) => [c, 0]));
578
+ }
579
+ const order = cols.map((c) => (gcols.includes(c) ? gcols.indexOf(c) : null));
580
+ let buf = "";
581
+ for (const row of rows) {
582
+ const out = order.map((i) => (i !== null ? row[i] : ""));
583
+ buf += csvLine(out);
584
+ if (buf.length > 1 << 20) {
585
+ fs.writeSync(fd, buf);
586
+ buf = "";
587
+ }
588
+ n++;
589
+ cols.forEach((c, i) => {
590
+ if (out[i] !== "")
591
+ nonmiss[c]++;
592
+ });
593
+ if (sample.length < 500)
594
+ sample.push(Object.fromEntries(cols.map((c, i) => [c, out[i]])));
595
+ }
596
+ fs.writeSync(fd, buf);
597
+ }
598
+ }
599
+ finally {
600
+ fs.closeSync(fd);
601
+ }
602
+ }
603
+ catch (e) {
604
+ fs.rmSync(tmp, { force: true });
605
+ throw e;
606
+ }
607
+ const notes = plan.notes.map((x) => `${name}: ${x}`);
608
+ if (plan.asec_reps && cols !== null) {
609
+ let rwNote;
610
+ [tmp, cols, n, sample, nonmiss, rwNote] = await withAsecReps(client, ds, tmp, cols, nonmiss);
611
+ notes.push(`${name}: ${rwNote}`);
612
+ }
613
+ if (emptyGeos)
614
+ notes.push(`${name}: ${emptyGeos} of ${plan.geos.length} geographies had no matching persons`);
615
+ if (cols === null) {
616
+ fs.rmSync(tmp, { force: true });
617
+ return [{ [name]: [], [`${name}_codebook`]: microCodebook(meta) }, notes];
618
+ }
619
+ notes.push(`${name}: codebook (every code's label, from the API's variable metadata) in ${name}_codebook.csv`);
620
+ return [{ [name]: new FileTable(tmp, cols, n, sample, nonmiss), [`${name}_codebook`]: microCodebook(meta) }, notes];
621
+ }
622
+ //# sourceMappingURL=_census_micro.js.map