data-analyst-mcp 0.0.0-stage → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (640) hide show
  1. package/README.md +261 -2
  2. package/dist/adapters/_acf_tidy.d.ts +18 -0
  3. package/dist/adapters/_acf_tidy.js +432 -0
  4. package/dist/adapters/_acf_tidy.js.map +1 -0
  5. package/dist/adapters/_actuaries_read.d.ts +38 -0
  6. package/dist/adapters/_actuaries_read.js +728 -0
  7. package/dist/adapters/_actuaries_read.js.map +1 -0
  8. package/dist/adapters/_census_micro.d.ts +70 -0
  9. package/dist/adapters/_census_micro.js +622 -0
  10. package/dist/adapters/_census_micro.js.map +1 -0
  11. package/dist/adapters/_census_util.d.ts +40 -0
  12. package/dist/adapters/_census_util.js +275 -0
  13. package/dist/adapters/_census_util.js.map +1 -0
  14. package/dist/adapters/_dol_states.d.ts +6 -0
  15. package/dist/adapters/_dol_states.js +58 -0
  16. package/dist/adapters/_dol_states.js.map +1 -0
  17. package/dist/adapters/_govinfo_entities.d.ts +4 -0
  18. package/dist/adapters/_govinfo_entities.js +6 -0
  19. package/dist/adapters/_govinfo_entities.js.map +1 -0
  20. package/dist/adapters/_govinfo_json.d.ts +13 -0
  21. package/dist/adapters/_govinfo_json.js +107 -0
  22. package/dist/adapters/_govinfo_json.js.map +1 -0
  23. package/dist/adapters/_govinfo_py.d.ts +22 -0
  24. package/dist/adapters/_govinfo_py.js +272 -0
  25. package/dist/adapters/_govinfo_py.js.map +1 -0
  26. package/dist/adapters/_govinfo_xml.d.ts +22 -0
  27. package/dist/adapters/_govinfo_xml.js +233 -0
  28. package/dist/adapters/_govinfo_xml.js.map +1 -0
  29. package/dist/adapters/_hud_excel.d.ts +2 -0
  30. package/dist/adapters/_hud_excel.js +166 -0
  31. package/dist/adapters/_hud_excel.js.map +1 -0
  32. package/dist/adapters/_omb_grid.d.ts +17 -0
  33. package/dist/adapters/_omb_grid.js +132 -0
  34. package/dist/adapters/_omb_grid.js.map +1 -0
  35. package/dist/adapters/_omb_util.d.ts +40 -0
  36. package/dist/adapters/_omb_util.js +192 -0
  37. package/dist/adapters/_omb_util.js.map +1 -0
  38. package/dist/adapters/_usaspending_state.d.ts +6 -0
  39. package/dist/adapters/_usaspending_state.js +54 -0
  40. package/dist/adapters/_usaspending_state.js.map +1 -0
  41. package/dist/adapters/_usda-fna_book.d.ts +59 -0
  42. package/dist/adapters/_usda-fna_book.js +869 -0
  43. package/dist/adapters/_usda-fna_book.js.map +1 -0
  44. package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
  45. package/dist/adapters/_usda-fna_pdf.js +361 -0
  46. package/dist/adapters/_usda-fna_pdf.js.map +1 -0
  47. package/dist/adapters/_usda-outlook_read.d.ts +70 -0
  48. package/dist/adapters/_usda-outlook_read.js +362 -0
  49. package/dist/adapters/_usda-outlook_read.js.map +1 -0
  50. package/dist/adapters/acf.d.ts +3 -0
  51. package/dist/adapters/acf.js +187 -0
  52. package/dist/adapters/acf.js.map +1 -0
  53. package/dist/adapters/actuaries.d.ts +3 -0
  54. package/dist/adapters/actuaries.js +293 -0
  55. package/dist/adapters/actuaries.js.map +1 -0
  56. package/dist/adapters/bts.d.ts +3 -0
  57. package/dist/adapters/bts.js +26 -0
  58. package/dist/adapters/bts.js.map +1 -0
  59. package/dist/adapters/census-geocoder.d.ts +10 -0
  60. package/dist/adapters/census-geocoder.js +47 -0
  61. package/dist/adapters/census-geocoder.js.map +1 -0
  62. package/dist/adapters/census.d.ts +17 -0
  63. package/dist/adapters/census.js +983 -0
  64. package/dist/adapters/census.js.map +1 -0
  65. package/dist/adapters/cfpb.d.ts +7 -0
  66. package/dist/adapters/cfpb.js +348 -0
  67. package/dist/adapters/cfpb.js.map +1 -0
  68. package/dist/adapters/clinical-trials.d.ts +3 -0
  69. package/dist/adapters/clinical-trials.js +135 -0
  70. package/dist/adapters/clinical-trials.js.map +1 -0
  71. package/dist/adapters/cms.d.ts +3 -0
  72. package/dist/adapters/cms.js +292 -0
  73. package/dist/adapters/cms.js.map +1 -0
  74. package/dist/adapters/college-scorecard.d.ts +3 -0
  75. package/dist/adapters/college-scorecard.js +122 -0
  76. package/dist/adapters/college-scorecard.js.map +1 -0
  77. package/dist/adapters/congress.d.ts +33 -0
  78. package/dist/adapters/congress.js +589 -0
  79. package/dist/adapters/congress.js.map +1 -0
  80. package/dist/adapters/dhs-ohss.d.ts +13 -0
  81. package/dist/adapters/dhs-ohss.js +598 -0
  82. package/dist/adapters/dhs-ohss.js.map +1 -0
  83. package/dist/adapters/doj-news.d.ts +7 -0
  84. package/dist/adapters/doj-news.js +286 -0
  85. package/dist/adapters/doj-news.js.map +1 -0
  86. package/dist/adapters/dol.d.ts +4 -0
  87. package/dist/adapters/dol.js +634 -0
  88. package/dist/adapters/dol.js.map +1 -0
  89. package/dist/adapters/eia.d.ts +15 -0
  90. package/dist/adapters/eia.js +456 -0
  91. package/dist/adapters/eia.js.map +1 -0
  92. package/dist/adapters/epa-aqs.d.ts +3 -0
  93. package/dist/adapters/epa-aqs.js +357 -0
  94. package/dist/adapters/epa-aqs.js.map +1 -0
  95. package/dist/adapters/epa.d.ts +3 -0
  96. package/dist/adapters/epa.js +298 -0
  97. package/dist/adapters/epa.js.map +1 -0
  98. package/dist/adapters/fbi.d.ts +3 -0
  99. package/dist/adapters/fbi.js +192 -0
  100. package/dist/adapters/fbi.js.map +1 -0
  101. package/dist/adapters/fda.d.ts +5 -0
  102. package/dist/adapters/fda.js +184 -0
  103. package/dist/adapters/fda.js.map +1 -0
  104. package/dist/adapters/fdic.d.ts +4 -0
  105. package/dist/adapters/fdic.js +95 -0
  106. package/dist/adapters/fdic.js.map +1 -0
  107. package/dist/adapters/fec.d.ts +3 -0
  108. package/dist/adapters/fec.js +142 -0
  109. package/dist/adapters/fec.js.map +1 -0
  110. package/dist/adapters/fed-board.d.ts +12 -0
  111. package/dist/adapters/fed-board.js +581 -0
  112. package/dist/adapters/fed-board.js.map +1 -0
  113. package/dist/adapters/federal-register.d.ts +11 -0
  114. package/dist/adapters/federal-register.js +43 -0
  115. package/dist/adapters/federal-register.js.map +1 -0
  116. package/dist/adapters/fhfa.d.ts +3 -0
  117. package/dist/adapters/fhfa.js +253 -0
  118. package/dist/adapters/fhfa.js.map +1 -0
  119. package/dist/adapters/fsa.d.ts +9 -0
  120. package/dist/adapters/fsa.js +335 -0
  121. package/dist/adapters/fsa.js.map +1 -0
  122. package/dist/adapters/govinfo.d.ts +30 -0
  123. package/dist/adapters/govinfo.js +636 -0
  124. package/dist/adapters/govinfo.js.map +1 -0
  125. package/dist/adapters/gsa-calc.d.ts +10 -0
  126. package/dist/adapters/gsa-calc.js +318 -0
  127. package/dist/adapters/gsa-calc.js.map +1 -0
  128. package/dist/adapters/hud.d.ts +3 -0
  129. package/dist/adapters/hud.js +200 -0
  130. package/dist/adapters/hud.js.map +1 -0
  131. package/dist/adapters/index.d.ts +3 -0
  132. package/dist/adapters/index.js +25 -0
  133. package/dist/adapters/index.js.map +1 -0
  134. package/dist/adapters/irs-soi.d.ts +6 -0
  135. package/dist/adapters/irs-soi.js +576 -0
  136. package/dist/adapters/irs-soi.js.map +1 -0
  137. package/dist/adapters/naep.d.ts +15 -0
  138. package/dist/adapters/naep.js +341 -0
  139. package/dist/adapters/naep.js.map +1 -0
  140. package/dist/adapters/nhtsa.d.ts +10 -0
  141. package/dist/adapters/nhtsa.js +239 -0
  142. package/dist/adapters/nhtsa.js.map +1 -0
  143. package/dist/adapters/nih.d.ts +5 -0
  144. package/dist/adapters/nih.js +196 -0
  145. package/dist/adapters/nih.js.map +1 -0
  146. package/dist/adapters/noaa.d.ts +3 -0
  147. package/dist/adapters/noaa.js +148 -0
  148. package/dist/adapters/noaa.js.map +1 -0
  149. package/dist/adapters/nrel.d.ts +3 -0
  150. package/dist/adapters/nrel.js +186 -0
  151. package/dist/adapters/nrel.js.map +1 -0
  152. package/dist/adapters/nws.d.ts +3 -0
  153. package/dist/adapters/nws.js +119 -0
  154. package/dist/adapters/nws.js.map +1 -0
  155. package/dist/adapters/omb.d.ts +15 -0
  156. package/dist/adapters/omb.js +517 -0
  157. package/dist/adapters/omb.js.map +1 -0
  158. package/dist/adapters/open-payments.d.ts +5 -0
  159. package/dist/adapters/open-payments.js +149 -0
  160. package/dist/adapters/open-payments.js.map +1 -0
  161. package/dist/adapters/pbgc.d.ts +13 -0
  162. package/dist/adapters/pbgc.js +283 -0
  163. package/dist/adapters/pbgc.js.map +1 -0
  164. package/dist/adapters/philfed.d.ts +12 -0
  165. package/dist/adapters/philfed.js +59 -0
  166. package/dist/adapters/philfed.js.map +1 -0
  167. package/dist/adapters/regulations.d.ts +6 -0
  168. package/dist/adapters/regulations.js +58 -0
  169. package/dist/adapters/regulations.js.map +1 -0
  170. package/dist/adapters/sec.d.ts +13 -0
  171. package/dist/adapters/sec.js +504 -0
  172. package/dist/adapters/sec.js.map +1 -0
  173. package/dist/adapters/senate-lobbying.d.ts +3 -0
  174. package/dist/adapters/senate-lobbying.js +181 -0
  175. package/dist/adapters/senate-lobbying.js.map +1 -0
  176. package/dist/adapters/snapqc.d.ts +32 -0
  177. package/dist/adapters/snapqc.js +701 -0
  178. package/dist/adapters/snapqc.js.map +1 -0
  179. package/dist/adapters/treasury-yields.d.ts +3 -0
  180. package/dist/adapters/treasury-yields.js +170 -0
  181. package/dist/adapters/treasury-yields.js.map +1 -0
  182. package/dist/adapters/usaspending.d.ts +3 -0
  183. package/dist/adapters/usaspending.js +340 -0
  184. package/dist/adapters/usaspending.js.map +1 -0
  185. package/dist/adapters/usda-fna.d.ts +7 -0
  186. package/dist/adapters/usda-fna.js +707 -0
  187. package/dist/adapters/usda-fna.js.map +1 -0
  188. package/dist/adapters/usda-fooddata.d.ts +3 -0
  189. package/dist/adapters/usda-fooddata.js +101 -0
  190. package/dist/adapters/usda-fooddata.js.map +1 -0
  191. package/dist/adapters/usda-nass.d.ts +3 -0
  192. package/dist/adapters/usda-nass.js +108 -0
  193. package/dist/adapters/usda-nass.js.map +1 -0
  194. package/dist/adapters/usda-outlook.d.ts +12 -0
  195. package/dist/adapters/usda-outlook.js +696 -0
  196. package/dist/adapters/usda-outlook.js.map +1 -0
  197. package/dist/adapters/usgs.d.ts +9 -0
  198. package/dist/adapters/usgs.js +265 -0
  199. package/dist/adapters/usgs.js.map +1 -0
  200. package/dist/adapters/usitc.d.ts +18 -0
  201. package/dist/adapters/usitc.js +170 -0
  202. package/dist/adapters/usitc.js.map +1 -0
  203. package/dist/adapters/uspto.d.ts +3 -0
  204. package/dist/adapters/uspto.js +47 -0
  205. package/dist/adapters/uspto.js.map +1 -0
  206. package/dist/adapters/world-bank.d.ts +3 -0
  207. package/dist/adapters/world-bank.js +163 -0
  208. package/dist/adapters/world-bank.js.map +1 -0
  209. package/dist/analysis/cite.d.ts +31 -0
  210. package/dist/analysis/cite.js +229 -0
  211. package/dist/analysis/cite.js.map +1 -0
  212. package/dist/analysis/combine.d.ts +27 -0
  213. package/dist/analysis/combine.js +198 -0
  214. package/dist/analysis/combine.js.map +1 -0
  215. package/dist/analysis/compute.d.ts +62 -0
  216. package/dist/analysis/compute.js +583 -0
  217. package/dist/analysis/compute.js.map +1 -0
  218. package/dist/analysis/expr.d.ts +69 -0
  219. package/dist/analysis/expr.js +780 -0
  220. package/dist/analysis/expr.js.map +1 -0
  221. package/dist/analysis/factfmt.d.ts +16 -0
  222. package/dist/analysis/factfmt.js +187 -0
  223. package/dist/analysis/factfmt.js.map +1 -0
  224. package/dist/analysis/facts.d.ts +63 -0
  225. package/dist/analysis/facts.js +355 -0
  226. package/dist/analysis/facts.js.map +1 -0
  227. package/dist/analysis/handoff.d.ts +58 -0
  228. package/dist/analysis/handoff.js +208 -0
  229. package/dist/analysis/handoff.js.map +1 -0
  230. package/dist/analysis/pyfmt.d.ts +16 -0
  231. package/dist/analysis/pyfmt.js +156 -0
  232. package/dist/analysis/pyfmt.js.map +1 -0
  233. package/dist/analysis/rerun.d.ts +13 -0
  234. package/dist/analysis/rerun.js +125 -0
  235. package/dist/analysis/rerun.js.map +1 -0
  236. package/dist/analysis/table.d.ts +18 -0
  237. package/dist/analysis/table.js +109 -0
  238. package/dist/analysis/table.js.map +1 -0
  239. package/dist/analysis/tools.d.ts +67 -0
  240. package/dist/analysis/tools.js +217 -0
  241. package/dist/analysis/tools.js.map +1 -0
  242. package/dist/analysis/write_table.d.ts +13 -0
  243. package/dist/analysis/write_table.js +59 -0
  244. package/dist/analysis/write_table.js.map +1 -0
  245. package/dist/bin/cli.d.ts +10 -0
  246. package/dist/bin/cli.js +177 -0
  247. package/dist/bin/cli.js.map +1 -0
  248. package/dist/core/cache.d.ts +14 -0
  249. package/dist/core/cache.js +90 -0
  250. package/dist/core/cache.js.map +1 -0
  251. package/dist/core/context.d.ts +12 -0
  252. package/dist/core/context.js +15 -0
  253. package/dist/core/context.js.map +1 -0
  254. package/dist/core/env.d.ts +14 -0
  255. package/dist/core/env.js +52 -0
  256. package/dist/core/env.js.map +1 -0
  257. package/dist/core/generic-rest.d.ts +61 -0
  258. package/dist/core/generic-rest.js +639 -0
  259. package/dist/core/generic-rest.js.map +1 -0
  260. package/dist/core/http.d.ts +120 -0
  261. package/dist/core/http.js +514 -0
  262. package/dist/core/http.js.map +1 -0
  263. package/dist/core/keys.d.ts +43 -0
  264. package/dist/core/keys.js +174 -0
  265. package/dist/core/keys.js.map +1 -0
  266. package/dist/core/manual.d.ts +41 -0
  267. package/dist/core/manual.js +218 -0
  268. package/dist/core/manual.js.map +1 -0
  269. package/dist/core/package.d.ts +58 -0
  270. package/dist/core/package.js +228 -0
  271. package/dist/core/package.js.map +1 -0
  272. package/dist/core/packs.d.ts +139 -0
  273. package/dist/core/packs.js +173 -0
  274. package/dist/core/packs.js.map +1 -0
  275. package/dist/core/policy.d.ts +69 -0
  276. package/dist/core/policy.js +189 -0
  277. package/dist/core/policy.js.map +1 -0
  278. package/dist/core/py.d.ts +40 -0
  279. package/dist/core/py.js +171 -0
  280. package/dist/core/py.js.map +1 -0
  281. package/dist/core/routing.d.ts +10 -0
  282. package/dist/core/routing.js +39 -0
  283. package/dist/core/routing.js.map +1 -0
  284. package/dist/core/tabular.d.ts +21 -0
  285. package/dist/core/tabular.js +229 -0
  286. package/dist/core/tabular.js.map +1 -0
  287. package/dist/deep/_bea_api.d.ts +102 -0
  288. package/dist/deep/_bea_api.js +457 -0
  289. package/dist/deep/_bea_api.js.map +1 -0
  290. package/dist/deep/_bea_fetch.d.ts +151 -0
  291. package/dist/deep/_bea_fetch.js +1274 -0
  292. package/dist/deep/_bea_fetch.js.map +1 -0
  293. package/dist/deep/_bea_util.d.ts +14 -0
  294. package/dist/deep/_bea_util.js +156 -0
  295. package/dist/deep/_bea_util.js.map +1 -0
  296. package/dist/deep/_bls_api.d.ts +84 -0
  297. package/dist/deep/_bls_api.js +231 -0
  298. package/dist/deep/_bls_api.js.map +1 -0
  299. package/dist/deep/_bls_ids.d.ts +43 -0
  300. package/dist/deep/_bls_ids.js +188 -0
  301. package/dist/deep/_bls_ids.js.map +1 -0
  302. package/dist/deep/_bls_lib.d.ts +61 -0
  303. package/dist/deep/_bls_lib.js +442 -0
  304. package/dist/deep/_bls_lib.js.map +1 -0
  305. package/dist/deep/_fred_api.d.ts +148 -0
  306. package/dist/deep/_fred_api.js +545 -0
  307. package/dist/deep/_fred_api.js.map +1 -0
  308. package/dist/deep/_fred_pack.d.ts +162 -0
  309. package/dist/deep/_fred_pack.js +1027 -0
  310. package/dist/deep/_fred_pack.js.map +1 -0
  311. package/dist/deep/_fred_py.d.ts +40 -0
  312. package/dist/deep/_fred_py.js +199 -0
  313. package/dist/deep/_fred_py.js.map +1 -0
  314. package/dist/deep/_ipums_bundle.d.ts +29 -0
  315. package/dist/deep/_ipums_bundle.js +194 -0
  316. package/dist/deep/_ipums_bundle.js.map +1 -0
  317. package/dist/deep/_ipums_ddi.d.ts +23 -0
  318. package/dist/deep/_ipums_ddi.js +162 -0
  319. package/dist/deep/_ipums_ddi.js.map +1 -0
  320. package/dist/deep/_opm_io.d.ts +26 -0
  321. package/dist/deep/_opm_io.js +383 -0
  322. package/dist/deep/_opm_io.js.map +1 -0
  323. package/dist/deep/_opm_parquet.d.ts +31 -0
  324. package/dist/deep/_opm_parquet.js +744 -0
  325. package/dist/deep/_opm_parquet.js.map +1 -0
  326. package/dist/deep/_opm_py.d.ts +6 -0
  327. package/dist/deep/_opm_py.js +74 -0
  328. package/dist/deep/_opm_py.js.map +1 -0
  329. package/dist/deep/_treasury_api.d.ts +48 -0
  330. package/dist/deep/_treasury_api.js +221 -0
  331. package/dist/deep/_treasury_api.js.map +1 -0
  332. package/dist/deep/_treasury_context.d.ts +34 -0
  333. package/dist/deep/_treasury_context.js +142 -0
  334. package/dist/deep/_treasury_context.js.map +1 -0
  335. package/dist/deep/_treasury_lib.d.ts +109 -0
  336. package/dist/deep/_treasury_lib.js +686 -0
  337. package/dist/deep/_treasury_lib.js.map +1 -0
  338. package/dist/deep/bea.d.ts +3 -0
  339. package/dist/deep/bea.js +248 -0
  340. package/dist/deep/bea.js.map +1 -0
  341. package/dist/deep/bls.d.ts +3 -0
  342. package/dist/deep/bls.js +220 -0
  343. package/dist/deep/bls.js.map +1 -0
  344. package/dist/deep/fred.d.ts +3 -0
  345. package/dist/deep/fred.js +325 -0
  346. package/dist/deep/fred.js.map +1 -0
  347. package/dist/deep/ipums.d.ts +59 -0
  348. package/dist/deep/ipums.js +744 -0
  349. package/dist/deep/ipums.js.map +1 -0
  350. package/dist/deep/opm.d.ts +56 -0
  351. package/dist/deep/opm.js +1033 -0
  352. package/dist/deep/opm.js.map +1 -0
  353. package/dist/deep/treasury.d.ts +3 -0
  354. package/dist/deep/treasury.js +218 -0
  355. package/dist/deep/treasury.js.map +1 -0
  356. package/dist/envelope.d.ts +47 -0
  357. package/dist/envelope.js +88 -0
  358. package/dist/envelope.js.map +1 -0
  359. package/dist/index.d.ts +277 -0
  360. package/dist/index.js +106 -0
  361. package/dist/index.js.map +1 -0
  362. package/dist/remote/app.d.ts +21 -0
  363. package/dist/remote/app.js +218 -0
  364. package/dist/remote/app.js.map +1 -0
  365. package/dist/remote/oauth.d.ts +14 -0
  366. package/dist/remote/oauth.js +90 -0
  367. package/dist/remote/oauth.js.map +1 -0
  368. package/dist/remote/pages.d.ts +8 -0
  369. package/dist/remote/pages.js +56 -0
  370. package/dist/remote/pages.js.map +1 -0
  371. package/dist/remote/sign.d.ts +4 -0
  372. package/dist/remote/sign.js +40 -0
  373. package/dist/remote/sign.js.map +1 -0
  374. package/dist/remote/store.d.ts +31 -0
  375. package/dist/remote/store.js +129 -0
  376. package/dist/remote/store.js.map +1 -0
  377. package/dist/report/audit_answer.d.ts +94 -0
  378. package/dist/report/audit_answer.js +1349 -0
  379. package/dist/report/audit_answer.js.map +1 -0
  380. package/dist/report/audit_report.d.ts +23 -0
  381. package/dist/report/audit_report.js +637 -0
  382. package/dist/report/audit_report.js.map +1 -0
  383. package/dist/report/build_report.d.ts +27 -0
  384. package/dist/report/build_report.js +1168 -0
  385. package/dist/report/build_report.js.map +1 -0
  386. package/dist/report/method_audit.d.ts +47 -0
  387. package/dist/report/method_audit.js +716 -0
  388. package/dist/report/method_audit.js.map +1 -0
  389. package/dist/report/policyset.d.ts +50 -0
  390. package/dist/report/policyset.js +421 -0
  391. package/dist/report/policyset.js.map +1 -0
  392. package/dist/report/py.d.ts +58 -0
  393. package/dist/report/py.js +311 -0
  394. package/dist/report/py.js.map +1 -0
  395. package/dist/report/scan_secrets.d.ts +24 -0
  396. package/dist/report/scan_secrets.js +265 -0
  397. package/dist/report/scan_secrets.js.map +1 -0
  398. package/dist/report/style_check.d.ts +29 -0
  399. package/dist/report/style_check.js +173 -0
  400. package/dist/report/style_check.js.map +1 -0
  401. package/dist/report/tools.d.ts +87 -0
  402. package/dist/report/tools.js +167 -0
  403. package/dist/report/tools.js.map +1 -0
  404. package/dist/server.d.ts +5 -0
  405. package/dist/server.js +247 -0
  406. package/dist/server.js.map +1 -0
  407. package/dist/session.d.ts +47 -0
  408. package/dist/session.js +107 -0
  409. package/dist/session.js.map +1 -0
  410. package/dist/toolkit.d.ts +21 -0
  411. package/dist/toolkit.js +2 -0
  412. package/dist/toolkit.js.map +1 -0
  413. package/dist/tools.d.ts +299 -0
  414. package/dist/tools.js +491 -0
  415. package/dist/tools.js.map +1 -0
  416. package/guide/WORKFLOW.md +134 -0
  417. package/guide/assets/disclaimer.txt +1 -0
  418. package/guide/assets/download_data.ps1 +162 -0
  419. package/guide/assets/download_data.sh +143 -0
  420. package/guide/assets/report_template.html +597 -0
  421. package/guide/references/report_patterns.md +199 -0
  422. package/guide/references/routing.md +171 -0
  423. package/guide/references/style_guide.md +398 -0
  424. package/package.json +54 -4
  425. package/packs/acf/endpoints.json +114 -0
  426. package/packs/acf/policy.json +104 -0
  427. package/packs/acf/source.md +138 -0
  428. package/packs/actuaries/endpoints.json +82 -0
  429. package/packs/actuaries/policy.json +109 -0
  430. package/packs/actuaries/source.md +137 -0
  431. package/packs/bea/assets/download_bea.ps1 +182 -0
  432. package/packs/bea/assets/download_bea.sh +144 -0
  433. package/packs/bea/docs/bea_api.md +461 -0
  434. package/packs/bea/docs/datasets.md +157 -0
  435. package/packs/bea/docs/report_patterns.md +112 -0
  436. package/packs/bea/policy.json +170 -0
  437. package/packs/bea/source.md +152 -0
  438. package/packs/bls/assets/download_bls.ps1 +152 -0
  439. package/packs/bls/assets/download_bls.sh +167 -0
  440. package/packs/bls/docs/analysis_patterns.md +108 -0
  441. package/packs/bls/docs/bls_api.md +73 -0
  442. package/packs/bls/docs/series_ids.md +149 -0
  443. package/packs/bls/policy.json +238 -0
  444. package/packs/bls/series_ids.json +1375 -0
  445. package/packs/bls/source.md +156 -0
  446. package/packs/bts/endpoints.json +347 -0
  447. package/packs/bts/policy.json +94 -0
  448. package/packs/bts/source.md +97 -0
  449. package/packs/cdc/endpoints.json +197 -0
  450. package/packs/cdc/policy.json +83 -0
  451. package/packs/cdc/source.md +74 -0
  452. package/packs/census/endpoints.json +99 -0
  453. package/packs/census/policy.json +238 -0
  454. package/packs/census/source.md +304 -0
  455. package/packs/census-geocoder/endpoints.json +70 -0
  456. package/packs/census-geocoder/policy.json +80 -0
  457. package/packs/census-geocoder/source.md +53 -0
  458. package/packs/cfpb/endpoints.json +327 -0
  459. package/packs/cfpb/policy.json +119 -0
  460. package/packs/cfpb/source.md +96 -0
  461. package/packs/clinical-trials/endpoints.json +167 -0
  462. package/packs/clinical-trials/policy.json +84 -0
  463. package/packs/clinical-trials/source.md +82 -0
  464. package/packs/cms/endpoints.json +69 -0
  465. package/packs/cms/policy.json +96 -0
  466. package/packs/cms/source.md +112 -0
  467. package/packs/college-scorecard/endpoints.json +253 -0
  468. package/packs/college-scorecard/policy.json +106 -0
  469. package/packs/college-scorecard/source.md +92 -0
  470. package/packs/congress/endpoints.json +1701 -0
  471. package/packs/congress/policy.json +117 -0
  472. package/packs/congress/source.md +190 -0
  473. package/packs/dhs-ohss/endpoints.json +66 -0
  474. package/packs/dhs-ohss/policy.json +107 -0
  475. package/packs/dhs-ohss/source.md +120 -0
  476. package/packs/doj-news/endpoints.json +59 -0
  477. package/packs/doj-news/policy.json +96 -0
  478. package/packs/doj-news/source.md +99 -0
  479. package/packs/dol/endpoints.json +818 -0
  480. package/packs/dol/policy.json +135 -0
  481. package/packs/dol/source.md +181 -0
  482. package/packs/eia/endpoints.json +718 -0
  483. package/packs/eia/policy.json +112 -0
  484. package/packs/eia/source.md +112 -0
  485. package/packs/epa/endpoints.json +292 -0
  486. package/packs/epa/policy.json +99 -0
  487. package/packs/epa/source.md +99 -0
  488. package/packs/epa-aqs/endpoints.json +174 -0
  489. package/packs/epa-aqs/policy.json +96 -0
  490. package/packs/epa-aqs/source.md +97 -0
  491. package/packs/fbi/endpoints.json +1842 -0
  492. package/packs/fbi/policy.json +118 -0
  493. package/packs/fbi/source.md +108 -0
  494. package/packs/fda/endpoints.json +238 -0
  495. package/packs/fda/policy.json +121 -0
  496. package/packs/fda/source.md +117 -0
  497. package/packs/fdic/endpoints.json +417 -0
  498. package/packs/fdic/policy.json +83 -0
  499. package/packs/fdic/source.md +92 -0
  500. package/packs/fec/endpoints.json +708 -0
  501. package/packs/fec/policy.json +93 -0
  502. package/packs/fec/source.md +90 -0
  503. package/packs/fed-board/endpoints.json +89 -0
  504. package/packs/fed-board/policy.json +136 -0
  505. package/packs/fed-board/source.md +118 -0
  506. package/packs/federal-register/endpoints.json +143 -0
  507. package/packs/federal-register/policy.json +86 -0
  508. package/packs/federal-register/source.md +98 -0
  509. package/packs/fema/endpoints.json +646 -0
  510. package/packs/fema/policy.json +103 -0
  511. package/packs/fema/source.md +103 -0
  512. package/packs/fhfa/endpoints.json +105 -0
  513. package/packs/fhfa/policy.json +109 -0
  514. package/packs/fhfa/source.md +108 -0
  515. package/packs/fred/assets/download_fred.ps1 +122 -0
  516. package/packs/fred/assets/download_fred.sh +123 -0
  517. package/packs/fred/docs/analysis_patterns.md +124 -0
  518. package/packs/fred/docs/fred_api.md +94 -0
  519. package/packs/fred/docs/series.md +188 -0
  520. package/packs/fred/policy.json +253 -0
  521. package/packs/fred/source.md +142 -0
  522. package/packs/fsa/endpoints.json +150 -0
  523. package/packs/fsa/policy.json +99 -0
  524. package/packs/fsa/source.md +102 -0
  525. package/packs/govinfo/endpoints.json +520 -0
  526. package/packs/govinfo/policy.json +116 -0
  527. package/packs/govinfo/source.md +241 -0
  528. package/packs/gsa-calc/endpoints.json +93 -0
  529. package/packs/gsa-calc/policy.json +97 -0
  530. package/packs/gsa-calc/source.md +108 -0
  531. package/packs/hud/endpoints.json +147 -0
  532. package/packs/hud/policy.json +84 -0
  533. package/packs/hud/source.md +97 -0
  534. package/packs/ipums/assets/download_ipums.ps1 +215 -0
  535. package/packs/ipums/assets/download_ipums.sh +192 -0
  536. package/packs/ipums/docs/ipums_api.md +67 -0
  537. package/packs/ipums/docs/report_patterns.md +117 -0
  538. package/packs/ipums/docs/variables.md +284 -0
  539. package/packs/ipums/lib/fetch_text.json +112 -0
  540. package/packs/ipums/policy.json +215 -0
  541. package/packs/ipums/source.md +169 -0
  542. package/packs/irs-soi/endpoints.json +114 -0
  543. package/packs/irs-soi/policy.json +107 -0
  544. package/packs/irs-soi/source.md +124 -0
  545. package/packs/medicaid/endpoints.json +412 -0
  546. package/packs/medicaid/policy.json +108 -0
  547. package/packs/medicaid/source.md +136 -0
  548. package/packs/naep/endpoints.json +217 -0
  549. package/packs/naep/policy.json +100 -0
  550. package/packs/naep/source.md +120 -0
  551. package/packs/nhtsa/endpoints.json +132 -0
  552. package/packs/nhtsa/policy.json +93 -0
  553. package/packs/nhtsa/source.md +109 -0
  554. package/packs/nih/endpoints.json +453 -0
  555. package/packs/nih/policy.json +86 -0
  556. package/packs/nih/source.md +108 -0
  557. package/packs/noaa/endpoints.json +432 -0
  558. package/packs/noaa/policy.json +95 -0
  559. package/packs/noaa/source.md +88 -0
  560. package/packs/nrel/endpoints.json +328 -0
  561. package/packs/nrel/policy.json +96 -0
  562. package/packs/nrel/source.md +91 -0
  563. package/packs/nws/endpoints.json +272 -0
  564. package/packs/nws/policy.json +82 -0
  565. package/packs/nws/source.md +81 -0
  566. package/packs/nyfed/endpoints.json +80 -0
  567. package/packs/nyfed/policy.json +92 -0
  568. package/packs/nyfed/source.md +89 -0
  569. package/packs/omb/endpoints.json +460 -0
  570. package/packs/omb/policy.json +106 -0
  571. package/packs/omb/source.md +130 -0
  572. package/packs/open-payments/endpoints.json +298 -0
  573. package/packs/open-payments/policy.json +101 -0
  574. package/packs/open-payments/source.md +99 -0
  575. package/packs/opm/assets/download_opm.ps1 +125 -0
  576. package/packs/opm/assets/download_opm.sh +85 -0
  577. package/packs/opm/docs/fields.md +184 -0
  578. package/packs/opm/docs/opm_api.md +71 -0
  579. package/packs/opm/docs/report_patterns.md +83 -0
  580. package/packs/opm/lib/data_dictionary.csv +73 -0
  581. package/packs/opm/policy.json +142 -0
  582. package/packs/opm/source.md +137 -0
  583. package/packs/pbgc/endpoints.json +483 -0
  584. package/packs/pbgc/policy.json +94 -0
  585. package/packs/pbgc/source.md +95 -0
  586. package/packs/philfed/endpoints.json +85 -0
  587. package/packs/philfed/policy.json +94 -0
  588. package/packs/philfed/source.md +125 -0
  589. package/packs/port.json +287 -0
  590. package/packs/regulations/endpoints.json +288 -0
  591. package/packs/regulations/policy.json +86 -0
  592. package/packs/regulations/source.md +84 -0
  593. package/packs/sec/endpoints.json +237 -0
  594. package/packs/sec/policy.json +86 -0
  595. package/packs/sec/source.md +101 -0
  596. package/packs/senate-lobbying/endpoints.json +554 -0
  597. package/packs/senate-lobbying/policy.json +99 -0
  598. package/packs/senate-lobbying/source.md +100 -0
  599. package/packs/snapqc/endpoints.json +1426 -0
  600. package/packs/snapqc/policy.json +92 -0
  601. package/packs/snapqc/source.md +96 -0
  602. package/packs/treasury/catalog.json +2267 -0
  603. package/packs/treasury/docs/api_guide.md +245 -0
  604. package/packs/treasury/docs/datasets.md +56 -0
  605. package/packs/treasury/docs/report_patterns.md +126 -0
  606. package/packs/treasury/policy.json +159 -0
  607. package/packs/treasury/source.md +170 -0
  608. package/packs/treasury-yields/endpoints.json +226 -0
  609. package/packs/treasury-yields/policy.json +86 -0
  610. package/packs/treasury-yields/source.md +80 -0
  611. package/packs/usaspending/endpoints.json +1316 -0
  612. package/packs/usaspending/policy.json +109 -0
  613. package/packs/usaspending/source.md +103 -0
  614. package/packs/usda-fna/endpoints.json +427 -0
  615. package/packs/usda-fna/policy.json +133 -0
  616. package/packs/usda-fna/source.md +217 -0
  617. package/packs/usda-fooddata/endpoints.json +181 -0
  618. package/packs/usda-fooddata/policy.json +94 -0
  619. package/packs/usda-fooddata/source.md +105 -0
  620. package/packs/usda-nass/endpoints.json +324 -0
  621. package/packs/usda-nass/policy.json +95 -0
  622. package/packs/usda-nass/source.md +85 -0
  623. package/packs/usda-outlook/endpoints.json +86 -0
  624. package/packs/usda-outlook/policy.json +109 -0
  625. package/packs/usda-outlook/source.md +144 -0
  626. package/packs/usgs/endpoints.json +313 -0
  627. package/packs/usgs/policy.json +95 -0
  628. package/packs/usgs/source.md +96 -0
  629. package/packs/usitc/endpoints.json +74 -0
  630. package/packs/usitc/policy.json +111 -0
  631. package/packs/usitc/source.md +90 -0
  632. package/packs/uspto/endpoints.json +141 -0
  633. package/packs/uspto/policy.json +95 -0
  634. package/packs/uspto/source.md +101 -0
  635. package/packs/va/endpoints.json +414 -0
  636. package/packs/va/policy.json +101 -0
  637. package/packs/va/source.md +129 -0
  638. package/packs/world-bank/endpoints.json +122 -0
  639. package/packs/world-bank/policy.json +101 -0
  640. package/packs/world-bank/source.md +93 -0
@@ -0,0 +1,1033 @@
1
+ /**
2
+ * OPM Federal Workforce Data (FWD / EHRI) files: the deep pack (port of sources/opm/fetch.py).
3
+ *
4
+ * spec:
5
+ * {"description": "Separations by agency, FY2025",
6
+ * "datasets": {"employment": {"from": "2015-09", "to": "2025-09", "months": [9]},
7
+ * "separations": {"from": "2024-10", "to": "latest"}}, // or "periods": ["2025-09", "latest"]
8
+ * "columns": ["agency_code", "agency", "separation_category"], // optional; default all
9
+ * "filters": {"department_code": ["VA", "HS"]}, // optional; strings exactly as in the data
10
+ * "aggregate": {"by": ["agency_code", "agency"]}, // optional; counts + pay sums
11
+ * "pin": {"employment_202509": 4}} // optional; exact versions
12
+ * Always kept: count, snapshot_yyyymm (employment), personnel_action_effective_date_yyyymm (actions), plus
13
+ * source_file and file_yyyymm. Output in data/opm/: opm_<dataset>.parquet, opm_<dataset>.csv (up to 1,000,000
14
+ * rows), manifest.json, request.json, data_dictionary.csv, README.md and spec_used.json. Raw files are cached in
15
+ * $OPM_CACHE_DIR or <cache root>/opm-fwd for policy cache.ttl_hours.
16
+ */
17
+ import { createHash } from "node:crypto";
18
+ import fs from "node:fs";
19
+ import path from "node:path";
20
+ import { unzipSync } from "fflate";
21
+ import zlib from "node:zlib";
22
+ import { trail } from "../core/context.js";
23
+ import { cacheRoot } from "../core/env.js";
24
+ import { Client, DataError } from "../core/http.js";
25
+ import { dataDir, fingerprint } from "../core/package.js";
26
+ import { pyFloatRepr, pyJsonDumps, pyUrlencode } from "../core/py.js";
27
+ import { readCsv } from "../core/tabular.js";
28
+ import * as io from "./_opm_io.js";
29
+ import { writeParquet } from "./_opm_parquet.js";
30
+ import { closeMatches, pyRepr } from "./_opm_py.js";
31
+ const SOURCE = "opm";
32
+ export const BASE = "https://data.opm.gov/api/v1/files";
33
+ export const DATASETS = ["employment", "accessions", "separations"];
34
+ const UA = "data-analyst/1.0 opm pack (+https://data.opm.gov/get-data/api-access)";
35
+ const HINTS = {
36
+ 400: "400 Bad Request: the request is malformed or over the 25 MB request limit.",
37
+ 404: "404 Not Found: no file for that dataset/year/month/version. List the files to see what exists.",
38
+ 429: "429 Too Many Requests: OPM rate limit reached; wait a minute and retry.",
39
+ 500: "500 Internal Server Error at OPM; retry later.",
40
+ };
41
+ const EST_MB = { employment: 70, accessions: 1, separations: 2 };
42
+ const PAY_COLS = ["annualized_adjusted_basic_pay", "work_schedule_code", "pay_basis_code"];
43
+ const ALWAYS = { employment: ["count", "snapshot_yyyymm"],
44
+ accessions: ["count", "personnel_action_effective_date_yyyymm"],
45
+ separations: ["count", "personnel_action_effective_date_yyyymm"] };
46
+ export const CSV_MAX_ROWS = 1_000_000;
47
+ const WRITER = "data-analyst-mcp";
48
+ const CITATION = "U.S. Office of Personnel Management, Federal Workforce Data (FWD), Enterprise Human Resources " +
49
+ "Integration (EHRI) {datasets} files, accessed {date} through the FWD API (https://data.opm.gov/get-data/api-access).";
50
+ const cite = (datasets, date) => CITATION.replace("{datasets}", datasets).replace("{date}", date);
51
+ const pad = (n, w = 2) => String(n).padStart(w, "0");
52
+ const sha256 = (b) => createHash("sha256").update(b).digest("hex");
53
+ const specError = (m) => new DataError(m, "spec");
54
+ const commas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ",");
55
+ const isObj = (x) => !!x && typeof x === "object" && !Array.isArray(x);
56
+ function localIso(d = new Date()) {
57
+ const off = -d.getTimezoneOffset();
58
+ const sign = off >= 0 ? "+" : "-";
59
+ return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}T${pad(d.getHours())}:${pad(d.getMinutes())}:` +
60
+ `${pad(d.getSeconds())}${sign}${pad(Math.floor(Math.abs(off) / 60))}:${pad(Math.abs(off) % 60)}`;
61
+ }
62
+ const utcIso = () => new Date().toISOString().replace(/\.\d{3}Z$/, "+00:00");
63
+ // ---------------------------------------------------------------- API client
64
+ export class OpmClient {
65
+ http;
66
+ constructor(retries = 4) {
67
+ this.http = new Client(BASE, { name: "data.opm.gov", rps: 4, burst: 8, retries, timeout: 60,
68
+ deadline: Math.max(180, 60 * (retries + 1)), userAgent: UA });
69
+ }
70
+ wrap(e) {
71
+ if (e instanceof DataError) {
72
+ if (e.kind === "network" || e.kind === "fixture_missing") {
73
+ throw new DataError(`cannot reach data.opm.gov (${e.message})`, e.kind);
74
+ }
75
+ throw new DataError((e.status && HINTS[e.status]) || e.message, e.kind, e.status, e.retryAfter);
76
+ }
77
+ throw e;
78
+ }
79
+ /** File metadata: [{filename, publishDate, version, current, month, year}, ...]. */
80
+ async listFiles(dataset, params = {}) {
81
+ if (!DATASETS.includes(dataset))
82
+ throw specError(`dataset must be one of ${pyRepr([...DATASETS])}`);
83
+ const q = Object.entries(params).filter(([, v]) => v !== null && v !== undefined)
84
+ .map(([k, v]) => [k, typeof v === "boolean" ? String(v) : v]);
85
+ const url = `${BASE}/${dataset}` + (q.length ? "?" + pyUrlencode(q) : "");
86
+ let r;
87
+ try {
88
+ r = await this.http.request("GET", url, null, { okStatuses: [200, 204] });
89
+ }
90
+ catch (e) {
91
+ this.wrap(e);
92
+ }
93
+ if (r.status === 204 || !r.body.toString("utf8").trim())
94
+ return [];
95
+ return r.json();
96
+ }
97
+ downloadUrl(dataset, year, month, version) {
98
+ return `${BASE}/${dataset}/${pad(Number(year), 4)}/${pad(Number(month))}/${Math.trunc(Number(version))}/download`;
99
+ }
100
+ /** One Parquet file to `dest` (atomic). Returns bytes written. */
101
+ async download(dataset, year, month, version, dest) {
102
+ const url = this.downloadUrl(dataset, year, month, version);
103
+ const t = this.http.timeout;
104
+ this.http.timeout = Math.max(t, 300);
105
+ let n;
106
+ try {
107
+ n = (await this.http.download(url, dest)).bytes;
108
+ }
109
+ catch (e) {
110
+ this.wrap(e);
111
+ }
112
+ finally {
113
+ this.http.timeout = t;
114
+ }
115
+ const fd = fs.openSync(dest, "r");
116
+ const magic = Buffer.alloc(4);
117
+ fs.readSync(fd, magic, 0, 4, 0);
118
+ fs.closeSync(fd);
119
+ if (magic.toString("latin1") !== "PAR1") {
120
+ fs.rmSync(dest, { force: true });
121
+ throw new DataError(`${url} did not return a Parquet file`, "http");
122
+ }
123
+ return n;
124
+ }
125
+ }
126
+ /** {y*100+m: metadata of the current version}. Falls back to the highest version. */
127
+ export function currentIndex(files) {
128
+ const out = new Map();
129
+ for (const f of files) {
130
+ const k = Number(f.year) * 100 + Number(f.month);
131
+ const old = out.get(k);
132
+ if (!old || (f.current && !old.current) || (!!f.current === !!old.current && Number(f.version) > Number(old.version)))
133
+ out.set(k, f);
134
+ }
135
+ return out;
136
+ }
137
+ const key = (y, m) => y * 100 + m;
138
+ const fmtP = (p) => `${Math.floor(p / 100)}-${pad(p % 100)}`;
139
+ // ---------------------------------------------------------------- spec
140
+ export function ym(s) {
141
+ const m = /^(\d{4})-?(\d{1,2})$/.exec(String(s ?? "None").trim());
142
+ if (!m)
143
+ throw specError(`period ${pyRepr(s)}: use YYYY-MM`);
144
+ const [y, mo] = [Number(m[1]), Number(m[2])];
145
+ if (mo < 1 || mo > 12)
146
+ throw specError(`period ${pyRepr(s)}: month must be 1-12`);
147
+ return key(y, mo);
148
+ }
149
+ /** The spec with the list shorthand expanded: ["employment"] + top-level periods. */
150
+ export function normalize(spec0) {
151
+ const spec = JSON.parse(JSON.stringify(spec0 ?? {}));
152
+ if (Array.isArray(spec.datasets)) {
153
+ spec.datasets = Object.fromEntries(spec.datasets.map((d) => [d, Object.fromEntries(["from", "to", "months", "periods"].filter((k) => k in spec).map((k) => [k, spec[k]]))]));
154
+ }
155
+ return spec;
156
+ }
157
+ function dictionaryPath(pack) {
158
+ const p = path.join(pack.dir, "lib", "data_dictionary.csv");
159
+ return fs.existsSync(p) ? p : null;
160
+ }
161
+ /** {dataset: columns the data dictionary defines for it} (empty when the dictionary is missing). */
162
+ export function datasetColumns(pack) {
163
+ const p = dictionaryPath(pack);
164
+ if (!p)
165
+ return {};
166
+ const out = Object.fromEntries(DATASETS.map((d) => [d, new Set()]));
167
+ for (const r of readCsv(fs.readFileSync(p, "utf8")))
168
+ for (const d of DATASETS)
169
+ if (r[d] === "Y")
170
+ out[d].add(String(r.field));
171
+ return out;
172
+ }
173
+ function periodProblems(ds, rule) {
174
+ const out = [];
175
+ const tryYm = (v) => {
176
+ try {
177
+ ym(v);
178
+ }
179
+ catch (e) {
180
+ out.push(`${ds}: ${e.message}`);
181
+ }
182
+ };
183
+ if (!isObj(rule))
184
+ return rule === null || rule === undefined ? [] : [`${ds}: the rule must be an object such as {"periods": ["2025-09"]}`];
185
+ if (rule.periods) {
186
+ if (!Array.isArray(rule.periods))
187
+ return [`${ds}: "periods" must be a list such as ["2025-09", "latest"]`];
188
+ for (const p of rule.periods)
189
+ if (String(p).toLowerCase() !== "latest")
190
+ tryYm(p);
191
+ }
192
+ else {
193
+ if (rule.from)
194
+ tryYm(rule.from);
195
+ if (String(rule.to ?? "latest").toLowerCase() !== "latest")
196
+ tryYm(rule.to);
197
+ for (const m of rule.months || [])
198
+ if (!(Number(m) >= 1 && Number(m) <= 12 && Number.isInteger(Number(m))))
199
+ out.push(`${ds}: months must be 1-12, not ${pyRepr(m)}`);
200
+ }
201
+ return out;
202
+ }
203
+ /** What load_spec refused (fix_spec), before any request. */
204
+ export function checkSpec(spec0, pack) {
205
+ const spec = normalize(spec0);
206
+ const ds = spec.datasets;
207
+ if (!isObj(ds) || !Object.keys(ds).length) {
208
+ return ['spec needs "datasets": {"employment": {"from": "2024-09", "to": "2025-09"}, ...}'];
209
+ }
210
+ const bad = Object.keys(ds).filter((d) => !DATASETS.includes(d));
211
+ if (bad.length)
212
+ return [`unknown dataset(s) ${pyRepr(bad)}; use ${pyRepr([...DATASETS])}`];
213
+ const out = [];
214
+ for (const [col, vals] of Object.entries(spec.filters || {})) {
215
+ const nums = (Array.isArray(vals) ? vals : [vals]).filter((v) => typeof v !== "string");
216
+ if (nums.length) {
217
+ out.push(`filter ${pyRepr(col)}: values must be strings exactly as in the data (for example "08" or "0343", ` +
218
+ `not ${pyRepr(nums[0])})`);
219
+ }
220
+ }
221
+ const known = datasetColumns(pack);
222
+ if (Object.keys(known).length) {
223
+ const have = new Set(["file_yyyymm", "source_file"]);
224
+ for (const d of Object.keys(ds)) {
225
+ for (const c of known[d] || [])
226
+ have.add(c);
227
+ for (const c of ALWAYS[d] || [])
228
+ have.add(c);
229
+ }
230
+ const named = [...(spec.columns || []), ...Object.keys(spec.filters || {}), ...((spec.aggregate || {}).by || [])];
231
+ const badCols = [...new Set(named)].filter((c) => !have.has(c));
232
+ if (badCols.length) {
233
+ const sorted = [...have].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
234
+ const near = badCols.map((c) => {
235
+ const sub = sorted.filter((h) => c.split("_").some((w) => w.length > 4 && h.includes(w)));
236
+ const v = [...new Set([...closeMatches(c, sorted, 3, 0.5), ...sub])].slice(0, 4);
237
+ return `${c} -> ${v.length ? pyRepr(v) : "none"}`;
238
+ });
239
+ out.push(`unknown column(s) ${pyRepr(badCols)} for ${pyRepr(Object.keys(ds).sort())} (lib/data_dictionary.csv); close: ` +
240
+ near.join("; "));
241
+ }
242
+ }
243
+ const agg = spec.aggregate;
244
+ if (agg && spec.columns?.length) {
245
+ const auto = new Set([...Object.values(ALWAYS).flat(), ...PAY_COLS]);
246
+ const lost = spec.columns.filter((c) => !(agg.by || []).includes(c) && !auto.has(c));
247
+ if (lost.length) {
248
+ out.push(`aggregate keeps only the aggregate.by columns, so ${pyRepr(lost)} would be dropped: add them to ` +
249
+ "aggregate.by (they stay empty for datasets without them) or remove them from columns");
250
+ }
251
+ }
252
+ for (const [d, rule] of Object.entries(ds))
253
+ out.push(...periodProblems(d, rule));
254
+ return out;
255
+ }
256
+ export function specFingerprint(spec) {
257
+ const keep = Object.fromEntries(["datasets", "columns", "filters", "aggregate", "pin"].map((k) => [k, spec[k] ?? null]));
258
+ return sha256(pyJsonDumps(keep, true)).slice(0, 16);
259
+ }
260
+ /** Periods wanted by one dataset rule, and the ones that do not exist. */
261
+ export function resolvePeriods(rule, available) {
262
+ const avail = [...available].sort((a, b) => a - b);
263
+ if (!avail.length)
264
+ return [[], ["no files listed"]];
265
+ const set = new Set(avail);
266
+ const latest = avail[avail.length - 1];
267
+ if (rule.periods?.length) {
268
+ const want = rule.periods.map((p) => (String(p).toLowerCase() === "latest" ? latest : ym(p)));
269
+ const missing = want.filter((p) => !set.has(p)).map(fmtP);
270
+ // a period listed twice would load the same file twice and double its counts
271
+ return [[...new Set(want.filter((p) => set.has(p)))].sort((a, b) => a - b), missing];
272
+ }
273
+ const lo = rule.from ? ym(rule.from) : avail[0];
274
+ const hi = String(rule.to ?? "latest").toLowerCase() === "latest" ? latest : ym(rule.to);
275
+ const months = new Set((rule.months?.length ? rule.months : [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]).map(Number));
276
+ const want = avail.filter((p) => lo <= p && p <= hi && months.has(p % 100));
277
+ if (!want.length) {
278
+ return [[], [`nothing available from ${fmtP(lo)} to ${fmtP(hi)} (available: ${fmtP(avail[0])} to ${fmtP(latest)})`]];
279
+ }
280
+ const missing = [];
281
+ let [y, m] = [Math.floor(lo / 100), lo % 100];
282
+ while (key(y, m) <= hi) {
283
+ if (months.has(m) && !set.has(key(y, m)) && key(y, m) <= latest)
284
+ missing.push(`${y}-${pad(m)}`);
285
+ m += 1;
286
+ if (m > 12)
287
+ [y, m] = [y + 1, 1];
288
+ }
289
+ return [want, missing];
290
+ }
291
+ export async function buildPlan(spec, client) {
292
+ const plan = { files: [], missing: {}, est_mb: 0 };
293
+ for (const [ds, rule] of Object.entries(spec.datasets)) {
294
+ const listing = await client.listFiles(ds);
295
+ const idx = currentIndex(listing);
296
+ const byVersion = new Map(listing.map((f) => [`${Number(f.year)}-${Number(f.month)}-${Number(f.version)}`, f]));
297
+ const [periods, missing] = resolvePeriods(rule || {}, [...idx.keys()]);
298
+ if (missing.length)
299
+ plan.missing[ds] = missing;
300
+ for (const p of periods) {
301
+ const [y, m] = [Math.floor(p / 100), p % 100];
302
+ let meta = { ...idx.get(p) };
303
+ const pin = (spec.pin || {})[`${ds}_${y}${pad(m)}`];
304
+ if (pin && Number(pin) !== Number(meta.version)) {
305
+ // the pinned version's own metadata (publish date), not the current version's
306
+ meta = { ...(byVersion.get(`${y}-${m}-${Math.trunc(Number(pin))}`) ||
307
+ { version: Math.trunc(Number(pin)), filename: `${ds}_${y}${pad(m)}_${Math.trunc(Number(pin))}`, publishDate: null }) };
308
+ meta.current = false;
309
+ }
310
+ plan.files.push({ dataset: ds, year: y, month: m, version: Number(meta.version), filename: meta.filename,
311
+ publishDate: meta.publishDate ?? null, current: !!meta.current, url: client.downloadUrl(ds, y, m, meta.version) });
312
+ plan.est_mb += EST_MB[ds];
313
+ }
314
+ }
315
+ return plan;
316
+ }
317
+ /** The --list summary of a dataset: periods available (current versions). */
318
+ export async function listSummary(dataset, client = new OpmClient()) {
319
+ const idx = currentIndex(await client.listFiles(dataset));
320
+ const ks = [...idx.keys()].sort((a, b) => a - b);
321
+ const years = [...new Set(ks.map((k) => Math.floor(k / 100)))].sort((a, b) => a - b);
322
+ const monthsOf = (y) => ks.filter((k) => Math.floor(k / 100) === y).map((k) => k % 100);
323
+ const quarterly = (y) => monthsOf(y).every((m) => [3, 6, 9, 12].includes(m));
324
+ return { dataset, files: ks.length, first: ks.length ? fmtP(ks[0]) : null, latest: ks.length ? fmtP(ks[ks.length - 1]) : null,
325
+ // years with only quarter-end files (employment before 2011), and other incomplete years
326
+ quarterly_years: years.filter(quarterly),
327
+ partial_years: Object.fromEntries(years.filter((y) => monthsOf(y).length < 12 && !quarterly(y)).map((y) => [String(y), monthsOf(y)])),
328
+ latest_files: ks.slice(-3).map((k) => idx.get(k)) };
329
+ }
330
+ function emptyTable(cols) {
331
+ return { cols: [...cols], rows: 0, data: Object.fromEntries(cols.map((c) => [c, []])), kinds: {} };
332
+ }
333
+ /** pd.to_numeric(errors="coerce") of one value, and whether its text is an integer. */
334
+ function num(v) {
335
+ if (v === null)
336
+ return [null, true];
337
+ if (typeof v === "number")
338
+ return [v, Number.isInteger(v)];
339
+ const s = v.trim();
340
+ if (/^[+-]?\d+$/.test(s))
341
+ return [Number(s), true];
342
+ if (/^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/.test(s))
343
+ return [Number(s), false];
344
+ return [null, true];
345
+ }
346
+ /**
347
+ * Collapse microdata to counts per group, with what pay statistics can be summed. pay_n / pay_sum cover the pay
348
+ * universe of stats.pay_summary (full-time, per annum, pay reported and positive), so pay_sum / pay_n is comparable
349
+ * to its mean; pay_redacted counts every redacted record in the group.
350
+ */
351
+ function aggregate(d, ds, agg) {
352
+ const period = ds === "employment" ? "snapshot_yyyymm" : "personnel_action_effective_date_yyyymm";
353
+ const by = [...new Set([period, "file_yyyymm", "source_file", ...(agg.by || [])])];
354
+ const n = d.rows;
355
+ const cnt = d.data.count.map((v) => num(v));
356
+ const nKind = cnt.every(([, i]) => i) ? "int" : "float";
357
+ const nv = cnt.map(([x]) => (x === null ? 1 : x));
358
+ const payCol = d.data.annualized_adjusted_basic_pay;
359
+ let red, payn, pays, payKind = "int";
360
+ if (payCol) {
361
+ const parsed = payCol.map((v) => num(v));
362
+ payKind = parsed.every(([, i]) => i) ? "int" : "float";
363
+ const pay = parsed.map(([x]) => (x !== null && x > 0 ? x : null));
364
+ for (const [col, val] of [["work_schedule_code", "F"], ["pay_basis_code", "PA"]]) {
365
+ const c = d.data[col];
366
+ if (c)
367
+ for (let i = 0; i < n; i++)
368
+ if (c[i] !== val)
369
+ pay[i] = null;
370
+ }
371
+ red = payCol.map((v) => (v === "REDACTED" ? 1 : 0));
372
+ payn = pay.map((x) => (x === null ? 0 : 1));
373
+ pays = pay.map((x) => x ?? 0);
374
+ }
375
+ else {
376
+ red = payn = pays = new Array(n).fill(0);
377
+ }
378
+ const groups = new Map();
379
+ for (let i = 0; i < n; i++) {
380
+ const k = by.map((c) => (d.data[c][i] ?? null));
381
+ const id = JSON.stringify(k);
382
+ let g = groups.get(id);
383
+ if (!g)
384
+ groups.set(id, (g = { key: k, h: 0, hc: 0, r: 0, pn: 0, ps: 0, pc: 0 }));
385
+ // floats: Kahan-compensated sums, as pandas' groupby sum
386
+ if (nKind === "float") {
387
+ const y = nv[i] - g.hc, t = g.h + y;
388
+ g.hc = t - g.h - y;
389
+ g.h = t;
390
+ }
391
+ else
392
+ g.h += nv[i];
393
+ g.r += red[i];
394
+ g.pn += payn[i];
395
+ if (payKind === "float") {
396
+ const y = pays[i] - g.pc, t = g.ps + y;
397
+ g.pc = t - g.ps - y;
398
+ g.ps = t;
399
+ }
400
+ else
401
+ g.ps += pays[i];
402
+ }
403
+ const cmp = (a, b) => {
404
+ for (let i = 0; i < a.length; i++) {
405
+ const [x, y] = [a[i], b[i]];
406
+ if (x === y)
407
+ continue;
408
+ if (x === null)
409
+ return 1;
410
+ if (y === null)
411
+ return -1;
412
+ return x < y ? -1 : 1;
413
+ }
414
+ return 0;
415
+ };
416
+ const sorted = [...groups.values()].sort((a, b) => cmp(a.key, b.key));
417
+ const cols = [...by, "headcount", "pay_redacted", "pay_n", "pay_sum"];
418
+ const t = emptyTable(cols);
419
+ for (const g of sorted) {
420
+ by.forEach((c, i) => t.data[c].push(g.key[i]));
421
+ t.data.headcount.push(g.h);
422
+ t.data.pay_redacted.push(g.r);
423
+ t.data.pay_n.push(g.pn);
424
+ t.data.pay_sum.push(g.ps);
425
+ }
426
+ t.rows = sorted.length;
427
+ t.kinds = { headcount: nKind, pay_redacted: "int", pay_n: "int", pay_sum: payKind };
428
+ return t;
429
+ }
430
+ /** Read one raw file (Parquet, TXT/CSV or JSON) in chunks, keep columns and rows. */
431
+ async function processFile(p, ds, meta, spec) {
432
+ const haveList = await io.columnsOf(p);
433
+ const have = new Set(haveList);
434
+ const wanted = spec.columns?.length ? spec.columns : undefined;
435
+ const filters = spec.filters || {};
436
+ const agg = spec.aggregate;
437
+ let cols;
438
+ if (wanted) {
439
+ cols = [...new Set([...ALWAYS[ds], ...wanted, ...Object.keys(filters), ...((agg || {}).by || []),
440
+ ...(agg ? PAY_COLS : [])])].filter((c) => c !== "file_yyyymm" && c !== "source_file");
441
+ }
442
+ else
443
+ cols = [...haveList];
444
+ const readCols = cols.filter((c) => have.has(c));
445
+ const missingCols = cols.filter((c) => !have.has(c));
446
+ const fvals = Object.entries(filters).map(([c, v]) => [c, new Set((Array.isArray(v) ? v : [v]).map(String))]);
447
+ let rowsRaw = 0;
448
+ const t = emptyTable([...cols, "file_yyyymm", "source_file"]);
449
+ for await (const f of io.iterFrames(p, readCols)) {
450
+ rowsRaw += f.rows;
451
+ const active = fvals.filter(([c]) => c in f.data);
452
+ for (let i = 0; i < f.rows; i++) {
453
+ if (active.some(([c, vs]) => { const v = f.data[c][i]; return v === null || !vs.has(v); }))
454
+ continue;
455
+ for (const c of readCols)
456
+ t.data[c].push(f.data[c]?.[i] ?? null);
457
+ t.rows++;
458
+ }
459
+ }
460
+ const fileYm = `${meta.year}${pad(Number(meta.month))}`;
461
+ for (const c of missingCols)
462
+ t.data[c] = new Array(t.rows).fill(null);
463
+ t.data.file_yyyymm = new Array(t.rows).fill(fileYm);
464
+ t.data.source_file = new Array(t.rows).fill(meta.filename);
465
+ const out = agg ? aggregate(t, ds, agg) : t;
466
+ return [out, { rows_raw: rowsRaw, rows_kept: out.rows, missing_columns: missingCols,
467
+ filters_not_applied: Object.keys(filters).filter((c) => !have.has(c)), source_format: await io.fileKind(p) }];
468
+ }
469
+ function concat(parts) {
470
+ const cols = [];
471
+ for (const p of parts)
472
+ for (const c of p.cols)
473
+ if (!cols.includes(c))
474
+ cols.push(c);
475
+ const t = emptyTable(cols);
476
+ for (const p of parts) {
477
+ for (const c of cols) {
478
+ const src = p.data[c];
479
+ const dst = t.data[c];
480
+ if (src)
481
+ for (const v of src)
482
+ dst.push(v);
483
+ else
484
+ for (let i = 0; i < p.rows; i++)
485
+ dst.push(null);
486
+ }
487
+ t.rows += p.rows;
488
+ for (const [c, k] of Object.entries(p.kinds))
489
+ if (t.kinds[c] !== "float")
490
+ t.kinds[c] = k;
491
+ }
492
+ return t;
493
+ }
494
+ /** A cell as pandas writes it: missing empty, floats as repr, minimal quoting. */
495
+ function cell(v, kind) {
496
+ if (v === null || v === undefined)
497
+ return "";
498
+ if (typeof v === "number")
499
+ return kind === "float" ? pyFloatRepr(v) : String(v);
500
+ return /[,"\r\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v;
501
+ }
502
+ function writeTableCsv(file, t, gz = false) {
503
+ const lines = [t.cols.map((c) => cell(c)).join(",")];
504
+ const chunks = [];
505
+ const fd = gz ? null : fs.openSync(file, "w");
506
+ const flush = () => {
507
+ const b = Buffer.from(lines.join("\r\n") + "\r\n", "utf8");
508
+ lines.length = 0;
509
+ if (fd !== null)
510
+ fs.writeSync(fd, b);
511
+ else
512
+ chunks.push(b);
513
+ };
514
+ try {
515
+ for (let i = 0; i < t.rows; i++) {
516
+ lines.push(t.cols.map((c) => cell(t.data[c][i], t.kinds[c])).join(","));
517
+ if (lines.length >= 20000)
518
+ flush();
519
+ }
520
+ flush();
521
+ }
522
+ finally {
523
+ if (fd !== null)
524
+ fs.closeSync(fd);
525
+ }
526
+ if (gz)
527
+ fs.writeFileSync(file, zlib.gzipSync(Buffer.concat(chunks)));
528
+ }
529
+ function writeTableParquet(file, t) {
530
+ const cols = t.cols.map((c) => ({ name: c, values: t.data[c],
531
+ kind: t.kinds[c] === "float" ? "double" : t.kinds[c] === "int" ? "int64" : "string" }));
532
+ writeParquet(file, cols, t.rows, WRITER);
533
+ }
534
+ function fileSha(p) {
535
+ const h = createHash("sha256");
536
+ const fd = fs.openSync(p, "r");
537
+ const buf = Buffer.alloc(1 << 20);
538
+ try {
539
+ for (;;) {
540
+ const n = fs.readSync(fd, buf, 0, buf.length, null);
541
+ if (!n)
542
+ break;
543
+ h.update(buf.subarray(0, n));
544
+ }
545
+ }
546
+ finally {
547
+ fs.closeSync(fd);
548
+ }
549
+ return h.digest("hex");
550
+ }
551
+ function writeRequestJson(d, spec, route, acquired, tables, files) {
552
+ const retrieved = utcIso();
553
+ const vintage = [...new Set(files.map((f) => `${f.dataset}_${f.year}${pad(Number(f.month))}_v${f.version ?? "None"}`))]
554
+ .sort().join(", ").slice(0, 400);
555
+ const reqs = trail().filter((s) => s.url.includes("data.opm.gov"));
556
+ const req = { source: SOURCE, spec, fingerprint: fingerprint(spec), route, acquired, retrieved, vintage,
557
+ requests: reqs.length ? reqs : files.map((f) => ({ method: "GET", url: f.url, sha256: f.sha256 })),
558
+ tables, problems: [], notes: [], policy: { redistribution: "allowed", cache: "persistent" } };
559
+ fs.writeFileSync(path.join(d, "request.json"), JSON.stringify(req, null, 2));
560
+ return retrieved;
561
+ }
562
+ function writeReadme(d, M) {
563
+ const ds = [...new Set(M.files.map((f) => f.dataset))].sort().join(", ");
564
+ const date = M.downloaded.slice(0, 10);
565
+ const late = M.files.filter((f) => f.current === false);
566
+ const miss = [...new Set(M.files.flatMap((f) => f.missing_columns || []))].sort();
567
+ const lines = [`# OPM Federal Workforce Data: ${M.spec.description || ds}`, "",
568
+ `Retrieved ${M.downloaded} by route \`${M.route}\` (${M.acquired || ""}) from ${M.api_base}.`, "",
569
+ "## Citation", "", cite(ds, date), "",
570
+ "## Files", "", "| Dataset | Period | Version | Published | Rows kept | SHA-256 (raw) |", "|---|---|---|---|---|---|"];
571
+ for (const f of M.files) {
572
+ lines.push(`| ${f.dataset} | ${f.year}-${pad(Number(f.month))} | ${f.version ?? "None"}${f.current !== false ? "" : " (pinned)"} | ` +
573
+ `${(f.publishDate || "").slice(0, 10)} | ${f.rows_kept !== undefined ? commas(f.rows_kept) : ""} | \`${(f.sha256 || "").slice(0, 16)}…\` |`);
574
+ }
575
+ lines.push("", "## Things to know before analyzing", "", "- Pay (`annualized_adjusted_basic_pay`), duty-station location, CBSA/CSA, locality pay area, " +
576
+ "bargaining unit and personnel office are `REDACTED` for all Department of Defense employees and for " +
577
+ "security/sensitive agencies and occupations (about 45 to 48 percent of employment records). " +
578
+ "Never average pay over the unredacted records and call it a governmentwide figure.", "- Coverage: EHRI excludes USPS, the intelligence agencies (CIA, NSA, DIA, NGA, ODNI), the White House " +
579
+ "Office, Foreign Service at State, TVA, the Fed Board of Governors, PHS Commissioned Corps, " +
580
+ "nonappropriated-fund staff, foreign nationals overseas, and most of the legislative and judicial " +
581
+ "branches (the Government Publishing Office, `LP`, and some judicial-branch records, `JL`, do appear: " +
582
+ "drop them by `agency_code` for an executive-branch count).", "- Accession and separation files hold actions processed in the file month; some are effective in " +
583
+ "earlier months (`personnel_action_effective_date_yyyymm`). Recent months gain late actions in later files.", "- OPM revises files: `manifest.json` records each version; put its versions in the spec's `pin` " +
584
+ "(`{\"<dataset>_<YYYYMM>\": <version>}`) to re-fetch exactly these.", "- Agency names change (e.g. Department of Defense / Department of War); group by `*_code` columns.", "- No sex, race, ethnicity, disability or individual identifiers are in the public files.");
585
+ if (miss.length)
586
+ lines.push(`- Columns absent from some (older) files and filled with nulls: ${miss.join(", ")}.`);
587
+ if (late.length)
588
+ lines.push(`- ${late.length} file(s) are pinned to a non-current version.`);
589
+ lines.push("", "## Use and redistribution", "", "Rule: **allowed**. OPM publishes these files as public data under its Public Data Release Policy; " +
590
+ "the filtered files may ship with a report and its replication package. Fields OPM withholds for " +
591
+ "privacy or security are published as `REDACTED`: disclose the coverage of every pay or location " +
592
+ "figure (`exhibits/coverage.csv`).");
593
+ fs.writeFileSync(path.join(d, "README.md"), lines.join("\n") + "\n");
594
+ }
595
+ function packageData(c, spec, tables, infos, route, acquired) {
596
+ const d = dataDir(c.out, SOURCE);
597
+ fs.mkdirSync(d, { recursive: true });
598
+ for (const f of fs.readdirSync(d))
599
+ if (/^opm_\w+\.(parquet|csv|csv\.gz)$/.test(f))
600
+ fs.rmSync(path.join(d, f));
601
+ const outputs = [], dsInfo = {}, tablesMeta = [];
602
+ for (const [ds, parts] of tables) {
603
+ if (!parts.length)
604
+ continue;
605
+ const t = concat(parts);
606
+ const info = { rows: t.rows, parquet: null, csv: null };
607
+ writeTableParquet(path.join(d, `opm_${ds}.parquet`), t);
608
+ outputs.push(`opm_${ds}.parquet`);
609
+ info.parquet = `opm_${ds}.parquet`;
610
+ if (t.rows <= CSV_MAX_ROWS) {
611
+ writeTableCsv(path.join(d, `opm_${ds}.csv`), t);
612
+ outputs.push(`opm_${ds}.csv`);
613
+ info.csv = `opm_${ds}.csv`;
614
+ }
615
+ dsInfo[ds] = info;
616
+ tablesMeta.push({ name: `opm_${ds}`, file: info.csv || info.parquet, rows: t.rows, columns: t.cols });
617
+ }
618
+ const dict = dictionaryPath(c.pack);
619
+ if (dict)
620
+ fs.copyFileSync(dict, path.join(d, "data_dictionary.csv"));
621
+ const M = { source: "U.S. Office of Personnel Management, Federal Workforce Data (EHRI)", api_base: BASE, route, acquired,
622
+ downloaded: localIso(), spec_fingerprint: specFingerprint(spec),
623
+ spec: Object.fromEntries(["description", "datasets", "columns", "filters", "aggregate", "pin"].map((k) => [k, spec[k] ?? null])),
624
+ columns: spec.columns || [], filters: spec.filters || {}, aggregate: spec.aggregate ?? null,
625
+ rows: Object.values(dsInfo).reduce((s, i) => s + i.rows, 0), datasets: dsInfo, outputs, files: infos,
626
+ parquet_writer: WRITER };
627
+ fs.writeFileSync(path.join(d, "manifest.json"), JSON.stringify(M, null, 2));
628
+ fs.writeFileSync(path.join(d, "spec_used.json"), JSON.stringify(spec, null, 2));
629
+ const retrieved = writeRequestJson(d, spec, route, acquired, tablesMeta, infos);
630
+ writeReadme(d, M);
631
+ return { M, tablesMeta, retrieved, dir: d };
632
+ }
633
+ function rawCacheRoot() {
634
+ return process.env.OPM_CACHE_DIR || path.join(cacheRoot(), "opm-fwd");
635
+ }
636
+ function cachedFresh(p, pack) {
637
+ try {
638
+ const ttl = Number(pack.policy.cache?.ttl_hours) || 720;
639
+ return fs.existsSync(p) && Date.now() - fs.statSync(p).mtimeMs < ttl * 3600 * 1000;
640
+ }
641
+ catch {
642
+ return false;
643
+ }
644
+ }
645
+ async function runDownload(c, spec, plan, o) {
646
+ const client = new OpmClient();
647
+ const cdir = rawCacheRoot();
648
+ if (!o.local)
649
+ fs.mkdirSync(cdir, { recursive: true });
650
+ const tables = new Map();
651
+ const infos = [];
652
+ for (const f of plan.files) {
653
+ const raw = o.local?.[f.filename] || path.join(cdir, f.filename + ".parquet");
654
+ const fresh = o.local !== undefined || cachedFresh(raw, c.pack);
655
+ if (!fs.existsSync(raw) || !fresh) {
656
+ if (o.local)
657
+ throw new DataError(`${f.filename} was not among the ingested files`, "not_found");
658
+ await client.download(f.dataset, f.year, f.month, f.version, raw);
659
+ }
660
+ const [t, info] = await processFile(raw, f.dataset, f, spec);
661
+ if (!tables.has(f.dataset))
662
+ tables.set(f.dataset, []);
663
+ tables.get(f.dataset).push(t);
664
+ infos.push({ ...f, ...info, bytes: fs.statSync(raw).size, sha256: fileSha(raw) });
665
+ }
666
+ const applies = datasetColumns(c.pack);
667
+ for (const f of infos) {
668
+ f.missing_columns = f.missing_columns.filter((x) => !Object.keys(applies).length || applies[f.dataset]?.has(x));
669
+ }
670
+ const { M, tablesMeta, retrieved, dir } = packageData(c, spec, tables, infos, o.route, o.acquired);
671
+ const warn = [];
672
+ for (const f of infos) {
673
+ if (f.missing_columns.length)
674
+ warn.push(`${f.filename}: columns not in this file (nulls): ${f.missing_columns.slice(0, 8).join(", ")}`);
675
+ if (f.filters_not_applied.length) {
676
+ warn.push(`${f.filename}: filter column(s) ${pyRepr(f.filters_not_applied)} absent, so rows were NOT filtered`);
677
+ }
678
+ else if (spec.filters && Object.keys(spec.filters).length && f.rows_raw && !f.rows_kept) {
679
+ warn.push(`${f.filename}: the filters kept 0 of ${f.rows_raw} rows; check the codes in spec.json`);
680
+ }
681
+ }
682
+ const empty = infos.length > 0 && !!spec.filters && Object.keys(spec.filters).length > 0 && !infos.some((f) => f.rows_kept);
683
+ if (empty) {
684
+ throw new DataError("the filters kept no rows in any file: check the codes in spec.json (data_dictionary.csv " +
685
+ "has sample values; filter values are strings exactly as in the data)", "spec");
686
+ }
687
+ const vintage = [...new Set(infos.map((f) => `${f.dataset}_${f.year}${pad(Number(f.month))}_v${f.version ?? "None"}`))]
688
+ .sort().join(", ").slice(0, 400);
689
+ return { data_dir: dir, tables: tablesMeta, notes: [...(o.warnings || []), ...warn].slice(0, 30), problems: [], retrieved,
690
+ citation: cite(Object.keys(M.datasets).sort().join(", "), M.downloaded.slice(0, 10)), vintage, route: o.route,
691
+ files: [path.join(dir, "manifest.json"), path.join(dir, "spec_used.json"), ...M.outputs.filter((x) => x.endsWith(".parquet"))
692
+ .map((x) => path.join(dir, x))] };
693
+ }
694
+ function existing(out, spec) {
695
+ const d = dataDir(out, SOURCE);
696
+ const mp = path.join(d, "manifest.json");
697
+ if (!fs.existsSync(mp))
698
+ return null;
699
+ let M;
700
+ try {
701
+ M = JSON.parse(fs.readFileSync(mp, "utf8"));
702
+ }
703
+ catch {
704
+ return null;
705
+ }
706
+ const filesOk = (M.outputs || []).every((f) => fs.existsSync(path.join(d, f)));
707
+ return { matches: M.spec_fingerprint === specFingerprint(spec) && filesOk, manifest: M };
708
+ }
709
+ function csvHeader(file) {
710
+ if (!file.endsWith(".csv") || !fs.existsSync(file))
711
+ return [];
712
+ const fd = fs.openSync(file, "r");
713
+ const b = Buffer.alloc(65536);
714
+ const n = fs.readSync(fd, b, 0, b.length, 0);
715
+ fs.closeSync(fd);
716
+ const line = b.subarray(0, n).toString("utf8").split(/\r?\n/)[0];
717
+ const row = readCsv(line + "\r\nx")[0];
718
+ return row ? Object.keys(row) : [];
719
+ }
720
+ function fromManifest(out, M, route, notes) {
721
+ const d = dataDir(out, SOURCE);
722
+ let req = {};
723
+ try {
724
+ req = JSON.parse(fs.readFileSync(path.join(d, "request.json"), "utf8"));
725
+ }
726
+ catch {
727
+ /* request.json is optional here */
728
+ }
729
+ const tables = Object.entries(M.datasets || {}).map(([ds, v]) => {
730
+ const file = (v || {}).csv || (v || {}).parquet;
731
+ return { name: `opm_${ds}`, file, rows: (v || {}).rows ?? 0, columns: csvHeader(path.join(d, file)) };
732
+ });
733
+ return { data_dir: d, tables, notes, problems: [], retrieved: req.retrieved || M.downloaded, vintage: req.vintage || "",
734
+ citation: cite(Object.keys(M.datasets || {}).sort().join(", "), String(M.downloaded || "").slice(0, 10)), route,
735
+ files: [path.join(d, "manifest.json"), path.join(d, "spec_used.json"), ...(M.outputs || [])
736
+ .filter((x) => x.endsWith(".parquet")).map((x) => path.join(d, x))] };
737
+ }
738
+ async function fetch(ctx) {
739
+ const spec = normalize(ctx.spec);
740
+ const probs = checkSpec(spec, ctx.pack);
741
+ if (probs.length)
742
+ throw specError(probs.join("; "));
743
+ const c = { pack: ctx.pack, out: ctx.workspace };
744
+ const ex = existing(c.out, spec);
745
+ if (ex && ex.matches && !ctx.force) {
746
+ const M = ex.manifest;
747
+ return fromManifest(c.out, M, "reuse_existing", [`${dataDir(c.out, SOURCE)} already holds this request ` +
748
+ `(${commas(M.rows || 0)} rows, downloaded ${M.downloaded}). Use force: true to download again (for example after ` +
749
+ "OPM publishes revised versions)."]);
750
+ }
751
+ const plan = await buildPlan(spec, new OpmClient());
752
+ if (Object.keys(plan.missing).length && !plan.files.length) {
753
+ throw specError(`Requested periods do not exist: ${pyRepr(plan.missing)}. request_schema lists the periods each ` +
754
+ "dataset has.");
755
+ }
756
+ const warnings = Object.keys(plan.missing).length ? [`Not available and skipped: ${pyRepr(plan.missing)}.`] : [];
757
+ return runDownload(c, spec, plan, { route: "api", acquired: "FWD API download", warnings });
758
+ }
759
+ // ---------------------------------------------------------------- ingest (hand-downloaded files)
760
+ const RAW_EXT = [".parquet", ".txt", ".csv", ".tsv", ".dat", ".json", ".jsonl", ".gz", ".zip"];
761
+ function prebuilt(c, spec, src) {
762
+ const dst = dataDir(c.out, SOURCE);
763
+ if (path.resolve(src) !== path.resolve(dst))
764
+ fs.cpSync(src, dst, { recursive: true });
765
+ const M = JSON.parse(fs.readFileSync(path.join(dst, "manifest.json"), "utf8"));
766
+ [M.route, M.acquired] = ["manual_download", "manual_prebuilt"];
767
+ fs.writeFileSync(path.join(dst, "manifest.json"), JSON.stringify(M, null, 2));
768
+ const tables = Object.entries(M.datasets || {}).map(([ds, v]) => ({ name: `opm_${ds}`, file: (v || {}).csv || (v || {}).parquet, rows: (v || {}).rows }));
769
+ writeRequestJson(dst, spec, "manual_download", "manual_prebuilt", tables, M.files || []);
770
+ const warn = M.spec_fingerprint === specFingerprint(spec) ? [] : ["the uploaded data/ folder was built from a different spec.json"];
771
+ return fromManifest(c.out, M, "manual_download", warn);
772
+ }
773
+ function walk(d) {
774
+ const out = [];
775
+ const rec = (dir) => {
776
+ const names = fs.readdirSync(dir);
777
+ out.push([dir, ...names.filter((n) => fs.statSync(path.join(dir, n)).isFile())]);
778
+ for (const n of names)
779
+ if (fs.statSync(path.join(dir, n)).isDirectory())
780
+ rec(path.join(dir, n));
781
+ };
782
+ rec(d);
783
+ return out;
784
+ }
785
+ async function ingest(ctx) {
786
+ const spec = normalize(ctx.spec);
787
+ const probs = checkSpec(spec, ctx.pack);
788
+ if (probs.length)
789
+ throw specError(probs.join("; "));
790
+ const c = { pack: ctx.pack, out: ctx.workspace };
791
+ const found = [];
792
+ const tmpRoot = path.join(dataDir(c.out, SOURCE), "_ingest");
793
+ fs.rmSync(tmpRoot, { recursive: true, force: true }); // never re-read files from an earlier upload
794
+ try {
795
+ const scan = (d) => {
796
+ for (const [root, ...files] of walk(d)) {
797
+ if (files.includes("manifest.json") && files.some((f) => /^opm_\w+\.(parquet|csv|csv\.gz)$/.test(f)))
798
+ return prebuilt(c, spec, root);
799
+ found.push(...files.filter((f) => RAW_EXT.some((e) => f.toLowerCase().endsWith(e)) && f !== "data_dictionary.csv")
800
+ .map((f) => path.join(root, f)));
801
+ }
802
+ return null;
803
+ };
804
+ for (const [i, p] of ctx.files.entries()) {
805
+ if (fs.statSync(p).isDirectory()) {
806
+ const r = scan(p);
807
+ if (r)
808
+ return r;
809
+ }
810
+ else if (p.toLowerCase().endsWith(".zip") && Object.keys(unzipSync(fs.readFileSync(p))).filter((n) => !n.endsWith("/")).length > 1) {
811
+ const dst = path.join(tmpRoot, `zip${i}`); // one folder per ZIP, alongside the other arguments
812
+ for (const [n, data] of Object.entries(unzipSync(fs.readFileSync(p)))) {
813
+ if (n.endsWith("/"))
814
+ continue;
815
+ const t = path.resolve(dst, n);
816
+ if (!t.startsWith(path.resolve(dst) + path.sep))
817
+ continue; // no paths outside the folder
818
+ fs.mkdirSync(path.dirname(t), { recursive: true });
819
+ fs.writeFileSync(t, data);
820
+ }
821
+ const r = scan(dst);
822
+ if (r)
823
+ return r;
824
+ }
825
+ else
826
+ found.push(p);
827
+ }
828
+ // a single Parquet file inside .zip or .gz: unpack it, since the Parquet reader needs the plain file
829
+ for (const [k, f] of found.entries()) {
830
+ if (/\.(zip|gz)$/i.test(f) && (await io.fileKind(f)) === "parquet") {
831
+ const member = io.innerName(f);
832
+ const dst = path.join(tmpRoot, `pq${k}`);
833
+ fs.mkdirSync(dst, { recursive: true });
834
+ const target = path.join(dst, path.basename(member));
835
+ fs.writeFileSync(target, f.toLowerCase().endsWith(".zip") ? unzipSync(fs.readFileSync(f))[member] : zlib.gunzipSync(fs.readFileSync(f)));
836
+ found[k] = target;
837
+ }
838
+ }
839
+ if (!found.length) {
840
+ throw new DataError("no data files found (expected .parquet, .txt, .csv or .json, optionally zipped, or a data/ " +
841
+ "folder built by fetch)", "not_found");
842
+ }
843
+ const files = [], local = {}, problems = [], warn = [];
844
+ const best = new Map(); // one file per dataset and month
845
+ for (const p of found) {
846
+ const who = await io.identify(p);
847
+ if (!who.dataset || !who.year) {
848
+ problems.push(`${path.basename(p)}: cannot tell the dataset and month; rename it like separations_202608_1.txt`);
849
+ continue;
850
+ }
851
+ const k = `${who.dataset}\t${who.year}\t${who.month}`;
852
+ const old = best.get(k);
853
+ if (old) {
854
+ // two copies of one month (two versions, or a website file and an API file) would double the counts: keep
855
+ // the highest version; an API file (versioned) beats a website file
856
+ const [keep, drop] = (who.version || 0) > (old[1].version || 0) ? [[p, who], old] : [old, [p, who]];
857
+ warn.push(`${path.basename(drop[0])} and ${path.basename(keep[0])} are both ${who.dataset} ${who.year}-${pad(who.month)}; ` +
858
+ `kept ${path.basename(keep[0])}`);
859
+ best.set(k, keep);
860
+ }
861
+ else
862
+ best.set(k, [p, who]);
863
+ }
864
+ const keys = [...best.keys()].sort((a, b) => {
865
+ const [x, y] = [a.split("\t"), b.split("\t")];
866
+ return x[0] !== y[0] ? (x[0] < y[0] ? -1 : 1) : Number(x[1]) - Number(y[1]) || Number(x[2]) - Number(y[2]);
867
+ });
868
+ for (const k of keys) {
869
+ const [p, who] = best.get(k);
870
+ const [ds, y, m] = [who.dataset, who.year, who.month];
871
+ const name = `${ds}_${y}${pad(m)}_${who.version ?? "web"}`;
872
+ if (who.how === "content") {
873
+ warn.push(`${path.basename(p)}: dataset and month inferred from the content (${ds} ${y}-${pad(m)}); check that ` +
874
+ "this is the month OPM labels the file with");
875
+ }
876
+ const rule = spec.datasets[ds];
877
+ if (rule === undefined) {
878
+ warn.push(`${path.basename(p)}: ${ds} is not in spec.json; skipped`);
879
+ continue;
880
+ }
881
+ // "latest" means the latest month among the uploaded files of this dataset
882
+ const avail = [...best.values()].filter(([, w]) => w.dataset === ds).map(([, w]) => key(w.year, w.month));
883
+ const [wanted] = resolvePeriods(rule || {}, avail);
884
+ if (!wanted.includes(key(y, m))) {
885
+ warn.push(`${path.basename(p)}: ${y}-${pad(m)} is outside the periods in spec.json; skipped`);
886
+ continue;
887
+ }
888
+ local[name] = p;
889
+ files.push({ dataset: ds, year: y, month: m, version: who.version, filename: name, publishDate: null,
890
+ current: null, identified_by: who.how,
891
+ url: who.version ? `${BASE}/${ds}/${y}/${pad(m)}/${who.version}/download` : "https://data.opm.gov/get-data/data-downloads" });
892
+ }
893
+ if (problems.length && !files.length)
894
+ throw new DataError(problems.join("; "), "not_found");
895
+ files.sort((a, b) => (a.dataset !== b.dataset ? (a.dataset < b.dataset ? -1 : 1) : key(a.year, a.month) - key(b.year, b.month)));
896
+ return await runDownload(c, spec, { files, missing: {}, est_mb: 0 }, { route: "manual_download", local,
897
+ acquired: "manual_ingest (files downloaded by hand by the user and uploaded)", warnings: [...warn, ...problems] });
898
+ }
899
+ finally {
900
+ fs.rmSync(tmpRoot, { recursive: true, force: true });
901
+ }
902
+ }
903
+ // ---------------------------------------------------------------- manual route
904
+ /** The request as the download scripts read it (tab-separated): periods / range / pin rows per dataset. */
905
+ export function scriptRules(spec0) {
906
+ const spec = normalize(spec0);
907
+ const rows = [];
908
+ const ymText = (x) => {
909
+ const p = ym(x);
910
+ return `${pad(Math.floor(p / 100), 4)}${pad(p % 100)}`;
911
+ };
912
+ for (const [ds, rule0] of Object.entries(spec.datasets)) {
913
+ const rule = rule0 || {};
914
+ if (rule.periods?.length) {
915
+ rows.push(`${ds}\tperiods\t${rule.periods.map((x) => (String(x).toLowerCase() === "latest" ? "latest" : ymText(x))).join(",")}`);
916
+ }
917
+ else {
918
+ const lo = rule.from ? ymText(rule.from) : "first";
919
+ const hi = String(rule.to ?? "latest").toLowerCase() === "latest" ? "latest" : ymText(rule.to);
920
+ const months = (rule.months?.length ? rule.months : [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]).map((x) => String(Math.trunc(Number(x)))).join(",");
921
+ rows.push(`${ds}\trange\t${lo}\t${hi}\t${months}`);
922
+ }
923
+ }
924
+ for (const [k, version] of Object.entries(spec.pin || {})) {
925
+ const m = /^([a-z]+)_(\d{6})$/.exec(k);
926
+ if (m)
927
+ rows.push(`${m[1]}\tpin\t${m[2]}\t${Math.trunc(Number(version))}`);
928
+ }
929
+ return rows;
930
+ }
931
+ /** download_opm.sh / .ps1 with the request built in: they list OPM's current files, pick the requested months
932
+ * and download them (curl / Windows PowerShell, no key). */
933
+ export function writeDownloadScripts(d, spec, pack) {
934
+ const rules = scriptRules(spec).join("\n");
935
+ const out = {};
936
+ for (const [kind, name] of [["bash", "download_opm.sh"], ["powershell", "download_opm.ps1"]]) {
937
+ const src = path.join(pack.dir, "assets", name);
938
+ if (!fs.existsSync(src))
939
+ continue;
940
+ let text = fs.readFileSync(src, "ascii").replace(/\r\n/g, "\n").replace("@RULES@", rules);
941
+ if (kind === "powershell")
942
+ text = text.replace(/\n/g, "\r\n");
943
+ const p = path.join(d, name);
944
+ fs.mkdirSync(d, { recursive: true });
945
+ fs.writeFileSync(p, text, { encoding: "ascii", mode: kind === "bash" ? 0o755 : 0o644 });
946
+ out[kind] = p;
947
+ }
948
+ return out;
949
+ }
950
+ /** Items for the manual bundle: only months whose file is known without listing OPM's files (a pinned version). */
951
+ function manualItems(spec0) {
952
+ const spec = normalize(spec0);
953
+ const client = new OpmClient(0);
954
+ const out = [];
955
+ for (const [k, v] of Object.entries(spec.pin || {})) {
956
+ const m = /^([a-z]+)_(\d{4})(\d{2})$/.exec(k);
957
+ if (!m || !(m[1] in (spec.datasets || {})))
958
+ continue;
959
+ const [ds, y, mo, ver] = [m[1], Number(m[2]), Number(m[3]), Math.trunc(Number(v))];
960
+ out.push({ name: `${ds}_${y}${pad(mo)}_${ver}`, url: client.downloadUrl(ds, y, mo, ver), filename: `${ds}_${y}${pad(mo)}_${ver}.parquet`,
961
+ needs_key: false, method: "GET", note: `OPM ${ds} file for ${y}-${pad(mo)}, version ${ver} (pinned)` });
962
+ }
963
+ return out;
964
+ }
965
+ // ---------------------------------------------------------------- schema, search
966
+ const SUMMARY = {
967
+ employment: "Monthly status snapshot: every employee on board on the last day of the month (~2 million rows, 26-75 MB a file); quarterly 2005 to March 2011, monthly from June 2011.",
968
+ accessions: "Personnel actions that brought people in, processed in the month (10k-60k rows); monthly from 2005-01.",
969
+ separations: "Personnel actions that took people out, processed in the month (10k-120k rows); monthly from 2005-01.",
970
+ };
971
+ function dictionaryRows(pack) {
972
+ const p = dictionaryPath(pack);
973
+ return p ? readCsv(fs.readFileSync(p, "utf8")) : [];
974
+ }
975
+ async function schema(names, pack) {
976
+ const dict = dictionaryRows(pack);
977
+ const out = [];
978
+ for (const n of names) {
979
+ if (!DATASETS.includes(n)) {
980
+ out.push({ name: n, error: `unknown dataset; use one of ${DATASETS.join(", ")}` });
981
+ continue;
982
+ }
983
+ let available;
984
+ try {
985
+ available = await listSummary(n);
986
+ }
987
+ catch (e) {
988
+ available = { error: `the file listing could not be read now (${e?.message || e}); the fetch lists it again` };
989
+ }
990
+ out.push({ name: n, summary: SUMMARY[n],
991
+ spec: { datasets: { [n]: { from: "YYYY-MM (optional; default the first file)", to: "YYYY-MM or \"latest\" (default)",
992
+ months: "optional list of months, e.g. [9] for September snapshots", periods: "or a list of single months: [\"2025-09\", \"latest\"]" } },
993
+ columns: "optional list of column names below (default all); always kept: " + [...ALWAYS[n], "file_yyyymm", "source_file"].join(", "),
994
+ filters: "optional {column: [values]}; values are strings exactly as in the data (\"08\", \"0343\")",
995
+ aggregate: "optional {\"by\": [columns]}: counts (headcount), pay_redacted, pay_n and pay_sum per group instead of microdata; keeps only the by columns",
996
+ pin: "optional {\"<dataset>_<YYYYMM>\": version}: exact file versions (from an earlier manifest.json)" },
997
+ always_kept: ALWAYS[n],
998
+ columns: dict.filter((r) => r[n] === "Y").map((r) => ({ name: r.field, definition: r.definition, example: r.sample_value })),
999
+ available });
1000
+ }
1001
+ return out;
1002
+ }
1003
+ function search(query, pack) {
1004
+ const words = query.toLowerCase().split(/\W+/).filter((w) => w.length > 1);
1005
+ if (!words.length)
1006
+ return [];
1007
+ const scored = dictionaryRows(pack).map((r) => {
1008
+ const text = `${r.field} ${r.definition}`.toLowerCase();
1009
+ return [words.filter((w) => text.includes(w)).length, r];
1010
+ }).filter(([s]) => s > 0);
1011
+ scored.sort((a, b) => b[0] - a[0]);
1012
+ return scored.slice(0, 25).map(([, r]) => ({ column: r.field, definition: r.definition, example: r.sample_value,
1013
+ datasets: DATASETS.filter((d) => r[d] === "Y") }));
1014
+ }
1015
+ function uses(spec) {
1016
+ const ds = normalize(spec).datasets;
1017
+ return isObj(ds) ? Object.keys(ds).filter((d) => DATASETS.includes(d)) : [];
1018
+ }
1019
+ const pack = {
1020
+ operations: DATASETS.map((d) => ({ name: d, summary: SUMMARY[d] })),
1021
+ schema,
1022
+ uses,
1023
+ specExample: { description: "September headcount by department, 2015-2025",
1024
+ datasets: { employment: { from: "2015-09", to: "2025-09", months: [9] } },
1025
+ columns: ["department_code", "department"], aggregate: { by: ["department_code", "department"] } },
1026
+ search,
1027
+ checkSpec,
1028
+ fetch,
1029
+ ingest,
1030
+ manualItems: (spec) => manualItems(spec),
1031
+ };
1032
+ export default pack;
1033
+ //# sourceMappingURL=opm.js.map