data-analyst-mcp 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (292) hide show
  1. package/README.md +286 -262
  2. package/dist/adapters/bjs.d.ts +13 -0
  3. package/dist/adapters/bjs.js +28 -0
  4. package/dist/adapters/bjs.js.map +1 -0
  5. package/dist/adapters/cbp.d.ts +5 -0
  6. package/dist/adapters/cbp.js +64 -0
  7. package/dist/adapters/cbp.js.map +1 -0
  8. package/dist/adapters/cdc-wonder.d.ts +5 -0
  9. package/dist/adapters/cdc-wonder.js +68 -0
  10. package/dist/adapters/cdc-wonder.js.map +1 -0
  11. package/dist/adapters/cpsc.d.ts +4 -0
  12. package/dist/adapters/cpsc.js +125 -0
  13. package/dist/adapters/cpsc.js.map +1 -0
  14. package/dist/adapters/fhwa.d.ts +9 -0
  15. package/dist/adapters/fhwa.js +16 -0
  16. package/dist/adapters/fhwa.js.map +1 -0
  17. package/dist/adapters/foreign-assistance.d.ts +9 -0
  18. package/dist/adapters/foreign-assistance.js +42 -0
  19. package/dist/adapters/foreign-assistance.js.map +1 -0
  20. package/dist/adapters/hud-ahar.d.ts +3 -0
  21. package/dist/adapters/hud-ahar.js +80 -0
  22. package/dist/adapters/hud-ahar.js.map +1 -0
  23. package/dist/adapters/nces.d.ts +10 -0
  24. package/dist/adapters/nces.js +32 -0
  25. package/dist/adapters/nces.js.map +1 -0
  26. package/dist/adapters/nsf.d.ts +11 -0
  27. package/dist/adapters/nsf.js +25 -0
  28. package/dist/adapters/nsf.js.map +1 -0
  29. package/dist/adapters/onrr.d.ts +9 -0
  30. package/dist/adapters/onrr.js +35 -0
  31. package/dist/adapters/onrr.js.map +1 -0
  32. package/dist/adapters/sba.d.ts +10 -0
  33. package/dist/adapters/sba.js +134 -0
  34. package/dist/adapters/sba.js.map +1 -0
  35. package/dist/adapters/treasury-tic.d.ts +30 -0
  36. package/dist/adapters/treasury-tic.js +149 -0
  37. package/dist/adapters/treasury-tic.js.map +1 -0
  38. package/dist/adapters/uscourts.d.ts +8 -0
  39. package/dist/adapters/uscourts.js +8 -0
  40. package/dist/adapters/uscourts.js.map +1 -0
  41. package/dist/analysis/cite.d.ts +9 -1
  42. package/dist/analysis/cite.js +9 -2
  43. package/dist/analysis/cite.js.map +1 -1
  44. package/dist/bin/cli.d.ts +2 -0
  45. package/dist/bin/cli.js +17 -0
  46. package/dist/bin/cli.js.map +1 -1
  47. package/dist/core/generic-rest.js +42 -17
  48. package/dist/core/generic-rest.js.map +1 -1
  49. package/dist/core/guard.d.ts +32 -0
  50. package/dist/core/guard.js +176 -0
  51. package/dist/core/guard.js.map +1 -0
  52. package/dist/core/http.d.ts +10 -0
  53. package/dist/core/http.js +36 -7
  54. package/dist/core/http.js.map +1 -1
  55. package/dist/core/keys.d.ts +13 -3
  56. package/dist/core/keys.js +67 -7
  57. package/dist/core/keys.js.map +1 -1
  58. package/dist/core/package.d.ts +12 -3
  59. package/dist/core/package.js +56 -12
  60. package/dist/core/package.js.map +1 -1
  61. package/dist/core/secret_patterns.d.ts +6 -0
  62. package/dist/core/secret_patterns.js +20 -0
  63. package/dist/core/secret_patterns.js.map +1 -0
  64. package/dist/core/tabular.d.ts +43 -1
  65. package/dist/core/tabular.js +149 -6
  66. package/dist/core/tabular.js.map +1 -1
  67. package/dist/envelope.js +3 -1
  68. package/dist/envelope.js.map +1 -1
  69. package/dist/index.d.ts +29 -22
  70. package/dist/index.js +45 -46
  71. package/dist/index.js.map +1 -1
  72. package/dist/remote/app.js +87 -31
  73. package/dist/remote/app.js.map +1 -1
  74. package/dist/remote/oauth.js +8 -1
  75. package/dist/remote/oauth.js.map +1 -1
  76. package/dist/remote/pages.d.ts +3 -7
  77. package/dist/remote/pages.js +49 -19
  78. package/dist/remote/pages.js.map +1 -1
  79. package/dist/remote/sign.d.ts +7 -1
  80. package/dist/remote/sign.js +33 -7
  81. package/dist/remote/sign.js.map +1 -1
  82. package/dist/remote/store.js +14 -5
  83. package/dist/remote/store.js.map +1 -1
  84. package/dist/report/audit_report.js +29 -7
  85. package/dist/report/audit_report.js.map +1 -1
  86. package/dist/report/build_report.js +34 -4
  87. package/dist/report/build_report.js.map +1 -1
  88. package/dist/report/method_audit.js +10 -1
  89. package/dist/report/method_audit.js.map +1 -1
  90. package/dist/report/scan_secrets.d.ts +4 -1
  91. package/dist/report/scan_secrets.js +38 -18
  92. package/dist/report/scan_secrets.js.map +1 -1
  93. package/dist/server.d.ts +3 -1
  94. package/dist/server.js +46 -13
  95. package/dist/server.js.map +1 -1
  96. package/dist/tools.js +27 -6
  97. package/dist/tools.js.map +1 -1
  98. package/guide/WORKFLOW.md +134 -134
  99. package/guide/assets/disclaimer.txt +1 -1
  100. package/guide/assets/download_data.ps1 +164 -162
  101. package/guide/assets/download_data.sh +8 -2
  102. package/guide/assets/report_template.html +597 -597
  103. package/guide/references/routing.md +198 -171
  104. package/guide/references/style_guide.md +398 -398
  105. package/package.json +56 -56
  106. package/packs/acf/endpoints.json +114 -114
  107. package/packs/acf/policy.json +104 -104
  108. package/packs/actuaries/endpoints.json +82 -82
  109. package/packs/actuaries/policy.json +109 -109
  110. package/packs/bea/assets/download_bea.ps1 +182 -182
  111. package/packs/bea/docs/bea_api.md +461 -461
  112. package/packs/bea/docs/datasets.md +157 -157
  113. package/packs/bea/docs/report_patterns.md +112 -112
  114. package/packs/bea/policy.json +170 -170
  115. package/packs/bea/source.md +152 -152
  116. package/packs/bjs/endpoints.json +150 -0
  117. package/packs/bjs/policy.json +114 -0
  118. package/packs/bjs/source.md +98 -0
  119. package/packs/bls/assets/download_bls.ps1 +152 -152
  120. package/packs/bls/docs/analysis_patterns.md +108 -108
  121. package/packs/bls/docs/bls_api.md +73 -73
  122. package/packs/bls/docs/series_ids.md +149 -149
  123. package/packs/bls/policy.json +238 -238
  124. package/packs/bts/endpoints.json +346 -346
  125. package/packs/bts/policy.json +94 -94
  126. package/packs/cbp/endpoints.json +31 -0
  127. package/packs/cbp/policy.json +85 -0
  128. package/packs/cbp/source.md +57 -0
  129. package/packs/cdc/endpoints.json +197 -197
  130. package/packs/cdc/policy.json +83 -83
  131. package/packs/cdc/source.md +1 -1
  132. package/packs/cdc-wonder/endpoints.json +26 -0
  133. package/packs/cdc-wonder/policy.json +84 -0
  134. package/packs/cdc-wonder/source.md +53 -0
  135. package/packs/census/endpoints.json +99 -99
  136. package/packs/cfpb/endpoints.json +326 -326
  137. package/packs/cfpb/policy.json +119 -119
  138. package/packs/clinical-trials/endpoints.json +166 -166
  139. package/packs/clinical-trials/policy.json +84 -84
  140. package/packs/cms/endpoints.json +69 -69
  141. package/packs/cms/policy.json +96 -96
  142. package/packs/college-scorecard/endpoints.json +252 -252
  143. package/packs/college-scorecard/policy.json +106 -106
  144. package/packs/college-scorecard/source.md +1 -1
  145. package/packs/congress/endpoints.json +1701 -1701
  146. package/packs/congress/policy.json +117 -117
  147. package/packs/cpsc/endpoints.json +91 -0
  148. package/packs/cpsc/policy.json +91 -0
  149. package/packs/cpsc/source.md +76 -0
  150. package/packs/dhs-ohss/endpoints.json +66 -66
  151. package/packs/dhs-ohss/policy.json +107 -107
  152. package/packs/dhs-ohss/source.md +1 -1
  153. package/packs/doj-news/endpoints.json +59 -59
  154. package/packs/doj-news/policy.json +96 -96
  155. package/packs/doj-news/source.md +1 -1
  156. package/packs/dol/endpoints.json +817 -817
  157. package/packs/dol/policy.json +135 -135
  158. package/packs/eia/endpoints.json +717 -717
  159. package/packs/eia/policy.json +112 -112
  160. package/packs/epa/endpoints.json +291 -291
  161. package/packs/epa/policy.json +99 -99
  162. package/packs/epa-aqs/endpoints.json +173 -173
  163. package/packs/epa-aqs/policy.json +96 -96
  164. package/packs/fbi/endpoints.json +1841 -1841
  165. package/packs/fbi/policy.json +118 -118
  166. package/packs/fbi/source.md +3 -3
  167. package/packs/fda/endpoints.json +237 -237
  168. package/packs/fda/policy.json +121 -121
  169. package/packs/fdic/endpoints.json +416 -416
  170. package/packs/fdic/policy.json +83 -83
  171. package/packs/fec/endpoints.json +707 -707
  172. package/packs/fec/policy.json +93 -93
  173. package/packs/fed-board/endpoints.json +89 -89
  174. package/packs/fed-board/policy.json +136 -136
  175. package/packs/federal-register/endpoints.json +143 -143
  176. package/packs/federal-register/policy.json +86 -86
  177. package/packs/fema/endpoints.json +645 -645
  178. package/packs/fema/policy.json +103 -103
  179. package/packs/fhfa/endpoints.json +105 -105
  180. package/packs/fhfa/policy.json +109 -109
  181. package/packs/fhwa/endpoints.json +48 -0
  182. package/packs/fhwa/policy.json +91 -0
  183. package/packs/fhwa/source.md +59 -0
  184. package/packs/foreign-assistance/endpoints.json +166 -0
  185. package/packs/foreign-assistance/policy.json +80 -0
  186. package/packs/foreign-assistance/source.md +76 -0
  187. package/packs/fred/assets/download_fred.ps1 +122 -122
  188. package/packs/fred/docs/analysis_patterns.md +124 -124
  189. package/packs/fred/docs/fred_api.md +94 -94
  190. package/packs/fred/docs/series.md +188 -188
  191. package/packs/fred/policy.json +253 -253
  192. package/packs/fsa/endpoints.json +150 -150
  193. package/packs/fsa/policy.json +99 -99
  194. package/packs/fsa/source.md +1 -1
  195. package/packs/govinfo/endpoints.json +519 -519
  196. package/packs/gsa-calc/endpoints.json +93 -93
  197. package/packs/gsa-calc/policy.json +97 -97
  198. package/packs/hud/endpoints.json +146 -146
  199. package/packs/hud/policy.json +84 -84
  200. package/packs/hud/source.md +1 -1
  201. package/packs/hud-ahar/endpoints.json +92 -0
  202. package/packs/hud-ahar/policy.json +88 -0
  203. package/packs/hud-ahar/source.md +72 -0
  204. package/packs/ipums/docs/report_patterns.md +117 -117
  205. package/packs/irs-soi/endpoints.json +114 -114
  206. package/packs/irs-soi/policy.json +107 -107
  207. package/packs/medicaid/endpoints.json +412 -412
  208. package/packs/medicaid/policy.json +108 -108
  209. package/packs/naep/endpoints.json +217 -217
  210. package/packs/naep/policy.json +100 -100
  211. package/packs/naep/source.md +1 -1
  212. package/packs/nces/endpoints.json +101 -0
  213. package/packs/nces/policy.json +99 -0
  214. package/packs/nces/source.md +75 -0
  215. package/packs/nhtsa/endpoints.json +132 -132
  216. package/packs/nhtsa/policy.json +93 -93
  217. package/packs/nhtsa/source.md +1 -1
  218. package/packs/nih/endpoints.json +452 -452
  219. package/packs/nih/policy.json +86 -86
  220. package/packs/noaa/endpoints.json +431 -431
  221. package/packs/noaa/policy.json +95 -95
  222. package/packs/nrel/endpoints.json +327 -327
  223. package/packs/nrel/policy.json +96 -96
  224. package/packs/nsf/endpoints.json +33 -0
  225. package/packs/nsf/policy.json +83 -0
  226. package/packs/nsf/source.md +59 -0
  227. package/packs/nws/endpoints.json +271 -271
  228. package/packs/nws/policy.json +82 -82
  229. package/packs/nyfed/endpoints.json +80 -80
  230. package/packs/nyfed/policy.json +92 -92
  231. package/packs/omb/endpoints.json +460 -460
  232. package/packs/omb/policy.json +106 -106
  233. package/packs/onrr/endpoints.json +198 -0
  234. package/packs/onrr/policy.json +91 -0
  235. package/packs/onrr/source.md +55 -0
  236. package/packs/open-payments/endpoints.json +297 -297
  237. package/packs/open-payments/policy.json +101 -101
  238. package/packs/opm/assets/download_opm.ps1 +125 -125
  239. package/packs/opm/docs/fields.md +184 -184
  240. package/packs/opm/docs/opm_api.md +71 -71
  241. package/packs/opm/docs/report_patterns.md +83 -83
  242. package/packs/opm/lib/data_dictionary.csv +73 -73
  243. package/packs/pbgc/endpoints.json +483 -483
  244. package/packs/pbgc/policy.json +94 -94
  245. package/packs/philfed/endpoints.json +85 -85
  246. package/packs/philfed/policy.json +94 -94
  247. package/packs/port.json +65 -0
  248. package/packs/regulations/endpoints.json +287 -287
  249. package/packs/regulations/policy.json +86 -86
  250. package/packs/sba/endpoints.json +34 -0
  251. package/packs/sba/policy.json +86 -0
  252. package/packs/sba/source.md +60 -0
  253. package/packs/sec/endpoints.json +236 -236
  254. package/packs/sec/policy.json +86 -86
  255. package/packs/senate-lobbying/endpoints.json +553 -553
  256. package/packs/senate-lobbying/policy.json +99 -99
  257. package/packs/snapqc/endpoints.json +1426 -1426
  258. package/packs/snapqc/policy.json +92 -92
  259. package/packs/treasury/catalog.json +2267 -2267
  260. package/packs/treasury/docs/api_guide.md +245 -245
  261. package/packs/treasury/docs/datasets.md +56 -56
  262. package/packs/treasury/docs/report_patterns.md +126 -126
  263. package/packs/treasury/policy.json +159 -159
  264. package/packs/treasury/source.md +170 -165
  265. package/packs/treasury-tic/endpoints.json +77 -0
  266. package/packs/treasury-tic/policy.json +89 -0
  267. package/packs/treasury-tic/source.md +80 -0
  268. package/packs/treasury-yields/endpoints.json +226 -226
  269. package/packs/treasury-yields/policy.json +86 -86
  270. package/packs/usaspending/endpoints.json +1315 -1315
  271. package/packs/usaspending/policy.json +109 -109
  272. package/packs/uscourts/endpoints.json +22 -0
  273. package/packs/uscourts/policy.json +87 -0
  274. package/packs/uscourts/source.md +56 -0
  275. package/packs/usda-fna/policy.json +133 -133
  276. package/packs/usda-fooddata/endpoints.json +180 -180
  277. package/packs/usda-fooddata/policy.json +94 -94
  278. package/packs/usda-nass/endpoints.json +323 -323
  279. package/packs/usda-nass/policy.json +95 -95
  280. package/packs/usda-outlook/endpoints.json +86 -86
  281. package/packs/usda-outlook/policy.json +109 -109
  282. package/packs/usgs/endpoints.json +313 -313
  283. package/packs/usgs/policy.json +95 -95
  284. package/packs/usitc/endpoints.json +74 -74
  285. package/packs/usitc/policy.json +111 -111
  286. package/packs/uspto/endpoints.json +141 -141
  287. package/packs/uspto/policy.json +95 -95
  288. package/packs/uspto/source.md +1 -1
  289. package/packs/va/endpoints.json +414 -414
  290. package/packs/va/policy.json +101 -101
  291. package/packs/world-bank/endpoints.json +122 -122
  292. package/packs/world-bank/policy.json +101 -101
@@ -1,101 +1,101 @@
1
- {
2
- "id": "open-payments",
3
- "title": "CMS Open Payments (openpaymentsdata.cms.gov)",
4
- "agency": "Centers for Medicare & Medicaid Services",
5
- "depth": "standard",
6
- "domains": [
7
- "health",
8
- "health care",
9
- "industry payments"
10
- ],
11
- "keywords": [
12
- "open payments",
13
- "sunshine act",
14
- "industry payments",
15
- "pharma payments",
16
- "payments to doctors",
17
- "physician payments",
18
- "drug company",
19
- "device company",
20
- "manufacturer",
21
- "consulting fee",
22
- "speaking fee",
23
- "food and beverage",
24
- "royalty",
25
- "research payments",
26
- "ownership interest",
27
- "teaching hospital",
28
- "npi"
29
- ],
30
- "questions": [
31
- "payments from drug and device companies to doctors in a state",
32
- "which companies paid the most to physicians",
33
- "payments to a specific physician by NPI",
34
- "industry payments by specialty",
35
- "research payments to teaching hospitals"
36
- ],
37
- "homepage": "https://openpaymentsdata.cms.gov/",
38
- "terms_url": "https://www.usa.gov/government-works",
39
- "auth": {
40
- "env": null,
41
- "required": "none",
42
- "placement": "none",
43
- "signup_url": null,
44
- "echoes_key": false
45
- },
46
- "limits": {
47
- "rps": 2,
48
- "burst": 4,
49
- "max_rows": 100000
50
- },
51
- "routes": {
52
- "api": {
53
- "base": "https://openpaymentsdata.cms.gov/api/1",
54
- "probe": "https://openpaymentsdata.cms.gov/api/1/metastore/schemas/dataset/items/02157882-0f43-473a-a777-c96eb5b1b623",
55
- "hosts": [
56
- "openpaymentsdata.cms.gov"
57
- ]
58
- },
59
- "manual": {
60
- "how": "Open the JSON links in INSTRUCTIONS.md (filters applied, 500 rows a page), save each under the shown name and upload; or search https://openpaymentsdata.cms.gov/ (Search > by physician, hospital or company) and export the results. Full-year payment files are several GB: download them from the dataset page only on a computer with room, then upload a filtered extract."
61
- }
62
- },
63
- "cache": {
64
- "mode": "run_only",
65
- "ttl_hours": null,
66
- "why": "Filtered queries are small; every dataset is republished when CMS refreshes the program years (all were last modified 2026-06-30). data/ is reused within a run."
67
- },
68
- "redistribution": {
69
- "mode": "allowed",
70
- "why": "Open Payments datasets carry the licence https://www.usa.gov/government-works (U.S. government work, public domain, 17 U.S.C. 105) in the CMS metastore."
71
- },
72
- "notices": [
73
- "Open Payments records are reported by manufacturers and group purchasing organizations and published as reported (recipients may dispute them); a payment does not by itself indicate a conflict of interest or wrongdoing."
74
- ],
75
- "retention": null,
76
- "citation": "Centers for Medicare & Medicaid Services, Open Payments: {dataset}, openpaymentsdata.cms.gov, retrieved {retrieved_long}.",
77
- "audit_hooks": [
78
- {
79
- "id": "op_not_wrongdoing",
80
- "kind": "require_text",
81
- "any": [
82
- "not verified",
83
- "does not indicate",
84
- "not necessarily",
85
- "conflict of interest",
86
- "wrongdoing",
87
- "reported by"
88
- ],
89
- "why": "Reports using Open Payments must say that payments are self-reported by industry and do not imply wrongdoing."
90
- }
91
- ],
92
- "clarify": [
93
- "Which program year(s)? Each year is a separate file; the latest published year is 2025 (published June 2026).",
94
- "General payments only, or also research payments and ownership interests? They are separate datasets and are not additive with the summaries.",
95
- "Physicians only, or also nurse practitioners and other non-physician practitioners (reported since program year 2021) and teaching hospitals?"
96
- ],
97
- "formats": [
98
- "json",
99
- "csv"
100
- ]
101
- }
1
+ {
2
+ "id": "open-payments",
3
+ "title": "CMS Open Payments (openpaymentsdata.cms.gov)",
4
+ "agency": "Centers for Medicare & Medicaid Services",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "health",
8
+ "health care",
9
+ "industry payments"
10
+ ],
11
+ "keywords": [
12
+ "open payments",
13
+ "sunshine act",
14
+ "industry payments",
15
+ "pharma payments",
16
+ "payments to doctors",
17
+ "physician payments",
18
+ "drug company",
19
+ "device company",
20
+ "manufacturer",
21
+ "consulting fee",
22
+ "speaking fee",
23
+ "food and beverage",
24
+ "royalty",
25
+ "research payments",
26
+ "ownership interest",
27
+ "teaching hospital",
28
+ "npi"
29
+ ],
30
+ "questions": [
31
+ "payments from drug and device companies to doctors in a state",
32
+ "which companies paid the most to physicians",
33
+ "payments to a specific physician by NPI",
34
+ "industry payments by specialty",
35
+ "research payments to teaching hospitals"
36
+ ],
37
+ "homepage": "https://openpaymentsdata.cms.gov/",
38
+ "terms_url": "https://www.usa.gov/government-works",
39
+ "auth": {
40
+ "env": null,
41
+ "required": "none",
42
+ "placement": "none",
43
+ "signup_url": null,
44
+ "echoes_key": false
45
+ },
46
+ "limits": {
47
+ "rps": 2,
48
+ "burst": 4,
49
+ "max_rows": 100000
50
+ },
51
+ "routes": {
52
+ "api": {
53
+ "base": "https://openpaymentsdata.cms.gov/api/1",
54
+ "probe": "https://openpaymentsdata.cms.gov/api/1/metastore/schemas/dataset/items/02157882-0f43-473a-a777-c96eb5b1b623",
55
+ "hosts": [
56
+ "openpaymentsdata.cms.gov"
57
+ ]
58
+ },
59
+ "manual": {
60
+ "how": "Open the JSON links in INSTRUCTIONS.md (filters applied, 500 rows a page), save each under the shown name and upload; or search https://openpaymentsdata.cms.gov/ (Search > by physician, hospital or company) and export the results. Full-year payment files are several GB: download them from the dataset page only on a computer with room, then upload a filtered extract."
61
+ }
62
+ },
63
+ "cache": {
64
+ "mode": "run_only",
65
+ "ttl_hours": null,
66
+ "why": "Filtered queries are small; every dataset is republished when CMS refreshes the program years (all were last modified 2026-06-30). data/ is reused within a run."
67
+ },
68
+ "redistribution": {
69
+ "mode": "allowed",
70
+ "why": "Open Payments datasets carry the licence https://www.usa.gov/government-works (U.S. government work, public domain, 17 U.S.C. 105) in the CMS metastore."
71
+ },
72
+ "notices": [
73
+ "Open Payments records are reported by manufacturers and group purchasing organizations and published as reported (recipients may dispute them); a payment does not by itself indicate a conflict of interest or wrongdoing."
74
+ ],
75
+ "retention": null,
76
+ "citation": "Centers for Medicare & Medicaid Services, Open Payments: {dataset}, openpaymentsdata.cms.gov, retrieved {retrieved_long}.",
77
+ "audit_hooks": [
78
+ {
79
+ "id": "op_not_wrongdoing",
80
+ "kind": "require_text",
81
+ "any": [
82
+ "not verified",
83
+ "does not indicate",
84
+ "not necessarily",
85
+ "conflict of interest",
86
+ "wrongdoing",
87
+ "reported by"
88
+ ],
89
+ "why": "Reports using Open Payments must say that payments are self-reported by industry and do not imply wrongdoing."
90
+ }
91
+ ],
92
+ "clarify": [
93
+ "Which program year(s)? Each year is a separate file; the latest published year is 2025 (published June 2026).",
94
+ "General payments only, or also research payments and ownership interests? They are separate datasets and are not additive with the summaries.",
95
+ "Physicians only, or also nurse practitioners and other non-physician practitioners (reported since program year 2021) and teaching hospitals?"
96
+ ],
97
+ "formats": [
98
+ "json",
99
+ "csv"
100
+ ]
101
+ }
@@ -1,125 +1,125 @@
1
- <#
2
- .SYNOPSIS
3
- Download the OPM Federal Workforce Data files for this analysis into the folder that holds this script.
4
- (Windows PowerShell counterpart of download_opm.sh.)
5
-
6
- .DESCRIPTION
7
- - The request is built into this script (written by data-analyst-mcp; do not edit it).
8
- - It lists the files OPM publishes now (data.opm.gov, no account or key), picks the months the request
9
- asks for, and saves each one as <dataset>_<YYYYMM>_<version>.parquet.
10
- - Employment files are 26-75 MB each; accession and separation files are under 4 MB.
11
- - Needs nothing beyond Windows PowerShell 5.1 or PowerShell 7. Run it again after an interruption:
12
- files already saved are kept.
13
-
14
- .PARAMETER NoPause
15
- Do not wait for Enter before closing the window.
16
-
17
- .EXAMPLE
18
- powershell -NoProfile -ExecutionPolicy Bypass -File .\download_opm.ps1
19
- #>
20
- [CmdletBinding()]
21
- param([switch]$NoPause)
22
-
23
- $ErrorActionPreference = 'Stop'
24
- $ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
25
- try {
26
- [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
27
- } catch { }
28
-
29
- $Api = 'https://data.opm.gov/api/v1/files'
30
- if ($env:OPM_API_BASE) { $Api = $env:OPM_API_BASE.TrimEnd('/') }
31
-
32
- # The request. Columns, tab-separated: dataset, then either
33
- # periods YYYYMM,... (or "latest") the listed months
34
- # range FROM|first TO|latest MONTHS every month from FROM to TO in MONTHS (1-12)
35
- # pin YYYYMM VERSION use this version of one month instead of the current one
36
- $Rules = @'
37
- @RULES@
38
- '@
39
-
40
- $Here = $PSScriptRoot
41
- if (-not $Here) { $Here = (Get-Location).Path }
42
-
43
- function Finish([int]$Code) {
44
- if (-not $NoPause -and -not [Console]::IsInputRedirected) {
45
- Write-Host ''
46
- [void](Read-Host -Prompt 'Press Enter to close')
47
- }
48
- exit $Code
49
- }
50
-
51
- function Fail([string]$Message) {
52
- Write-Host "ERROR: $Message" -ForegroundColor Red
53
- Finish 1
54
- }
55
-
56
- $rows = @(foreach ($line in ($Rules -split "`r?`n")) {
57
- if (-not $line.Trim()) { continue }
58
- $c = $line -split "`t"
59
- [pscustomobject]@{ Ds = $c[0]; Kind = $c[1]; A = $c[2]; B = $c[3]; C = $c[4] }
60
- })
61
- $saved = @(); $failed = 0; $missing = @()
62
- foreach ($ds in @($rows | ForEach-Object { $_.Ds } | Sort-Object -Unique)) {
63
- Write-Host "Listing the $ds files..."
64
- try {
65
- $reply = Invoke-RestMethod -Uri "$Api/$($ds)?current=true" -UseBasicParsing -UserAgent 'data-analyst/1.0'
66
- if ($reply -is [string]) { $reply = $reply | ConvertFrom-Json } # served without a JSON content type
67
- $list = @($reply | ForEach-Object { $_ })
68
- } catch {
69
- Fail "Could not list the $ds files at $Api ($($_.Exception.Message)). Check the connection, VPN and proxy, then run again."
70
- }
71
- $avail = @{}
72
- foreach ($f in $list) {
73
- if ([string]$f.filename -match "^$($ds)_(\d{6})_(\d+)$") { $avail[$Matches[1]] = $Matches[2] }
74
- }
75
- if ($avail.Count -eq 0) { Fail "OPM listed no $ds files. Run again later." }
76
- $periods = @($avail.Keys | Sort-Object)
77
- $first = $periods[0]; $latest = $periods[-1]
78
- $want = @()
79
- foreach ($r in @($rows | Where-Object { $_.Ds -eq $ds })) {
80
- if ($r.Kind -eq 'periods') {
81
- foreach ($p in $r.A.Split(',')) {
82
- if ($p -eq 'latest') { $p = $latest }
83
- if ($avail.ContainsKey($p)) { $want += $p } else { $missing += "$ds $p" }
84
- }
85
- } elseif ($r.Kind -eq 'range') {
86
- $lo = $r.A; $hi = $r.B
87
- if ($lo -eq 'first') { $lo = $first }
88
- if ($hi -eq 'latest') { $hi = $latest }
89
- $months = @($r.C.Split(',') | ForEach-Object { [int]$_ })
90
- foreach ($p in $periods) {
91
- if ($p -ge $lo -and $p -le $hi -and $months -contains [int]$p.Substring(4, 2)) { $want += $p }
92
- }
93
- }
94
- }
95
- foreach ($p in @($want | Sort-Object -Unique)) {
96
- $v = $avail[$p]
97
- $pin = @($rows | Where-Object { $_.Ds -eq $ds -and $_.Kind -eq 'pin' -and $_.A -eq $p })
98
- if ($pin.Count -gt 0) { $v = $pin[0].B }
99
- $name = "$($ds)_$($p)_$($v).parquet"
100
- $dest = Join-Path $Here $name
101
- if ((Test-Path -LiteralPath $dest) -and (Get-Item -LiteralPath $dest).Length -gt 0) {
102
- Write-Host "already saved: $name"; $saved += $dest; continue
103
- }
104
- Write-Host "Downloading $name ..."
105
- try {
106
- Invoke-WebRequest -Uri "$Api/$ds/$($p.Substring(0, 4))/$($p.Substring(4, 2))/$v/download" -OutFile "$dest.part" `
107
- -UseBasicParsing -UserAgent 'data-analyst/1.0'
108
- Move-Item -LiteralPath "$dest.part" -Destination $dest -Force
109
- $saved += $dest
110
- } catch {
111
- if (Test-Path -LiteralPath "$dest.part") { Remove-Item -LiteralPath "$dest.part" -Force }
112
- Write-Host "FAILED $name ($($_.Exception.Message))" -ForegroundColor Red
113
- $failed++
114
- }
115
- }
116
- }
117
-
118
- Write-Host ''
119
- if ($missing.Count -gt 0) { Write-Host "Not published by OPM (skipped): $($missing -join ', ')" }
120
- if ($saved.Count -gt 0) {
121
- Write-Host 'Done. Upload these files to the chat (zip them first if they are large; do not rename them):' -ForegroundColor Green
122
- foreach ($f in $saved) { Write-Host " $f" }
123
- }
124
- if ($failed -gt 0) { Write-Host "$failed file(s) failed: run this script again." -ForegroundColor Red; Finish 1 }
125
- Finish 0
1
+ <#
2
+ .SYNOPSIS
3
+ Download the OPM Federal Workforce Data files for this analysis into the folder that holds this script.
4
+ (Windows PowerShell counterpart of download_opm.sh.)
5
+
6
+ .DESCRIPTION
7
+ - The request is built into this script (written by data-analyst-mcp; do not edit it).
8
+ - It lists the files OPM publishes now (data.opm.gov, no account or key), picks the months the request
9
+ asks for, and saves each one as <dataset>_<YYYYMM>_<version>.parquet.
10
+ - Employment files are 26-75 MB each; accession and separation files are under 4 MB.
11
+ - Needs nothing beyond Windows PowerShell 5.1 or PowerShell 7. Run it again after an interruption:
12
+ files already saved are kept.
13
+
14
+ .PARAMETER NoPause
15
+ Do not wait for Enter before closing the window.
16
+
17
+ .EXAMPLE
18
+ powershell -NoProfile -ExecutionPolicy Bypass -File .\download_opm.ps1
19
+ #>
20
+ [CmdletBinding()]
21
+ param([switch]$NoPause)
22
+
23
+ $ErrorActionPreference = 'Stop'
24
+ $ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
25
+ try {
26
+ [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
27
+ } catch { }
28
+
29
+ $Api = 'https://data.opm.gov/api/v1/files'
30
+ if ($env:OPM_API_BASE) { $Api = $env:OPM_API_BASE.TrimEnd('/') }
31
+
32
+ # The request. Columns, tab-separated: dataset, then either
33
+ # periods YYYYMM,... (or "latest") the listed months
34
+ # range FROM|first TO|latest MONTHS every month from FROM to TO in MONTHS (1-12)
35
+ # pin YYYYMM VERSION use this version of one month instead of the current one
36
+ $Rules = @'
37
+ @RULES@
38
+ '@
39
+
40
+ $Here = $PSScriptRoot
41
+ if (-not $Here) { $Here = (Get-Location).Path }
42
+
43
+ function Finish([int]$Code) {
44
+ if (-not $NoPause -and -not [Console]::IsInputRedirected) {
45
+ Write-Host ''
46
+ [void](Read-Host -Prompt 'Press Enter to close')
47
+ }
48
+ exit $Code
49
+ }
50
+
51
+ function Fail([string]$Message) {
52
+ Write-Host "ERROR: $Message" -ForegroundColor Red
53
+ Finish 1
54
+ }
55
+
56
+ $rows = @(foreach ($line in ($Rules -split "`r?`n")) {
57
+ if (-not $line.Trim()) { continue }
58
+ $c = $line -split "`t"
59
+ [pscustomobject]@{ Ds = $c[0]; Kind = $c[1]; A = $c[2]; B = $c[3]; C = $c[4] }
60
+ })
61
+ $saved = @(); $failed = 0; $missing = @()
62
+ foreach ($ds in @($rows | ForEach-Object { $_.Ds } | Sort-Object -Unique)) {
63
+ Write-Host "Listing the $ds files..."
64
+ try {
65
+ $reply = Invoke-RestMethod -Uri "$Api/$($ds)?current=true" -UseBasicParsing -UserAgent 'data-analyst/1.0'
66
+ if ($reply -is [string]) { $reply = $reply | ConvertFrom-Json } # served without a JSON content type
67
+ $list = @($reply | ForEach-Object { $_ })
68
+ } catch {
69
+ Fail "Could not list the $ds files at $Api ($($_.Exception.Message)). Check the connection, VPN and proxy, then run again."
70
+ }
71
+ $avail = @{}
72
+ foreach ($f in $list) {
73
+ if ([string]$f.filename -match "^$($ds)_(\d{6})_(\d+)$") { $avail[$Matches[1]] = $Matches[2] }
74
+ }
75
+ if ($avail.Count -eq 0) { Fail "OPM listed no $ds files. Run again later." }
76
+ $periods = @($avail.Keys | Sort-Object)
77
+ $first = $periods[0]; $latest = $periods[-1]
78
+ $want = @()
79
+ foreach ($r in @($rows | Where-Object { $_.Ds -eq $ds })) {
80
+ if ($r.Kind -eq 'periods') {
81
+ foreach ($p in $r.A.Split(',')) {
82
+ if ($p -eq 'latest') { $p = $latest }
83
+ if ($avail.ContainsKey($p)) { $want += $p } else { $missing += "$ds $p" }
84
+ }
85
+ } elseif ($r.Kind -eq 'range') {
86
+ $lo = $r.A; $hi = $r.B
87
+ if ($lo -eq 'first') { $lo = $first }
88
+ if ($hi -eq 'latest') { $hi = $latest }
89
+ $months = @($r.C.Split(',') | ForEach-Object { [int]$_ })
90
+ foreach ($p in $periods) {
91
+ if ($p -ge $lo -and $p -le $hi -and $months -contains [int]$p.Substring(4, 2)) { $want += $p }
92
+ }
93
+ }
94
+ }
95
+ foreach ($p in @($want | Sort-Object -Unique)) {
96
+ $v = $avail[$p]
97
+ $pin = @($rows | Where-Object { $_.Ds -eq $ds -and $_.Kind -eq 'pin' -and $_.A -eq $p })
98
+ if ($pin.Count -gt 0) { $v = $pin[0].B }
99
+ $name = "$($ds)_$($p)_$($v).parquet"
100
+ $dest = Join-Path $Here $name
101
+ if ((Test-Path -LiteralPath $dest) -and (Get-Item -LiteralPath $dest).Length -gt 0) {
102
+ Write-Host "already saved: $name"; $saved += $dest; continue
103
+ }
104
+ Write-Host "Downloading $name ..."
105
+ try {
106
+ Invoke-WebRequest -Uri "$Api/$ds/$($p.Substring(0, 4))/$($p.Substring(4, 2))/$v/download" -OutFile "$dest.part" `
107
+ -UseBasicParsing -UserAgent 'data-analyst/1.0'
108
+ Move-Item -LiteralPath "$dest.part" -Destination $dest -Force
109
+ $saved += $dest
110
+ } catch {
111
+ if (Test-Path -LiteralPath "$dest.part") { Remove-Item -LiteralPath "$dest.part" -Force }
112
+ Write-Host "FAILED $name ($($_.Exception.Message))" -ForegroundColor Red
113
+ $failed++
114
+ }
115
+ }
116
+ }
117
+
118
+ Write-Host ''
119
+ if ($missing.Count -gt 0) { Write-Host "Not published by OPM (skipped): $($missing -join ', ')" }
120
+ if ($saved.Count -gt 0) {
121
+ Write-Host 'Done. Upload these files to the chat (zip them first if they are large; do not rename them):' -ForegroundColor Green
122
+ foreach ($f in $saved) { Write-Host " $f" }
123
+ }
124
+ if ($failed -gt 0) { Write-Host "$failed file(s) failed: run this script again." -ForegroundColor Red; Finish 1 }
125
+ Finish 0