data-analyst-mcp 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +286 -262
- package/dist/adapters/bjs.d.ts +13 -0
- package/dist/adapters/bjs.js +28 -0
- package/dist/adapters/bjs.js.map +1 -0
- package/dist/adapters/cbp.d.ts +5 -0
- package/dist/adapters/cbp.js +64 -0
- package/dist/adapters/cbp.js.map +1 -0
- package/dist/adapters/cdc-wonder.d.ts +5 -0
- package/dist/adapters/cdc-wonder.js +68 -0
- package/dist/adapters/cdc-wonder.js.map +1 -0
- package/dist/adapters/cpsc.d.ts +4 -0
- package/dist/adapters/cpsc.js +125 -0
- package/dist/adapters/cpsc.js.map +1 -0
- package/dist/adapters/fhwa.d.ts +9 -0
- package/dist/adapters/fhwa.js +16 -0
- package/dist/adapters/fhwa.js.map +1 -0
- package/dist/adapters/foreign-assistance.d.ts +9 -0
- package/dist/adapters/foreign-assistance.js +42 -0
- package/dist/adapters/foreign-assistance.js.map +1 -0
- package/dist/adapters/hud-ahar.d.ts +3 -0
- package/dist/adapters/hud-ahar.js +80 -0
- package/dist/adapters/hud-ahar.js.map +1 -0
- package/dist/adapters/nces.d.ts +10 -0
- package/dist/adapters/nces.js +32 -0
- package/dist/adapters/nces.js.map +1 -0
- package/dist/adapters/nsf.d.ts +11 -0
- package/dist/adapters/nsf.js +25 -0
- package/dist/adapters/nsf.js.map +1 -0
- package/dist/adapters/onrr.d.ts +9 -0
- package/dist/adapters/onrr.js +35 -0
- package/dist/adapters/onrr.js.map +1 -0
- package/dist/adapters/sba.d.ts +10 -0
- package/dist/adapters/sba.js +134 -0
- package/dist/adapters/sba.js.map +1 -0
- package/dist/adapters/treasury-tic.d.ts +30 -0
- package/dist/adapters/treasury-tic.js +149 -0
- package/dist/adapters/treasury-tic.js.map +1 -0
- package/dist/adapters/uscourts.d.ts +8 -0
- package/dist/adapters/uscourts.js +8 -0
- package/dist/adapters/uscourts.js.map +1 -0
- package/dist/analysis/cite.d.ts +9 -1
- package/dist/analysis/cite.js +9 -2
- package/dist/analysis/cite.js.map +1 -1
- package/dist/bin/cli.d.ts +2 -0
- package/dist/bin/cli.js +17 -0
- package/dist/bin/cli.js.map +1 -1
- package/dist/core/generic-rest.js +42 -17
- package/dist/core/generic-rest.js.map +1 -1
- package/dist/core/guard.d.ts +32 -0
- package/dist/core/guard.js +176 -0
- package/dist/core/guard.js.map +1 -0
- package/dist/core/http.d.ts +10 -0
- package/dist/core/http.js +36 -7
- package/dist/core/http.js.map +1 -1
- package/dist/core/keys.d.ts +13 -3
- package/dist/core/keys.js +67 -7
- package/dist/core/keys.js.map +1 -1
- package/dist/core/package.d.ts +12 -3
- package/dist/core/package.js +56 -12
- package/dist/core/package.js.map +1 -1
- package/dist/core/secret_patterns.d.ts +6 -0
- package/dist/core/secret_patterns.js +20 -0
- package/dist/core/secret_patterns.js.map +1 -0
- package/dist/core/tabular.d.ts +43 -1
- package/dist/core/tabular.js +149 -6
- package/dist/core/tabular.js.map +1 -1
- package/dist/envelope.js +3 -1
- package/dist/envelope.js.map +1 -1
- package/dist/index.d.ts +29 -22
- package/dist/index.js +45 -46
- package/dist/index.js.map +1 -1
- package/dist/remote/app.js +87 -31
- package/dist/remote/app.js.map +1 -1
- package/dist/remote/oauth.js +8 -1
- package/dist/remote/oauth.js.map +1 -1
- package/dist/remote/pages.d.ts +3 -7
- package/dist/remote/pages.js +49 -19
- package/dist/remote/pages.js.map +1 -1
- package/dist/remote/sign.d.ts +7 -1
- package/dist/remote/sign.js +33 -7
- package/dist/remote/sign.js.map +1 -1
- package/dist/remote/store.js +14 -5
- package/dist/remote/store.js.map +1 -1
- package/dist/report/audit_report.js +29 -7
- package/dist/report/audit_report.js.map +1 -1
- package/dist/report/build_report.js +34 -4
- package/dist/report/build_report.js.map +1 -1
- package/dist/report/method_audit.js +10 -1
- package/dist/report/method_audit.js.map +1 -1
- package/dist/report/scan_secrets.d.ts +4 -1
- package/dist/report/scan_secrets.js +38 -18
- package/dist/report/scan_secrets.js.map +1 -1
- package/dist/server.d.ts +3 -1
- package/dist/server.js +46 -13
- package/dist/server.js.map +1 -1
- package/dist/tools.js +27 -6
- package/dist/tools.js.map +1 -1
- package/guide/WORKFLOW.md +134 -134
- package/guide/assets/disclaimer.txt +1 -1
- package/guide/assets/download_data.ps1 +164 -162
- package/guide/assets/download_data.sh +8 -2
- package/guide/assets/report_template.html +597 -597
- package/guide/references/routing.md +198 -171
- package/guide/references/style_guide.md +398 -398
- package/package.json +56 -56
- package/packs/acf/endpoints.json +114 -114
- package/packs/acf/policy.json +104 -104
- package/packs/actuaries/endpoints.json +82 -82
- package/packs/actuaries/policy.json +109 -109
- package/packs/bea/assets/download_bea.ps1 +182 -182
- package/packs/bea/docs/bea_api.md +461 -461
- package/packs/bea/docs/datasets.md +157 -157
- package/packs/bea/docs/report_patterns.md +112 -112
- package/packs/bea/policy.json +170 -170
- package/packs/bea/source.md +152 -152
- package/packs/bjs/endpoints.json +150 -0
- package/packs/bjs/policy.json +114 -0
- package/packs/bjs/source.md +98 -0
- package/packs/bls/assets/download_bls.ps1 +152 -152
- package/packs/bls/docs/analysis_patterns.md +108 -108
- package/packs/bls/docs/bls_api.md +73 -73
- package/packs/bls/docs/series_ids.md +149 -149
- package/packs/bls/policy.json +238 -238
- package/packs/bts/endpoints.json +346 -346
- package/packs/bts/policy.json +94 -94
- package/packs/cbp/endpoints.json +31 -0
- package/packs/cbp/policy.json +85 -0
- package/packs/cbp/source.md +57 -0
- package/packs/cdc/endpoints.json +197 -197
- package/packs/cdc/policy.json +83 -83
- package/packs/cdc/source.md +1 -1
- package/packs/cdc-wonder/endpoints.json +26 -0
- package/packs/cdc-wonder/policy.json +84 -0
- package/packs/cdc-wonder/source.md +53 -0
- package/packs/census/endpoints.json +99 -99
- package/packs/cfpb/endpoints.json +326 -326
- package/packs/cfpb/policy.json +119 -119
- package/packs/clinical-trials/endpoints.json +166 -166
- package/packs/clinical-trials/policy.json +84 -84
- package/packs/cms/endpoints.json +69 -69
- package/packs/cms/policy.json +96 -96
- package/packs/college-scorecard/endpoints.json +252 -252
- package/packs/college-scorecard/policy.json +106 -106
- package/packs/college-scorecard/source.md +1 -1
- package/packs/congress/endpoints.json +1701 -1701
- package/packs/congress/policy.json +117 -117
- package/packs/cpsc/endpoints.json +91 -0
- package/packs/cpsc/policy.json +91 -0
- package/packs/cpsc/source.md +76 -0
- package/packs/dhs-ohss/endpoints.json +66 -66
- package/packs/dhs-ohss/policy.json +107 -107
- package/packs/dhs-ohss/source.md +1 -1
- package/packs/doj-news/endpoints.json +59 -59
- package/packs/doj-news/policy.json +96 -96
- package/packs/doj-news/source.md +1 -1
- package/packs/dol/endpoints.json +817 -817
- package/packs/dol/policy.json +135 -135
- package/packs/eia/endpoints.json +717 -717
- package/packs/eia/policy.json +112 -112
- package/packs/epa/endpoints.json +291 -291
- package/packs/epa/policy.json +99 -99
- package/packs/epa-aqs/endpoints.json +173 -173
- package/packs/epa-aqs/policy.json +96 -96
- package/packs/fbi/endpoints.json +1841 -1841
- package/packs/fbi/policy.json +118 -118
- package/packs/fbi/source.md +3 -3
- package/packs/fda/endpoints.json +237 -237
- package/packs/fda/policy.json +121 -121
- package/packs/fdic/endpoints.json +416 -416
- package/packs/fdic/policy.json +83 -83
- package/packs/fec/endpoints.json +707 -707
- package/packs/fec/policy.json +93 -93
- package/packs/fed-board/endpoints.json +89 -89
- package/packs/fed-board/policy.json +136 -136
- package/packs/federal-register/endpoints.json +143 -143
- package/packs/federal-register/policy.json +86 -86
- package/packs/fema/endpoints.json +645 -645
- package/packs/fema/policy.json +103 -103
- package/packs/fhfa/endpoints.json +105 -105
- package/packs/fhfa/policy.json +109 -109
- package/packs/fhwa/endpoints.json +48 -0
- package/packs/fhwa/policy.json +91 -0
- package/packs/fhwa/source.md +59 -0
- package/packs/foreign-assistance/endpoints.json +166 -0
- package/packs/foreign-assistance/policy.json +80 -0
- package/packs/foreign-assistance/source.md +76 -0
- package/packs/fred/assets/download_fred.ps1 +122 -122
- package/packs/fred/docs/analysis_patterns.md +124 -124
- package/packs/fred/docs/fred_api.md +94 -94
- package/packs/fred/docs/series.md +188 -188
- package/packs/fred/policy.json +253 -253
- package/packs/fsa/endpoints.json +150 -150
- package/packs/fsa/policy.json +99 -99
- package/packs/fsa/source.md +1 -1
- package/packs/govinfo/endpoints.json +519 -519
- package/packs/gsa-calc/endpoints.json +93 -93
- package/packs/gsa-calc/policy.json +97 -97
- package/packs/hud/endpoints.json +146 -146
- package/packs/hud/policy.json +84 -84
- package/packs/hud/source.md +1 -1
- package/packs/hud-ahar/endpoints.json +92 -0
- package/packs/hud-ahar/policy.json +88 -0
- package/packs/hud-ahar/source.md +72 -0
- package/packs/ipums/docs/report_patterns.md +117 -117
- package/packs/irs-soi/endpoints.json +114 -114
- package/packs/irs-soi/policy.json +107 -107
- package/packs/medicaid/endpoints.json +412 -412
- package/packs/medicaid/policy.json +108 -108
- package/packs/naep/endpoints.json +217 -217
- package/packs/naep/policy.json +100 -100
- package/packs/naep/source.md +1 -1
- package/packs/nces/endpoints.json +101 -0
- package/packs/nces/policy.json +99 -0
- package/packs/nces/source.md +75 -0
- package/packs/nhtsa/endpoints.json +132 -132
- package/packs/nhtsa/policy.json +93 -93
- package/packs/nhtsa/source.md +1 -1
- package/packs/nih/endpoints.json +452 -452
- package/packs/nih/policy.json +86 -86
- package/packs/noaa/endpoints.json +431 -431
- package/packs/noaa/policy.json +95 -95
- package/packs/nrel/endpoints.json +327 -327
- package/packs/nrel/policy.json +96 -96
- package/packs/nsf/endpoints.json +33 -0
- package/packs/nsf/policy.json +83 -0
- package/packs/nsf/source.md +59 -0
- package/packs/nws/endpoints.json +271 -271
- package/packs/nws/policy.json +82 -82
- package/packs/nyfed/endpoints.json +80 -80
- package/packs/nyfed/policy.json +92 -92
- package/packs/omb/endpoints.json +460 -460
- package/packs/omb/policy.json +106 -106
- package/packs/onrr/endpoints.json +198 -0
- package/packs/onrr/policy.json +91 -0
- package/packs/onrr/source.md +55 -0
- package/packs/open-payments/endpoints.json +297 -297
- package/packs/open-payments/policy.json +101 -101
- package/packs/opm/assets/download_opm.ps1 +125 -125
- package/packs/opm/docs/fields.md +184 -184
- package/packs/opm/docs/opm_api.md +71 -71
- package/packs/opm/docs/report_patterns.md +83 -83
- package/packs/opm/lib/data_dictionary.csv +73 -73
- package/packs/pbgc/endpoints.json +483 -483
- package/packs/pbgc/policy.json +94 -94
- package/packs/philfed/endpoints.json +85 -85
- package/packs/philfed/policy.json +94 -94
- package/packs/port.json +65 -0
- package/packs/regulations/endpoints.json +287 -287
- package/packs/regulations/policy.json +86 -86
- package/packs/sba/endpoints.json +34 -0
- package/packs/sba/policy.json +86 -0
- package/packs/sba/source.md +60 -0
- package/packs/sec/endpoints.json +236 -236
- package/packs/sec/policy.json +86 -86
- package/packs/senate-lobbying/endpoints.json +553 -553
- package/packs/senate-lobbying/policy.json +99 -99
- package/packs/snapqc/endpoints.json +1426 -1426
- package/packs/snapqc/policy.json +92 -92
- package/packs/treasury/catalog.json +2267 -2267
- package/packs/treasury/docs/api_guide.md +245 -245
- package/packs/treasury/docs/datasets.md +56 -56
- package/packs/treasury/docs/report_patterns.md +126 -126
- package/packs/treasury/policy.json +159 -159
- package/packs/treasury/source.md +170 -165
- package/packs/treasury-tic/endpoints.json +77 -0
- package/packs/treasury-tic/policy.json +89 -0
- package/packs/treasury-tic/source.md +80 -0
- package/packs/treasury-yields/endpoints.json +226 -226
- package/packs/treasury-yields/policy.json +86 -86
- package/packs/usaspending/endpoints.json +1315 -1315
- package/packs/usaspending/policy.json +109 -109
- package/packs/uscourts/endpoints.json +22 -0
- package/packs/uscourts/policy.json +87 -0
- package/packs/uscourts/source.md +56 -0
- package/packs/usda-fna/policy.json +133 -133
- package/packs/usda-fooddata/endpoints.json +180 -180
- package/packs/usda-fooddata/policy.json +94 -94
- package/packs/usda-nass/endpoints.json +323 -323
- package/packs/usda-nass/policy.json +95 -95
- package/packs/usda-outlook/endpoints.json +86 -86
- package/packs/usda-outlook/policy.json +109 -109
- package/packs/usgs/endpoints.json +313 -313
- package/packs/usgs/policy.json +95 -95
- package/packs/usitc/endpoints.json +74 -74
- package/packs/usitc/policy.json +111 -111
- package/packs/uspto/endpoints.json +141 -141
- package/packs/uspto/policy.json +95 -95
- package/packs/uspto/source.md +1 -1
- package/packs/va/endpoints.json +414 -414
- package/packs/va/policy.json +101 -101
- package/packs/world-bank/endpoints.json +122 -122
- package/packs/world-bank/policy.json +101 -101
|
@@ -1,101 +1,101 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "open-payments",
|
|
3
|
-
"title": "CMS Open Payments (openpaymentsdata.cms.gov)",
|
|
4
|
-
"agency": "Centers for Medicare & Medicaid Services",
|
|
5
|
-
"depth": "standard",
|
|
6
|
-
"domains": [
|
|
7
|
-
"health",
|
|
8
|
-
"health care",
|
|
9
|
-
"industry payments"
|
|
10
|
-
],
|
|
11
|
-
"keywords": [
|
|
12
|
-
"open payments",
|
|
13
|
-
"sunshine act",
|
|
14
|
-
"industry payments",
|
|
15
|
-
"pharma payments",
|
|
16
|
-
"payments to doctors",
|
|
17
|
-
"physician payments",
|
|
18
|
-
"drug company",
|
|
19
|
-
"device company",
|
|
20
|
-
"manufacturer",
|
|
21
|
-
"consulting fee",
|
|
22
|
-
"speaking fee",
|
|
23
|
-
"food and beverage",
|
|
24
|
-
"royalty",
|
|
25
|
-
"research payments",
|
|
26
|
-
"ownership interest",
|
|
27
|
-
"teaching hospital",
|
|
28
|
-
"npi"
|
|
29
|
-
],
|
|
30
|
-
"questions": [
|
|
31
|
-
"payments from drug and device companies to doctors in a state",
|
|
32
|
-
"which companies paid the most to physicians",
|
|
33
|
-
"payments to a specific physician by NPI",
|
|
34
|
-
"industry payments by specialty",
|
|
35
|
-
"research payments to teaching hospitals"
|
|
36
|
-
],
|
|
37
|
-
"homepage": "https://openpaymentsdata.cms.gov/",
|
|
38
|
-
"terms_url": "https://www.usa.gov/government-works",
|
|
39
|
-
"auth": {
|
|
40
|
-
"env": null,
|
|
41
|
-
"required": "none",
|
|
42
|
-
"placement": "none",
|
|
43
|
-
"signup_url": null,
|
|
44
|
-
"echoes_key": false
|
|
45
|
-
},
|
|
46
|
-
"limits": {
|
|
47
|
-
"rps": 2,
|
|
48
|
-
"burst": 4,
|
|
49
|
-
"max_rows": 100000
|
|
50
|
-
},
|
|
51
|
-
"routes": {
|
|
52
|
-
"api": {
|
|
53
|
-
"base": "https://openpaymentsdata.cms.gov/api/1",
|
|
54
|
-
"probe": "https://openpaymentsdata.cms.gov/api/1/metastore/schemas/dataset/items/02157882-0f43-473a-a777-c96eb5b1b623",
|
|
55
|
-
"hosts": [
|
|
56
|
-
"openpaymentsdata.cms.gov"
|
|
57
|
-
]
|
|
58
|
-
},
|
|
59
|
-
"manual": {
|
|
60
|
-
"how": "Open the JSON links in INSTRUCTIONS.md (filters applied, 500 rows a page), save each under the shown name and upload; or search https://openpaymentsdata.cms.gov/ (Search > by physician, hospital or company) and export the results. Full-year payment files are several GB: download them from the dataset page only on a computer with room, then upload a filtered extract."
|
|
61
|
-
}
|
|
62
|
-
},
|
|
63
|
-
"cache": {
|
|
64
|
-
"mode": "run_only",
|
|
65
|
-
"ttl_hours": null,
|
|
66
|
-
"why": "Filtered queries are small; every dataset is republished when CMS refreshes the program years (all were last modified 2026-06-30). data/ is reused within a run."
|
|
67
|
-
},
|
|
68
|
-
"redistribution": {
|
|
69
|
-
"mode": "allowed",
|
|
70
|
-
"why": "Open Payments datasets carry the licence https://www.usa.gov/government-works (U.S. government work, public domain, 17 U.S.C. 105) in the CMS metastore."
|
|
71
|
-
},
|
|
72
|
-
"notices": [
|
|
73
|
-
"Open Payments records are reported by manufacturers and group purchasing organizations and published as reported (recipients may dispute them); a payment does not by itself indicate a conflict of interest or wrongdoing."
|
|
74
|
-
],
|
|
75
|
-
"retention": null,
|
|
76
|
-
"citation": "Centers for Medicare & Medicaid Services, Open Payments: {dataset}, openpaymentsdata.cms.gov, retrieved {retrieved_long}.",
|
|
77
|
-
"audit_hooks": [
|
|
78
|
-
{
|
|
79
|
-
"id": "op_not_wrongdoing",
|
|
80
|
-
"kind": "require_text",
|
|
81
|
-
"any": [
|
|
82
|
-
"not verified",
|
|
83
|
-
"does not indicate",
|
|
84
|
-
"not necessarily",
|
|
85
|
-
"conflict of interest",
|
|
86
|
-
"wrongdoing",
|
|
87
|
-
"reported by"
|
|
88
|
-
],
|
|
89
|
-
"why": "Reports using Open Payments must say that payments are self-reported by industry and do not imply wrongdoing."
|
|
90
|
-
}
|
|
91
|
-
],
|
|
92
|
-
"clarify": [
|
|
93
|
-
"Which program year(s)? Each year is a separate file; the latest published year is 2025 (published June 2026).",
|
|
94
|
-
"General payments only, or also research payments and ownership interests? They are separate datasets and are not additive with the summaries.",
|
|
95
|
-
"Physicians only, or also nurse practitioners and other non-physician practitioners (reported since program year 2021) and teaching hospitals?"
|
|
96
|
-
],
|
|
97
|
-
"formats": [
|
|
98
|
-
"json",
|
|
99
|
-
"csv"
|
|
100
|
-
]
|
|
101
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "open-payments",
|
|
3
|
+
"title": "CMS Open Payments (openpaymentsdata.cms.gov)",
|
|
4
|
+
"agency": "Centers for Medicare & Medicaid Services",
|
|
5
|
+
"depth": "standard",
|
|
6
|
+
"domains": [
|
|
7
|
+
"health",
|
|
8
|
+
"health care",
|
|
9
|
+
"industry payments"
|
|
10
|
+
],
|
|
11
|
+
"keywords": [
|
|
12
|
+
"open payments",
|
|
13
|
+
"sunshine act",
|
|
14
|
+
"industry payments",
|
|
15
|
+
"pharma payments",
|
|
16
|
+
"payments to doctors",
|
|
17
|
+
"physician payments",
|
|
18
|
+
"drug company",
|
|
19
|
+
"device company",
|
|
20
|
+
"manufacturer",
|
|
21
|
+
"consulting fee",
|
|
22
|
+
"speaking fee",
|
|
23
|
+
"food and beverage",
|
|
24
|
+
"royalty",
|
|
25
|
+
"research payments",
|
|
26
|
+
"ownership interest",
|
|
27
|
+
"teaching hospital",
|
|
28
|
+
"npi"
|
|
29
|
+
],
|
|
30
|
+
"questions": [
|
|
31
|
+
"payments from drug and device companies to doctors in a state",
|
|
32
|
+
"which companies paid the most to physicians",
|
|
33
|
+
"payments to a specific physician by NPI",
|
|
34
|
+
"industry payments by specialty",
|
|
35
|
+
"research payments to teaching hospitals"
|
|
36
|
+
],
|
|
37
|
+
"homepage": "https://openpaymentsdata.cms.gov/",
|
|
38
|
+
"terms_url": "https://www.usa.gov/government-works",
|
|
39
|
+
"auth": {
|
|
40
|
+
"env": null,
|
|
41
|
+
"required": "none",
|
|
42
|
+
"placement": "none",
|
|
43
|
+
"signup_url": null,
|
|
44
|
+
"echoes_key": false
|
|
45
|
+
},
|
|
46
|
+
"limits": {
|
|
47
|
+
"rps": 2,
|
|
48
|
+
"burst": 4,
|
|
49
|
+
"max_rows": 100000
|
|
50
|
+
},
|
|
51
|
+
"routes": {
|
|
52
|
+
"api": {
|
|
53
|
+
"base": "https://openpaymentsdata.cms.gov/api/1",
|
|
54
|
+
"probe": "https://openpaymentsdata.cms.gov/api/1/metastore/schemas/dataset/items/02157882-0f43-473a-a777-c96eb5b1b623",
|
|
55
|
+
"hosts": [
|
|
56
|
+
"openpaymentsdata.cms.gov"
|
|
57
|
+
]
|
|
58
|
+
},
|
|
59
|
+
"manual": {
|
|
60
|
+
"how": "Open the JSON links in INSTRUCTIONS.md (filters applied, 500 rows a page), save each under the shown name and upload; or search https://openpaymentsdata.cms.gov/ (Search > by physician, hospital or company) and export the results. Full-year payment files are several GB: download them from the dataset page only on a computer with room, then upload a filtered extract."
|
|
61
|
+
}
|
|
62
|
+
},
|
|
63
|
+
"cache": {
|
|
64
|
+
"mode": "run_only",
|
|
65
|
+
"ttl_hours": null,
|
|
66
|
+
"why": "Filtered queries are small; every dataset is republished when CMS refreshes the program years (all were last modified 2026-06-30). data/ is reused within a run."
|
|
67
|
+
},
|
|
68
|
+
"redistribution": {
|
|
69
|
+
"mode": "allowed",
|
|
70
|
+
"why": "Open Payments datasets carry the licence https://www.usa.gov/government-works (U.S. government work, public domain, 17 U.S.C. 105) in the CMS metastore."
|
|
71
|
+
},
|
|
72
|
+
"notices": [
|
|
73
|
+
"Open Payments records are reported by manufacturers and group purchasing organizations and published as reported (recipients may dispute them); a payment does not by itself indicate a conflict of interest or wrongdoing."
|
|
74
|
+
],
|
|
75
|
+
"retention": null,
|
|
76
|
+
"citation": "Centers for Medicare & Medicaid Services, Open Payments: {dataset}, openpaymentsdata.cms.gov, retrieved {retrieved_long}.",
|
|
77
|
+
"audit_hooks": [
|
|
78
|
+
{
|
|
79
|
+
"id": "op_not_wrongdoing",
|
|
80
|
+
"kind": "require_text",
|
|
81
|
+
"any": [
|
|
82
|
+
"not verified",
|
|
83
|
+
"does not indicate",
|
|
84
|
+
"not necessarily",
|
|
85
|
+
"conflict of interest",
|
|
86
|
+
"wrongdoing",
|
|
87
|
+
"reported by"
|
|
88
|
+
],
|
|
89
|
+
"why": "Reports using Open Payments must say that payments are self-reported by industry and do not imply wrongdoing."
|
|
90
|
+
}
|
|
91
|
+
],
|
|
92
|
+
"clarify": [
|
|
93
|
+
"Which program year(s)? Each year is a separate file; the latest published year is 2025 (published June 2026).",
|
|
94
|
+
"General payments only, or also research payments and ownership interests? They are separate datasets and are not additive with the summaries.",
|
|
95
|
+
"Physicians only, or also nurse practitioners and other non-physician practitioners (reported since program year 2021) and teaching hospitals?"
|
|
96
|
+
],
|
|
97
|
+
"formats": [
|
|
98
|
+
"json",
|
|
99
|
+
"csv"
|
|
100
|
+
]
|
|
101
|
+
}
|
|
@@ -1,125 +1,125 @@
|
|
|
1
|
-
<#
|
|
2
|
-
.SYNOPSIS
|
|
3
|
-
Download the OPM Federal Workforce Data files for this analysis into the folder that holds this script.
|
|
4
|
-
(Windows PowerShell counterpart of download_opm.sh.)
|
|
5
|
-
|
|
6
|
-
.DESCRIPTION
|
|
7
|
-
- The request is built into this script (written by data-analyst-mcp; do not edit it).
|
|
8
|
-
- It lists the files OPM publishes now (data.opm.gov, no account or key), picks the months the request
|
|
9
|
-
asks for, and saves each one as <dataset>_<YYYYMM>_<version>.parquet.
|
|
10
|
-
- Employment files are 26-75 MB each; accession and separation files are under 4 MB.
|
|
11
|
-
- Needs nothing beyond Windows PowerShell 5.1 or PowerShell 7. Run it again after an interruption:
|
|
12
|
-
files already saved are kept.
|
|
13
|
-
|
|
14
|
-
.PARAMETER NoPause
|
|
15
|
-
Do not wait for Enter before closing the window.
|
|
16
|
-
|
|
17
|
-
.EXAMPLE
|
|
18
|
-
powershell -NoProfile -ExecutionPolicy Bypass -File .\download_opm.ps1
|
|
19
|
-
#>
|
|
20
|
-
[CmdletBinding()]
|
|
21
|
-
param([switch]$NoPause)
|
|
22
|
-
|
|
23
|
-
$ErrorActionPreference = 'Stop'
|
|
24
|
-
$ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
|
|
25
|
-
try {
|
|
26
|
-
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
|
27
|
-
} catch { }
|
|
28
|
-
|
|
29
|
-
$Api = 'https://data.opm.gov/api/v1/files'
|
|
30
|
-
if ($env:OPM_API_BASE) { $Api = $env:OPM_API_BASE.TrimEnd('/') }
|
|
31
|
-
|
|
32
|
-
# The request. Columns, tab-separated: dataset, then either
|
|
33
|
-
# periods YYYYMM,... (or "latest") the listed months
|
|
34
|
-
# range FROM|first TO|latest MONTHS every month from FROM to TO in MONTHS (1-12)
|
|
35
|
-
# pin YYYYMM VERSION use this version of one month instead of the current one
|
|
36
|
-
$Rules = @'
|
|
37
|
-
@RULES@
|
|
38
|
-
'@
|
|
39
|
-
|
|
40
|
-
$Here = $PSScriptRoot
|
|
41
|
-
if (-not $Here) { $Here = (Get-Location).Path }
|
|
42
|
-
|
|
43
|
-
function Finish([int]$Code) {
|
|
44
|
-
if (-not $NoPause -and -not [Console]::IsInputRedirected) {
|
|
45
|
-
Write-Host ''
|
|
46
|
-
[void](Read-Host -Prompt 'Press Enter to close')
|
|
47
|
-
}
|
|
48
|
-
exit $Code
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
function Fail([string]$Message) {
|
|
52
|
-
Write-Host "ERROR: $Message" -ForegroundColor Red
|
|
53
|
-
Finish 1
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
$rows = @(foreach ($line in ($Rules -split "`r?`n")) {
|
|
57
|
-
if (-not $line.Trim()) { continue }
|
|
58
|
-
$c = $line -split "`t"
|
|
59
|
-
[pscustomobject]@{ Ds = $c[0]; Kind = $c[1]; A = $c[2]; B = $c[3]; C = $c[4] }
|
|
60
|
-
})
|
|
61
|
-
$saved = @(); $failed = 0; $missing = @()
|
|
62
|
-
foreach ($ds in @($rows | ForEach-Object { $_.Ds } | Sort-Object -Unique)) {
|
|
63
|
-
Write-Host "Listing the $ds files..."
|
|
64
|
-
try {
|
|
65
|
-
$reply = Invoke-RestMethod -Uri "$Api/$($ds)?current=true" -UseBasicParsing -UserAgent 'data-analyst/1.0'
|
|
66
|
-
if ($reply -is [string]) { $reply = $reply | ConvertFrom-Json } # served without a JSON content type
|
|
67
|
-
$list = @($reply | ForEach-Object { $_ })
|
|
68
|
-
} catch {
|
|
69
|
-
Fail "Could not list the $ds files at $Api ($($_.Exception.Message)). Check the connection, VPN and proxy, then run again."
|
|
70
|
-
}
|
|
71
|
-
$avail = @{}
|
|
72
|
-
foreach ($f in $list) {
|
|
73
|
-
if ([string]$f.filename -match "^$($ds)_(\d{6})_(\d+)$") { $avail[$Matches[1]] = $Matches[2] }
|
|
74
|
-
}
|
|
75
|
-
if ($avail.Count -eq 0) { Fail "OPM listed no $ds files. Run again later." }
|
|
76
|
-
$periods = @($avail.Keys | Sort-Object)
|
|
77
|
-
$first = $periods[0]; $latest = $periods[-1]
|
|
78
|
-
$want = @()
|
|
79
|
-
foreach ($r in @($rows | Where-Object { $_.Ds -eq $ds })) {
|
|
80
|
-
if ($r.Kind -eq 'periods') {
|
|
81
|
-
foreach ($p in $r.A.Split(',')) {
|
|
82
|
-
if ($p -eq 'latest') { $p = $latest }
|
|
83
|
-
if ($avail.ContainsKey($p)) { $want += $p } else { $missing += "$ds $p" }
|
|
84
|
-
}
|
|
85
|
-
} elseif ($r.Kind -eq 'range') {
|
|
86
|
-
$lo = $r.A; $hi = $r.B
|
|
87
|
-
if ($lo -eq 'first') { $lo = $first }
|
|
88
|
-
if ($hi -eq 'latest') { $hi = $latest }
|
|
89
|
-
$months = @($r.C.Split(',') | ForEach-Object { [int]$_ })
|
|
90
|
-
foreach ($p in $periods) {
|
|
91
|
-
if ($p -ge $lo -and $p -le $hi -and $months -contains [int]$p.Substring(4, 2)) { $want += $p }
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
foreach ($p in @($want | Sort-Object -Unique)) {
|
|
96
|
-
$v = $avail[$p]
|
|
97
|
-
$pin = @($rows | Where-Object { $_.Ds -eq $ds -and $_.Kind -eq 'pin' -and $_.A -eq $p })
|
|
98
|
-
if ($pin.Count -gt 0) { $v = $pin[0].B }
|
|
99
|
-
$name = "$($ds)_$($p)_$($v).parquet"
|
|
100
|
-
$dest = Join-Path $Here $name
|
|
101
|
-
if ((Test-Path -LiteralPath $dest) -and (Get-Item -LiteralPath $dest).Length -gt 0) {
|
|
102
|
-
Write-Host "already saved: $name"; $saved += $dest; continue
|
|
103
|
-
}
|
|
104
|
-
Write-Host "Downloading $name ..."
|
|
105
|
-
try {
|
|
106
|
-
Invoke-WebRequest -Uri "$Api/$ds/$($p.Substring(0, 4))/$($p.Substring(4, 2))/$v/download" -OutFile "$dest.part" `
|
|
107
|
-
-UseBasicParsing -UserAgent 'data-analyst/1.0'
|
|
108
|
-
Move-Item -LiteralPath "$dest.part" -Destination $dest -Force
|
|
109
|
-
$saved += $dest
|
|
110
|
-
} catch {
|
|
111
|
-
if (Test-Path -LiteralPath "$dest.part") { Remove-Item -LiteralPath "$dest.part" -Force }
|
|
112
|
-
Write-Host "FAILED $name ($($_.Exception.Message))" -ForegroundColor Red
|
|
113
|
-
$failed++
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
Write-Host ''
|
|
119
|
-
if ($missing.Count -gt 0) { Write-Host "Not published by OPM (skipped): $($missing -join ', ')" }
|
|
120
|
-
if ($saved.Count -gt 0) {
|
|
121
|
-
Write-Host 'Done. Upload these files to the chat (zip them first if they are large; do not rename them):' -ForegroundColor Green
|
|
122
|
-
foreach ($f in $saved) { Write-Host " $f" }
|
|
123
|
-
}
|
|
124
|
-
if ($failed -gt 0) { Write-Host "$failed file(s) failed: run this script again." -ForegroundColor Red; Finish 1 }
|
|
125
|
-
Finish 0
|
|
1
|
+
<#
|
|
2
|
+
.SYNOPSIS
|
|
3
|
+
Download the OPM Federal Workforce Data files for this analysis into the folder that holds this script.
|
|
4
|
+
(Windows PowerShell counterpart of download_opm.sh.)
|
|
5
|
+
|
|
6
|
+
.DESCRIPTION
|
|
7
|
+
- The request is built into this script (written by data-analyst-mcp; do not edit it).
|
|
8
|
+
- It lists the files OPM publishes now (data.opm.gov, no account or key), picks the months the request
|
|
9
|
+
asks for, and saves each one as <dataset>_<YYYYMM>_<version>.parquet.
|
|
10
|
+
- Employment files are 26-75 MB each; accession and separation files are under 4 MB.
|
|
11
|
+
- Needs nothing beyond Windows PowerShell 5.1 or PowerShell 7. Run it again after an interruption:
|
|
12
|
+
files already saved are kept.
|
|
13
|
+
|
|
14
|
+
.PARAMETER NoPause
|
|
15
|
+
Do not wait for Enter before closing the window.
|
|
16
|
+
|
|
17
|
+
.EXAMPLE
|
|
18
|
+
powershell -NoProfile -ExecutionPolicy Bypass -File .\download_opm.ps1
|
|
19
|
+
#>
|
|
20
|
+
[CmdletBinding()]
|
|
21
|
+
param([switch]$NoPause)
|
|
22
|
+
|
|
23
|
+
$ErrorActionPreference = 'Stop'
|
|
24
|
+
$ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
|
|
25
|
+
try {
|
|
26
|
+
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
|
27
|
+
} catch { }
|
|
28
|
+
|
|
29
|
+
$Api = 'https://data.opm.gov/api/v1/files'
|
|
30
|
+
if ($env:OPM_API_BASE) { $Api = $env:OPM_API_BASE.TrimEnd('/') }
|
|
31
|
+
|
|
32
|
+
# The request. Columns, tab-separated: dataset, then either
|
|
33
|
+
# periods YYYYMM,... (or "latest") the listed months
|
|
34
|
+
# range FROM|first TO|latest MONTHS every month from FROM to TO in MONTHS (1-12)
|
|
35
|
+
# pin YYYYMM VERSION use this version of one month instead of the current one
|
|
36
|
+
$Rules = @'
|
|
37
|
+
@RULES@
|
|
38
|
+
'@
|
|
39
|
+
|
|
40
|
+
$Here = $PSScriptRoot
|
|
41
|
+
if (-not $Here) { $Here = (Get-Location).Path }
|
|
42
|
+
|
|
43
|
+
function Finish([int]$Code) {
|
|
44
|
+
if (-not $NoPause -and -not [Console]::IsInputRedirected) {
|
|
45
|
+
Write-Host ''
|
|
46
|
+
[void](Read-Host -Prompt 'Press Enter to close')
|
|
47
|
+
}
|
|
48
|
+
exit $Code
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function Fail([string]$Message) {
|
|
52
|
+
Write-Host "ERROR: $Message" -ForegroundColor Red
|
|
53
|
+
Finish 1
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
$rows = @(foreach ($line in ($Rules -split "`r?`n")) {
|
|
57
|
+
if (-not $line.Trim()) { continue }
|
|
58
|
+
$c = $line -split "`t"
|
|
59
|
+
[pscustomobject]@{ Ds = $c[0]; Kind = $c[1]; A = $c[2]; B = $c[3]; C = $c[4] }
|
|
60
|
+
})
|
|
61
|
+
$saved = @(); $failed = 0; $missing = @()
|
|
62
|
+
foreach ($ds in @($rows | ForEach-Object { $_.Ds } | Sort-Object -Unique)) {
|
|
63
|
+
Write-Host "Listing the $ds files..."
|
|
64
|
+
try {
|
|
65
|
+
$reply = Invoke-RestMethod -Uri "$Api/$($ds)?current=true" -UseBasicParsing -UserAgent 'data-analyst/1.0'
|
|
66
|
+
if ($reply -is [string]) { $reply = $reply | ConvertFrom-Json } # served without a JSON content type
|
|
67
|
+
$list = @($reply | ForEach-Object { $_ })
|
|
68
|
+
} catch {
|
|
69
|
+
Fail "Could not list the $ds files at $Api ($($_.Exception.Message)). Check the connection, VPN and proxy, then run again."
|
|
70
|
+
}
|
|
71
|
+
$avail = @{}
|
|
72
|
+
foreach ($f in $list) {
|
|
73
|
+
if ([string]$f.filename -match "^$($ds)_(\d{6})_(\d+)$") { $avail[$Matches[1]] = $Matches[2] }
|
|
74
|
+
}
|
|
75
|
+
if ($avail.Count -eq 0) { Fail "OPM listed no $ds files. Run again later." }
|
|
76
|
+
$periods = @($avail.Keys | Sort-Object)
|
|
77
|
+
$first = $periods[0]; $latest = $periods[-1]
|
|
78
|
+
$want = @()
|
|
79
|
+
foreach ($r in @($rows | Where-Object { $_.Ds -eq $ds })) {
|
|
80
|
+
if ($r.Kind -eq 'periods') {
|
|
81
|
+
foreach ($p in $r.A.Split(',')) {
|
|
82
|
+
if ($p -eq 'latest') { $p = $latest }
|
|
83
|
+
if ($avail.ContainsKey($p)) { $want += $p } else { $missing += "$ds $p" }
|
|
84
|
+
}
|
|
85
|
+
} elseif ($r.Kind -eq 'range') {
|
|
86
|
+
$lo = $r.A; $hi = $r.B
|
|
87
|
+
if ($lo -eq 'first') { $lo = $first }
|
|
88
|
+
if ($hi -eq 'latest') { $hi = $latest }
|
|
89
|
+
$months = @($r.C.Split(',') | ForEach-Object { [int]$_ })
|
|
90
|
+
foreach ($p in $periods) {
|
|
91
|
+
if ($p -ge $lo -and $p -le $hi -and $months -contains [int]$p.Substring(4, 2)) { $want += $p }
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
foreach ($p in @($want | Sort-Object -Unique)) {
|
|
96
|
+
$v = $avail[$p]
|
|
97
|
+
$pin = @($rows | Where-Object { $_.Ds -eq $ds -and $_.Kind -eq 'pin' -and $_.A -eq $p })
|
|
98
|
+
if ($pin.Count -gt 0) { $v = $pin[0].B }
|
|
99
|
+
$name = "$($ds)_$($p)_$($v).parquet"
|
|
100
|
+
$dest = Join-Path $Here $name
|
|
101
|
+
if ((Test-Path -LiteralPath $dest) -and (Get-Item -LiteralPath $dest).Length -gt 0) {
|
|
102
|
+
Write-Host "already saved: $name"; $saved += $dest; continue
|
|
103
|
+
}
|
|
104
|
+
Write-Host "Downloading $name ..."
|
|
105
|
+
try {
|
|
106
|
+
Invoke-WebRequest -Uri "$Api/$ds/$($p.Substring(0, 4))/$($p.Substring(4, 2))/$v/download" -OutFile "$dest.part" `
|
|
107
|
+
-UseBasicParsing -UserAgent 'data-analyst/1.0'
|
|
108
|
+
Move-Item -LiteralPath "$dest.part" -Destination $dest -Force
|
|
109
|
+
$saved += $dest
|
|
110
|
+
} catch {
|
|
111
|
+
if (Test-Path -LiteralPath "$dest.part") { Remove-Item -LiteralPath "$dest.part" -Force }
|
|
112
|
+
Write-Host "FAILED $name ($($_.Exception.Message))" -ForegroundColor Red
|
|
113
|
+
$failed++
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
Write-Host ''
|
|
119
|
+
if ($missing.Count -gt 0) { Write-Host "Not published by OPM (skipped): $($missing -join ', ')" }
|
|
120
|
+
if ($saved.Count -gt 0) {
|
|
121
|
+
Write-Host 'Done. Upload these files to the chat (zip them first if they are large; do not rename them):' -ForegroundColor Green
|
|
122
|
+
foreach ($f in $saved) { Write-Host " $f" }
|
|
123
|
+
}
|
|
124
|
+
if ($failed -gt 0) { Write-Host "$failed file(s) failed: run this script again." -ForegroundColor Red; Finish 1 }
|
|
125
|
+
Finish 0
|