data-analyst-mcp 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/README.md +262 -262
  2. package/dist/report/method_audit.js +7 -1
  3. package/dist/report/method_audit.js.map +1 -1
  4. package/dist/report/scan_secrets.d.ts +4 -1
  5. package/dist/report/scan_secrets.js +36 -5
  6. package/dist/report/scan_secrets.js.map +1 -1
  7. package/dist/server.d.ts +1 -1
  8. package/dist/server.js +3 -2
  9. package/dist/server.js.map +1 -1
  10. package/guide/WORKFLOW.md +134 -134
  11. package/guide/assets/disclaimer.txt +1 -1
  12. package/guide/assets/download_data.ps1 +162 -162
  13. package/guide/assets/report_template.html +597 -597
  14. package/guide/references/routing.md +171 -171
  15. package/guide/references/style_guide.md +398 -398
  16. package/package.json +56 -56
  17. package/packs/acf/endpoints.json +114 -114
  18. package/packs/acf/policy.json +104 -104
  19. package/packs/actuaries/endpoints.json +82 -82
  20. package/packs/actuaries/policy.json +109 -109
  21. package/packs/bea/assets/download_bea.ps1 +182 -182
  22. package/packs/bea/docs/bea_api.md +461 -461
  23. package/packs/bea/docs/datasets.md +157 -157
  24. package/packs/bea/docs/report_patterns.md +112 -112
  25. package/packs/bea/policy.json +170 -170
  26. package/packs/bea/source.md +152 -152
  27. package/packs/bls/assets/download_bls.ps1 +152 -152
  28. package/packs/bls/docs/analysis_patterns.md +108 -108
  29. package/packs/bls/docs/bls_api.md +73 -73
  30. package/packs/bls/docs/series_ids.md +149 -149
  31. package/packs/bls/policy.json +238 -238
  32. package/packs/bts/endpoints.json +346 -346
  33. package/packs/bts/policy.json +94 -94
  34. package/packs/cdc/endpoints.json +197 -197
  35. package/packs/cdc/policy.json +83 -83
  36. package/packs/census/endpoints.json +99 -99
  37. package/packs/cfpb/endpoints.json +326 -326
  38. package/packs/cfpb/policy.json +119 -119
  39. package/packs/clinical-trials/endpoints.json +166 -166
  40. package/packs/clinical-trials/policy.json +84 -84
  41. package/packs/cms/endpoints.json +69 -69
  42. package/packs/cms/policy.json +96 -96
  43. package/packs/college-scorecard/endpoints.json +252 -252
  44. package/packs/college-scorecard/policy.json +106 -106
  45. package/packs/congress/endpoints.json +1701 -1701
  46. package/packs/congress/policy.json +117 -117
  47. package/packs/dhs-ohss/endpoints.json +66 -66
  48. package/packs/dhs-ohss/policy.json +107 -107
  49. package/packs/doj-news/endpoints.json +59 -59
  50. package/packs/doj-news/policy.json +96 -96
  51. package/packs/dol/endpoints.json +817 -817
  52. package/packs/dol/policy.json +135 -135
  53. package/packs/eia/endpoints.json +717 -717
  54. package/packs/eia/policy.json +112 -112
  55. package/packs/epa/endpoints.json +291 -291
  56. package/packs/epa/policy.json +99 -99
  57. package/packs/epa-aqs/endpoints.json +173 -173
  58. package/packs/epa-aqs/policy.json +96 -96
  59. package/packs/fbi/endpoints.json +1841 -1841
  60. package/packs/fbi/policy.json +118 -118
  61. package/packs/fda/endpoints.json +237 -237
  62. package/packs/fda/policy.json +121 -121
  63. package/packs/fdic/endpoints.json +416 -416
  64. package/packs/fdic/policy.json +83 -83
  65. package/packs/fec/endpoints.json +707 -707
  66. package/packs/fec/policy.json +93 -93
  67. package/packs/fed-board/endpoints.json +89 -89
  68. package/packs/fed-board/policy.json +136 -136
  69. package/packs/federal-register/endpoints.json +143 -143
  70. package/packs/federal-register/policy.json +86 -86
  71. package/packs/fema/endpoints.json +645 -645
  72. package/packs/fema/policy.json +103 -103
  73. package/packs/fhfa/endpoints.json +105 -105
  74. package/packs/fhfa/policy.json +109 -109
  75. package/packs/fred/assets/download_fred.ps1 +122 -122
  76. package/packs/fred/docs/analysis_patterns.md +124 -124
  77. package/packs/fred/docs/fred_api.md +94 -94
  78. package/packs/fred/docs/series.md +188 -188
  79. package/packs/fred/policy.json +253 -253
  80. package/packs/fsa/endpoints.json +150 -150
  81. package/packs/fsa/policy.json +99 -99
  82. package/packs/govinfo/endpoints.json +519 -519
  83. package/packs/gsa-calc/endpoints.json +93 -93
  84. package/packs/gsa-calc/policy.json +97 -97
  85. package/packs/hud/endpoints.json +146 -146
  86. package/packs/hud/policy.json +84 -84
  87. package/packs/ipums/docs/report_patterns.md +117 -117
  88. package/packs/irs-soi/endpoints.json +114 -114
  89. package/packs/irs-soi/policy.json +107 -107
  90. package/packs/medicaid/endpoints.json +412 -412
  91. package/packs/medicaid/policy.json +108 -108
  92. package/packs/naep/endpoints.json +217 -217
  93. package/packs/naep/policy.json +100 -100
  94. package/packs/nhtsa/endpoints.json +132 -132
  95. package/packs/nhtsa/policy.json +93 -93
  96. package/packs/nih/endpoints.json +452 -452
  97. package/packs/nih/policy.json +86 -86
  98. package/packs/noaa/endpoints.json +431 -431
  99. package/packs/noaa/policy.json +95 -95
  100. package/packs/nrel/endpoints.json +327 -327
  101. package/packs/nrel/policy.json +96 -96
  102. package/packs/nws/endpoints.json +271 -271
  103. package/packs/nws/policy.json +82 -82
  104. package/packs/nyfed/endpoints.json +80 -80
  105. package/packs/nyfed/policy.json +92 -92
  106. package/packs/omb/endpoints.json +460 -460
  107. package/packs/omb/policy.json +106 -106
  108. package/packs/open-payments/endpoints.json +297 -297
  109. package/packs/open-payments/policy.json +101 -101
  110. package/packs/opm/assets/download_opm.ps1 +125 -125
  111. package/packs/opm/docs/fields.md +184 -184
  112. package/packs/opm/docs/opm_api.md +71 -71
  113. package/packs/opm/docs/report_patterns.md +83 -83
  114. package/packs/opm/lib/data_dictionary.csv +73 -73
  115. package/packs/pbgc/endpoints.json +483 -483
  116. package/packs/pbgc/policy.json +94 -94
  117. package/packs/philfed/endpoints.json +85 -85
  118. package/packs/philfed/policy.json +94 -94
  119. package/packs/regulations/endpoints.json +287 -287
  120. package/packs/regulations/policy.json +86 -86
  121. package/packs/sec/endpoints.json +236 -236
  122. package/packs/sec/policy.json +86 -86
  123. package/packs/senate-lobbying/endpoints.json +553 -553
  124. package/packs/senate-lobbying/policy.json +99 -99
  125. package/packs/snapqc/endpoints.json +1426 -1426
  126. package/packs/snapqc/policy.json +92 -92
  127. package/packs/treasury/catalog.json +2267 -2267
  128. package/packs/treasury/docs/api_guide.md +245 -245
  129. package/packs/treasury/docs/datasets.md +56 -56
  130. package/packs/treasury/docs/report_patterns.md +126 -126
  131. package/packs/treasury/policy.json +159 -159
  132. package/packs/treasury/source.md +170 -165
  133. package/packs/treasury-yields/endpoints.json +226 -226
  134. package/packs/treasury-yields/policy.json +86 -86
  135. package/packs/usaspending/endpoints.json +1315 -1315
  136. package/packs/usaspending/policy.json +109 -109
  137. package/packs/usda-fna/policy.json +133 -133
  138. package/packs/usda-fooddata/endpoints.json +180 -180
  139. package/packs/usda-fooddata/policy.json +94 -94
  140. package/packs/usda-nass/endpoints.json +323 -323
  141. package/packs/usda-nass/policy.json +95 -95
  142. package/packs/usda-outlook/endpoints.json +86 -86
  143. package/packs/usda-outlook/policy.json +109 -109
  144. package/packs/usgs/endpoints.json +313 -313
  145. package/packs/usgs/policy.json +95 -95
  146. package/packs/usitc/endpoints.json +74 -74
  147. package/packs/usitc/policy.json +111 -111
  148. package/packs/uspto/endpoints.json +141 -141
  149. package/packs/uspto/policy.json +95 -95
  150. package/packs/va/endpoints.json +414 -414
  151. package/packs/va/policy.json +101 -101
  152. package/packs/world-bank/endpoints.json +122 -122
  153. package/packs/world-bank/policy.json +101 -101
@@ -1,101 +1,101 @@
1
- {
2
- "id": "open-payments",
3
- "title": "CMS Open Payments (openpaymentsdata.cms.gov)",
4
- "agency": "Centers for Medicare & Medicaid Services",
5
- "depth": "standard",
6
- "domains": [
7
- "health",
8
- "health care",
9
- "industry payments"
10
- ],
11
- "keywords": [
12
- "open payments",
13
- "sunshine act",
14
- "industry payments",
15
- "pharma payments",
16
- "payments to doctors",
17
- "physician payments",
18
- "drug company",
19
- "device company",
20
- "manufacturer",
21
- "consulting fee",
22
- "speaking fee",
23
- "food and beverage",
24
- "royalty",
25
- "research payments",
26
- "ownership interest",
27
- "teaching hospital",
28
- "npi"
29
- ],
30
- "questions": [
31
- "payments from drug and device companies to doctors in a state",
32
- "which companies paid the most to physicians",
33
- "payments to a specific physician by NPI",
34
- "industry payments by specialty",
35
- "research payments to teaching hospitals"
36
- ],
37
- "homepage": "https://openpaymentsdata.cms.gov/",
38
- "terms_url": "https://www.usa.gov/government-works",
39
- "auth": {
40
- "env": null,
41
- "required": "none",
42
- "placement": "none",
43
- "signup_url": null,
44
- "echoes_key": false
45
- },
46
- "limits": {
47
- "rps": 2,
48
- "burst": 4,
49
- "max_rows": 100000
50
- },
51
- "routes": {
52
- "api": {
53
- "base": "https://openpaymentsdata.cms.gov/api/1",
54
- "probe": "https://openpaymentsdata.cms.gov/api/1/metastore/schemas/dataset/items/02157882-0f43-473a-a777-c96eb5b1b623",
55
- "hosts": [
56
- "openpaymentsdata.cms.gov"
57
- ]
58
- },
59
- "manual": {
60
- "how": "Open the JSON links in INSTRUCTIONS.md (filters applied, 500 rows a page), save each under the shown name and upload; or search https://openpaymentsdata.cms.gov/ (Search > by physician, hospital or company) and export the results. Full-year payment files are several GB: download them from the dataset page only on a computer with room, then upload a filtered extract."
61
- }
62
- },
63
- "cache": {
64
- "mode": "run_only",
65
- "ttl_hours": null,
66
- "why": "Filtered queries are small; every dataset is republished when CMS refreshes the program years (all were last modified 2026-06-30). data/ is reused within a run."
67
- },
68
- "redistribution": {
69
- "mode": "allowed",
70
- "why": "Open Payments datasets carry the licence https://www.usa.gov/government-works (U.S. government work, public domain, 17 U.S.C. 105) in the CMS metastore."
71
- },
72
- "notices": [
73
- "Open Payments records are reported by manufacturers and group purchasing organizations and published as reported (recipients may dispute them); a payment does not by itself indicate a conflict of interest or wrongdoing."
74
- ],
75
- "retention": null,
76
- "citation": "Centers for Medicare & Medicaid Services, Open Payments: {dataset}, openpaymentsdata.cms.gov, retrieved {retrieved_long}.",
77
- "audit_hooks": [
78
- {
79
- "id": "op_not_wrongdoing",
80
- "kind": "require_text",
81
- "any": [
82
- "not verified",
83
- "does not indicate",
84
- "not necessarily",
85
- "conflict of interest",
86
- "wrongdoing",
87
- "reported by"
88
- ],
89
- "why": "Reports using Open Payments must say that payments are self-reported by industry and do not imply wrongdoing."
90
- }
91
- ],
92
- "clarify": [
93
- "Which program year(s)? Each year is a separate file; the latest published year is 2025 (published June 2026).",
94
- "General payments only, or also research payments and ownership interests? They are separate datasets and are not additive with the summaries.",
95
- "Physicians only, or also nurse practitioners and other non-physician practitioners (reported since program year 2021) and teaching hospitals?"
96
- ],
97
- "formats": [
98
- "json",
99
- "csv"
100
- ]
101
- }
1
+ {
2
+ "id": "open-payments",
3
+ "title": "CMS Open Payments (openpaymentsdata.cms.gov)",
4
+ "agency": "Centers for Medicare & Medicaid Services",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "health",
8
+ "health care",
9
+ "industry payments"
10
+ ],
11
+ "keywords": [
12
+ "open payments",
13
+ "sunshine act",
14
+ "industry payments",
15
+ "pharma payments",
16
+ "payments to doctors",
17
+ "physician payments",
18
+ "drug company",
19
+ "device company",
20
+ "manufacturer",
21
+ "consulting fee",
22
+ "speaking fee",
23
+ "food and beverage",
24
+ "royalty",
25
+ "research payments",
26
+ "ownership interest",
27
+ "teaching hospital",
28
+ "npi"
29
+ ],
30
+ "questions": [
31
+ "payments from drug and device companies to doctors in a state",
32
+ "which companies paid the most to physicians",
33
+ "payments to a specific physician by NPI",
34
+ "industry payments by specialty",
35
+ "research payments to teaching hospitals"
36
+ ],
37
+ "homepage": "https://openpaymentsdata.cms.gov/",
38
+ "terms_url": "https://www.usa.gov/government-works",
39
+ "auth": {
40
+ "env": null,
41
+ "required": "none",
42
+ "placement": "none",
43
+ "signup_url": null,
44
+ "echoes_key": false
45
+ },
46
+ "limits": {
47
+ "rps": 2,
48
+ "burst": 4,
49
+ "max_rows": 100000
50
+ },
51
+ "routes": {
52
+ "api": {
53
+ "base": "https://openpaymentsdata.cms.gov/api/1",
54
+ "probe": "https://openpaymentsdata.cms.gov/api/1/metastore/schemas/dataset/items/02157882-0f43-473a-a777-c96eb5b1b623",
55
+ "hosts": [
56
+ "openpaymentsdata.cms.gov"
57
+ ]
58
+ },
59
+ "manual": {
60
+ "how": "Open the JSON links in INSTRUCTIONS.md (filters applied, 500 rows a page), save each under the shown name and upload; or search https://openpaymentsdata.cms.gov/ (Search > by physician, hospital or company) and export the results. Full-year payment files are several GB: download them from the dataset page only on a computer with room, then upload a filtered extract."
61
+ }
62
+ },
63
+ "cache": {
64
+ "mode": "run_only",
65
+ "ttl_hours": null,
66
+ "why": "Filtered queries are small; every dataset is republished when CMS refreshes the program years (all were last modified 2026-06-30). data/ is reused within a run."
67
+ },
68
+ "redistribution": {
69
+ "mode": "allowed",
70
+ "why": "Open Payments datasets carry the licence https://www.usa.gov/government-works (U.S. government work, public domain, 17 U.S.C. 105) in the CMS metastore."
71
+ },
72
+ "notices": [
73
+ "Open Payments records are reported by manufacturers and group purchasing organizations and published as reported (recipients may dispute them); a payment does not by itself indicate a conflict of interest or wrongdoing."
74
+ ],
75
+ "retention": null,
76
+ "citation": "Centers for Medicare & Medicaid Services, Open Payments: {dataset}, openpaymentsdata.cms.gov, retrieved {retrieved_long}.",
77
+ "audit_hooks": [
78
+ {
79
+ "id": "op_not_wrongdoing",
80
+ "kind": "require_text",
81
+ "any": [
82
+ "not verified",
83
+ "does not indicate",
84
+ "not necessarily",
85
+ "conflict of interest",
86
+ "wrongdoing",
87
+ "reported by"
88
+ ],
89
+ "why": "Reports using Open Payments must say that payments are self-reported by industry and do not imply wrongdoing."
90
+ }
91
+ ],
92
+ "clarify": [
93
+ "Which program year(s)? Each year is a separate file; the latest published year is 2025 (published June 2026).",
94
+ "General payments only, or also research payments and ownership interests? They are separate datasets and are not additive with the summaries.",
95
+ "Physicians only, or also nurse practitioners and other non-physician practitioners (reported since program year 2021) and teaching hospitals?"
96
+ ],
97
+ "formats": [
98
+ "json",
99
+ "csv"
100
+ ]
101
+ }
@@ -1,125 +1,125 @@
1
- <#
2
- .SYNOPSIS
3
- Download the OPM Federal Workforce Data files for this analysis into the folder that holds this script.
4
- (Windows PowerShell counterpart of download_opm.sh.)
5
-
6
- .DESCRIPTION
7
- - The request is built into this script (written by data-analyst-mcp; do not edit it).
8
- - It lists the files OPM publishes now (data.opm.gov, no account or key), picks the months the request
9
- asks for, and saves each one as <dataset>_<YYYYMM>_<version>.parquet.
10
- - Employment files are 26-75 MB each; accession and separation files are under 4 MB.
11
- - Needs nothing beyond Windows PowerShell 5.1 or PowerShell 7. Run it again after an interruption:
12
- files already saved are kept.
13
-
14
- .PARAMETER NoPause
15
- Do not wait for Enter before closing the window.
16
-
17
- .EXAMPLE
18
- powershell -NoProfile -ExecutionPolicy Bypass -File .\download_opm.ps1
19
- #>
20
- [CmdletBinding()]
21
- param([switch]$NoPause)
22
-
23
- $ErrorActionPreference = 'Stop'
24
- $ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
25
- try {
26
- [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
27
- } catch { }
28
-
29
- $Api = 'https://data.opm.gov/api/v1/files'
30
- if ($env:OPM_API_BASE) { $Api = $env:OPM_API_BASE.TrimEnd('/') }
31
-
32
- # The request. Columns, tab-separated: dataset, then either
33
- # periods YYYYMM,... (or "latest") the listed months
34
- # range FROM|first TO|latest MONTHS every month from FROM to TO in MONTHS (1-12)
35
- # pin YYYYMM VERSION use this version of one month instead of the current one
36
- $Rules = @'
37
- @RULES@
38
- '@
39
-
40
- $Here = $PSScriptRoot
41
- if (-not $Here) { $Here = (Get-Location).Path }
42
-
43
- function Finish([int]$Code) {
44
- if (-not $NoPause -and -not [Console]::IsInputRedirected) {
45
- Write-Host ''
46
- [void](Read-Host -Prompt 'Press Enter to close')
47
- }
48
- exit $Code
49
- }
50
-
51
- function Fail([string]$Message) {
52
- Write-Host "ERROR: $Message" -ForegroundColor Red
53
- Finish 1
54
- }
55
-
56
- $rows = @(foreach ($line in ($Rules -split "`r?`n")) {
57
- if (-not $line.Trim()) { continue }
58
- $c = $line -split "`t"
59
- [pscustomobject]@{ Ds = $c[0]; Kind = $c[1]; A = $c[2]; B = $c[3]; C = $c[4] }
60
- })
61
- $saved = @(); $failed = 0; $missing = @()
62
- foreach ($ds in @($rows | ForEach-Object { $_.Ds } | Sort-Object -Unique)) {
63
- Write-Host "Listing the $ds files..."
64
- try {
65
- $reply = Invoke-RestMethod -Uri "$Api/$($ds)?current=true" -UseBasicParsing -UserAgent 'data-analyst/1.0'
66
- if ($reply -is [string]) { $reply = $reply | ConvertFrom-Json } # served without a JSON content type
67
- $list = @($reply | ForEach-Object { $_ })
68
- } catch {
69
- Fail "Could not list the $ds files at $Api ($($_.Exception.Message)). Check the connection, VPN and proxy, then run again."
70
- }
71
- $avail = @{}
72
- foreach ($f in $list) {
73
- if ([string]$f.filename -match "^$($ds)_(\d{6})_(\d+)$") { $avail[$Matches[1]] = $Matches[2] }
74
- }
75
- if ($avail.Count -eq 0) { Fail "OPM listed no $ds files. Run again later." }
76
- $periods = @($avail.Keys | Sort-Object)
77
- $first = $periods[0]; $latest = $periods[-1]
78
- $want = @()
79
- foreach ($r in @($rows | Where-Object { $_.Ds -eq $ds })) {
80
- if ($r.Kind -eq 'periods') {
81
- foreach ($p in $r.A.Split(',')) {
82
- if ($p -eq 'latest') { $p = $latest }
83
- if ($avail.ContainsKey($p)) { $want += $p } else { $missing += "$ds $p" }
84
- }
85
- } elseif ($r.Kind -eq 'range') {
86
- $lo = $r.A; $hi = $r.B
87
- if ($lo -eq 'first') { $lo = $first }
88
- if ($hi -eq 'latest') { $hi = $latest }
89
- $months = @($r.C.Split(',') | ForEach-Object { [int]$_ })
90
- foreach ($p in $periods) {
91
- if ($p -ge $lo -and $p -le $hi -and $months -contains [int]$p.Substring(4, 2)) { $want += $p }
92
- }
93
- }
94
- }
95
- foreach ($p in @($want | Sort-Object -Unique)) {
96
- $v = $avail[$p]
97
- $pin = @($rows | Where-Object { $_.Ds -eq $ds -and $_.Kind -eq 'pin' -and $_.A -eq $p })
98
- if ($pin.Count -gt 0) { $v = $pin[0].B }
99
- $name = "$($ds)_$($p)_$($v).parquet"
100
- $dest = Join-Path $Here $name
101
- if ((Test-Path -LiteralPath $dest) -and (Get-Item -LiteralPath $dest).Length -gt 0) {
102
- Write-Host "already saved: $name"; $saved += $dest; continue
103
- }
104
- Write-Host "Downloading $name ..."
105
- try {
106
- Invoke-WebRequest -Uri "$Api/$ds/$($p.Substring(0, 4))/$($p.Substring(4, 2))/$v/download" -OutFile "$dest.part" `
107
- -UseBasicParsing -UserAgent 'data-analyst/1.0'
108
- Move-Item -LiteralPath "$dest.part" -Destination $dest -Force
109
- $saved += $dest
110
- } catch {
111
- if (Test-Path -LiteralPath "$dest.part") { Remove-Item -LiteralPath "$dest.part" -Force }
112
- Write-Host "FAILED $name ($($_.Exception.Message))" -ForegroundColor Red
113
- $failed++
114
- }
115
- }
116
- }
117
-
118
- Write-Host ''
119
- if ($missing.Count -gt 0) { Write-Host "Not published by OPM (skipped): $($missing -join ', ')" }
120
- if ($saved.Count -gt 0) {
121
- Write-Host 'Done. Upload these files to the chat (zip them first if they are large; do not rename them):' -ForegroundColor Green
122
- foreach ($f in $saved) { Write-Host " $f" }
123
- }
124
- if ($failed -gt 0) { Write-Host "$failed file(s) failed: run this script again." -ForegroundColor Red; Finish 1 }
125
- Finish 0
1
+ <#
2
+ .SYNOPSIS
3
+ Download the OPM Federal Workforce Data files for this analysis into the folder that holds this script.
4
+ (Windows PowerShell counterpart of download_opm.sh.)
5
+
6
+ .DESCRIPTION
7
+ - The request is built into this script (written by data-analyst-mcp; do not edit it).
8
+ - It lists the files OPM publishes now (data.opm.gov, no account or key), picks the months the request
9
+ asks for, and saves each one as <dataset>_<YYYYMM>_<version>.parquet.
10
+ - Employment files are 26-75 MB each; accession and separation files are under 4 MB.
11
+ - Needs nothing beyond Windows PowerShell 5.1 or PowerShell 7. Run it again after an interruption:
12
+ files already saved are kept.
13
+
14
+ .PARAMETER NoPause
15
+ Do not wait for Enter before closing the window.
16
+
17
+ .EXAMPLE
18
+ powershell -NoProfile -ExecutionPolicy Bypass -File .\download_opm.ps1
19
+ #>
20
+ [CmdletBinding()]
21
+ param([switch]$NoPause)
22
+
23
+ $ErrorActionPreference = 'Stop'
24
+ $ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
25
+ try {
26
+ [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
27
+ } catch { }
28
+
29
+ $Api = 'https://data.opm.gov/api/v1/files'
30
+ if ($env:OPM_API_BASE) { $Api = $env:OPM_API_BASE.TrimEnd('/') }
31
+
32
+ # The request. Columns, tab-separated: dataset, then either
33
+ # periods YYYYMM,... (or "latest") the listed months
34
+ # range FROM|first TO|latest MONTHS every month from FROM to TO in MONTHS (1-12)
35
+ # pin YYYYMM VERSION use this version of one month instead of the current one
36
+ $Rules = @'
37
+ @RULES@
38
+ '@
39
+
40
+ $Here = $PSScriptRoot
41
+ if (-not $Here) { $Here = (Get-Location).Path }
42
+
43
+ function Finish([int]$Code) {
44
+ if (-not $NoPause -and -not [Console]::IsInputRedirected) {
45
+ Write-Host ''
46
+ [void](Read-Host -Prompt 'Press Enter to close')
47
+ }
48
+ exit $Code
49
+ }
50
+
51
+ function Fail([string]$Message) {
52
+ Write-Host "ERROR: $Message" -ForegroundColor Red
53
+ Finish 1
54
+ }
55
+
56
+ $rows = @(foreach ($line in ($Rules -split "`r?`n")) {
57
+ if (-not $line.Trim()) { continue }
58
+ $c = $line -split "`t"
59
+ [pscustomobject]@{ Ds = $c[0]; Kind = $c[1]; A = $c[2]; B = $c[3]; C = $c[4] }
60
+ })
61
+ $saved = @(); $failed = 0; $missing = @()
62
+ foreach ($ds in @($rows | ForEach-Object { $_.Ds } | Sort-Object -Unique)) {
63
+ Write-Host "Listing the $ds files..."
64
+ try {
65
+ $reply = Invoke-RestMethod -Uri "$Api/$($ds)?current=true" -UseBasicParsing -UserAgent 'data-analyst/1.0'
66
+ if ($reply -is [string]) { $reply = $reply | ConvertFrom-Json } # served without a JSON content type
67
+ $list = @($reply | ForEach-Object { $_ })
68
+ } catch {
69
+ Fail "Could not list the $ds files at $Api ($($_.Exception.Message)). Check the connection, VPN and proxy, then run again."
70
+ }
71
+ $avail = @{}
72
+ foreach ($f in $list) {
73
+ if ([string]$f.filename -match "^$($ds)_(\d{6})_(\d+)$") { $avail[$Matches[1]] = $Matches[2] }
74
+ }
75
+ if ($avail.Count -eq 0) { Fail "OPM listed no $ds files. Run again later." }
76
+ $periods = @($avail.Keys | Sort-Object)
77
+ $first = $periods[0]; $latest = $periods[-1]
78
+ $want = @()
79
+ foreach ($r in @($rows | Where-Object { $_.Ds -eq $ds })) {
80
+ if ($r.Kind -eq 'periods') {
81
+ foreach ($p in $r.A.Split(',')) {
82
+ if ($p -eq 'latest') { $p = $latest }
83
+ if ($avail.ContainsKey($p)) { $want += $p } else { $missing += "$ds $p" }
84
+ }
85
+ } elseif ($r.Kind -eq 'range') {
86
+ $lo = $r.A; $hi = $r.B
87
+ if ($lo -eq 'first') { $lo = $first }
88
+ if ($hi -eq 'latest') { $hi = $latest }
89
+ $months = @($r.C.Split(',') | ForEach-Object { [int]$_ })
90
+ foreach ($p in $periods) {
91
+ if ($p -ge $lo -and $p -le $hi -and $months -contains [int]$p.Substring(4, 2)) { $want += $p }
92
+ }
93
+ }
94
+ }
95
+ foreach ($p in @($want | Sort-Object -Unique)) {
96
+ $v = $avail[$p]
97
+ $pin = @($rows | Where-Object { $_.Ds -eq $ds -and $_.Kind -eq 'pin' -and $_.A -eq $p })
98
+ if ($pin.Count -gt 0) { $v = $pin[0].B }
99
+ $name = "$($ds)_$($p)_$($v).parquet"
100
+ $dest = Join-Path $Here $name
101
+ if ((Test-Path -LiteralPath $dest) -and (Get-Item -LiteralPath $dest).Length -gt 0) {
102
+ Write-Host "already saved: $name"; $saved += $dest; continue
103
+ }
104
+ Write-Host "Downloading $name ..."
105
+ try {
106
+ Invoke-WebRequest -Uri "$Api/$ds/$($p.Substring(0, 4))/$($p.Substring(4, 2))/$v/download" -OutFile "$dest.part" `
107
+ -UseBasicParsing -UserAgent 'data-analyst/1.0'
108
+ Move-Item -LiteralPath "$dest.part" -Destination $dest -Force
109
+ $saved += $dest
110
+ } catch {
111
+ if (Test-Path -LiteralPath "$dest.part") { Remove-Item -LiteralPath "$dest.part" -Force }
112
+ Write-Host "FAILED $name ($($_.Exception.Message))" -ForegroundColor Red
113
+ $failed++
114
+ }
115
+ }
116
+ }
117
+
118
+ Write-Host ''
119
+ if ($missing.Count -gt 0) { Write-Host "Not published by OPM (skipped): $($missing -join ', ')" }
120
+ if ($saved.Count -gt 0) {
121
+ Write-Host 'Done. Upload these files to the chat (zip them first if they are large; do not rename them):' -ForegroundColor Green
122
+ foreach ($f in $saved) { Write-Host " $f" }
123
+ }
124
+ if ($failed -gt 0) { Write-Host "$failed file(s) failed: run this script again." -ForegroundColor Red; Finish 1 }
125
+ Finish 0