data-analyst-mcp 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (292) hide show
  1. package/README.md +286 -262
  2. package/dist/adapters/bjs.d.ts +13 -0
  3. package/dist/adapters/bjs.js +28 -0
  4. package/dist/adapters/bjs.js.map +1 -0
  5. package/dist/adapters/cbp.d.ts +5 -0
  6. package/dist/adapters/cbp.js +64 -0
  7. package/dist/adapters/cbp.js.map +1 -0
  8. package/dist/adapters/cdc-wonder.d.ts +5 -0
  9. package/dist/adapters/cdc-wonder.js +68 -0
  10. package/dist/adapters/cdc-wonder.js.map +1 -0
  11. package/dist/adapters/cpsc.d.ts +4 -0
  12. package/dist/adapters/cpsc.js +125 -0
  13. package/dist/adapters/cpsc.js.map +1 -0
  14. package/dist/adapters/fhwa.d.ts +9 -0
  15. package/dist/adapters/fhwa.js +16 -0
  16. package/dist/adapters/fhwa.js.map +1 -0
  17. package/dist/adapters/foreign-assistance.d.ts +9 -0
  18. package/dist/adapters/foreign-assistance.js +42 -0
  19. package/dist/adapters/foreign-assistance.js.map +1 -0
  20. package/dist/adapters/hud-ahar.d.ts +3 -0
  21. package/dist/adapters/hud-ahar.js +80 -0
  22. package/dist/adapters/hud-ahar.js.map +1 -0
  23. package/dist/adapters/nces.d.ts +10 -0
  24. package/dist/adapters/nces.js +32 -0
  25. package/dist/adapters/nces.js.map +1 -0
  26. package/dist/adapters/nsf.d.ts +11 -0
  27. package/dist/adapters/nsf.js +25 -0
  28. package/dist/adapters/nsf.js.map +1 -0
  29. package/dist/adapters/onrr.d.ts +9 -0
  30. package/dist/adapters/onrr.js +35 -0
  31. package/dist/adapters/onrr.js.map +1 -0
  32. package/dist/adapters/sba.d.ts +10 -0
  33. package/dist/adapters/sba.js +134 -0
  34. package/dist/adapters/sba.js.map +1 -0
  35. package/dist/adapters/treasury-tic.d.ts +30 -0
  36. package/dist/adapters/treasury-tic.js +149 -0
  37. package/dist/adapters/treasury-tic.js.map +1 -0
  38. package/dist/adapters/uscourts.d.ts +8 -0
  39. package/dist/adapters/uscourts.js +8 -0
  40. package/dist/adapters/uscourts.js.map +1 -0
  41. package/dist/analysis/cite.d.ts +9 -1
  42. package/dist/analysis/cite.js +9 -2
  43. package/dist/analysis/cite.js.map +1 -1
  44. package/dist/bin/cli.d.ts +2 -0
  45. package/dist/bin/cli.js +17 -0
  46. package/dist/bin/cli.js.map +1 -1
  47. package/dist/core/generic-rest.js +42 -17
  48. package/dist/core/generic-rest.js.map +1 -1
  49. package/dist/core/guard.d.ts +32 -0
  50. package/dist/core/guard.js +176 -0
  51. package/dist/core/guard.js.map +1 -0
  52. package/dist/core/http.d.ts +10 -0
  53. package/dist/core/http.js +36 -7
  54. package/dist/core/http.js.map +1 -1
  55. package/dist/core/keys.d.ts +13 -3
  56. package/dist/core/keys.js +67 -7
  57. package/dist/core/keys.js.map +1 -1
  58. package/dist/core/package.d.ts +12 -3
  59. package/dist/core/package.js +56 -12
  60. package/dist/core/package.js.map +1 -1
  61. package/dist/core/secret_patterns.d.ts +6 -0
  62. package/dist/core/secret_patterns.js +20 -0
  63. package/dist/core/secret_patterns.js.map +1 -0
  64. package/dist/core/tabular.d.ts +43 -1
  65. package/dist/core/tabular.js +149 -6
  66. package/dist/core/tabular.js.map +1 -1
  67. package/dist/envelope.js +3 -1
  68. package/dist/envelope.js.map +1 -1
  69. package/dist/index.d.ts +29 -22
  70. package/dist/index.js +45 -46
  71. package/dist/index.js.map +1 -1
  72. package/dist/remote/app.js +87 -31
  73. package/dist/remote/app.js.map +1 -1
  74. package/dist/remote/oauth.js +8 -1
  75. package/dist/remote/oauth.js.map +1 -1
  76. package/dist/remote/pages.d.ts +3 -7
  77. package/dist/remote/pages.js +49 -19
  78. package/dist/remote/pages.js.map +1 -1
  79. package/dist/remote/sign.d.ts +7 -1
  80. package/dist/remote/sign.js +33 -7
  81. package/dist/remote/sign.js.map +1 -1
  82. package/dist/remote/store.js +14 -5
  83. package/dist/remote/store.js.map +1 -1
  84. package/dist/report/audit_report.js +29 -7
  85. package/dist/report/audit_report.js.map +1 -1
  86. package/dist/report/build_report.js +34 -4
  87. package/dist/report/build_report.js.map +1 -1
  88. package/dist/report/method_audit.js +10 -1
  89. package/dist/report/method_audit.js.map +1 -1
  90. package/dist/report/scan_secrets.d.ts +4 -1
  91. package/dist/report/scan_secrets.js +38 -18
  92. package/dist/report/scan_secrets.js.map +1 -1
  93. package/dist/server.d.ts +3 -1
  94. package/dist/server.js +46 -13
  95. package/dist/server.js.map +1 -1
  96. package/dist/tools.js +27 -6
  97. package/dist/tools.js.map +1 -1
  98. package/guide/WORKFLOW.md +134 -134
  99. package/guide/assets/disclaimer.txt +1 -1
  100. package/guide/assets/download_data.ps1 +164 -162
  101. package/guide/assets/download_data.sh +8 -2
  102. package/guide/assets/report_template.html +597 -597
  103. package/guide/references/routing.md +198 -171
  104. package/guide/references/style_guide.md +398 -398
  105. package/package.json +56 -56
  106. package/packs/acf/endpoints.json +114 -114
  107. package/packs/acf/policy.json +104 -104
  108. package/packs/actuaries/endpoints.json +82 -82
  109. package/packs/actuaries/policy.json +109 -109
  110. package/packs/bea/assets/download_bea.ps1 +182 -182
  111. package/packs/bea/docs/bea_api.md +461 -461
  112. package/packs/bea/docs/datasets.md +157 -157
  113. package/packs/bea/docs/report_patterns.md +112 -112
  114. package/packs/bea/policy.json +170 -170
  115. package/packs/bea/source.md +152 -152
  116. package/packs/bjs/endpoints.json +150 -0
  117. package/packs/bjs/policy.json +114 -0
  118. package/packs/bjs/source.md +98 -0
  119. package/packs/bls/assets/download_bls.ps1 +152 -152
  120. package/packs/bls/docs/analysis_patterns.md +108 -108
  121. package/packs/bls/docs/bls_api.md +73 -73
  122. package/packs/bls/docs/series_ids.md +149 -149
  123. package/packs/bls/policy.json +238 -238
  124. package/packs/bts/endpoints.json +346 -346
  125. package/packs/bts/policy.json +94 -94
  126. package/packs/cbp/endpoints.json +31 -0
  127. package/packs/cbp/policy.json +85 -0
  128. package/packs/cbp/source.md +57 -0
  129. package/packs/cdc/endpoints.json +197 -197
  130. package/packs/cdc/policy.json +83 -83
  131. package/packs/cdc/source.md +1 -1
  132. package/packs/cdc-wonder/endpoints.json +26 -0
  133. package/packs/cdc-wonder/policy.json +84 -0
  134. package/packs/cdc-wonder/source.md +53 -0
  135. package/packs/census/endpoints.json +99 -99
  136. package/packs/cfpb/endpoints.json +326 -326
  137. package/packs/cfpb/policy.json +119 -119
  138. package/packs/clinical-trials/endpoints.json +166 -166
  139. package/packs/clinical-trials/policy.json +84 -84
  140. package/packs/cms/endpoints.json +69 -69
  141. package/packs/cms/policy.json +96 -96
  142. package/packs/college-scorecard/endpoints.json +252 -252
  143. package/packs/college-scorecard/policy.json +106 -106
  144. package/packs/college-scorecard/source.md +1 -1
  145. package/packs/congress/endpoints.json +1701 -1701
  146. package/packs/congress/policy.json +117 -117
  147. package/packs/cpsc/endpoints.json +91 -0
  148. package/packs/cpsc/policy.json +91 -0
  149. package/packs/cpsc/source.md +76 -0
  150. package/packs/dhs-ohss/endpoints.json +66 -66
  151. package/packs/dhs-ohss/policy.json +107 -107
  152. package/packs/dhs-ohss/source.md +1 -1
  153. package/packs/doj-news/endpoints.json +59 -59
  154. package/packs/doj-news/policy.json +96 -96
  155. package/packs/doj-news/source.md +1 -1
  156. package/packs/dol/endpoints.json +817 -817
  157. package/packs/dol/policy.json +135 -135
  158. package/packs/eia/endpoints.json +717 -717
  159. package/packs/eia/policy.json +112 -112
  160. package/packs/epa/endpoints.json +291 -291
  161. package/packs/epa/policy.json +99 -99
  162. package/packs/epa-aqs/endpoints.json +173 -173
  163. package/packs/epa-aqs/policy.json +96 -96
  164. package/packs/fbi/endpoints.json +1841 -1841
  165. package/packs/fbi/policy.json +118 -118
  166. package/packs/fbi/source.md +3 -3
  167. package/packs/fda/endpoints.json +237 -237
  168. package/packs/fda/policy.json +121 -121
  169. package/packs/fdic/endpoints.json +416 -416
  170. package/packs/fdic/policy.json +83 -83
  171. package/packs/fec/endpoints.json +707 -707
  172. package/packs/fec/policy.json +93 -93
  173. package/packs/fed-board/endpoints.json +89 -89
  174. package/packs/fed-board/policy.json +136 -136
  175. package/packs/federal-register/endpoints.json +143 -143
  176. package/packs/federal-register/policy.json +86 -86
  177. package/packs/fema/endpoints.json +645 -645
  178. package/packs/fema/policy.json +103 -103
  179. package/packs/fhfa/endpoints.json +105 -105
  180. package/packs/fhfa/policy.json +109 -109
  181. package/packs/fhwa/endpoints.json +48 -0
  182. package/packs/fhwa/policy.json +91 -0
  183. package/packs/fhwa/source.md +59 -0
  184. package/packs/foreign-assistance/endpoints.json +166 -0
  185. package/packs/foreign-assistance/policy.json +80 -0
  186. package/packs/foreign-assistance/source.md +76 -0
  187. package/packs/fred/assets/download_fred.ps1 +122 -122
  188. package/packs/fred/docs/analysis_patterns.md +124 -124
  189. package/packs/fred/docs/fred_api.md +94 -94
  190. package/packs/fred/docs/series.md +188 -188
  191. package/packs/fred/policy.json +253 -253
  192. package/packs/fsa/endpoints.json +150 -150
  193. package/packs/fsa/policy.json +99 -99
  194. package/packs/fsa/source.md +1 -1
  195. package/packs/govinfo/endpoints.json +519 -519
  196. package/packs/gsa-calc/endpoints.json +93 -93
  197. package/packs/gsa-calc/policy.json +97 -97
  198. package/packs/hud/endpoints.json +146 -146
  199. package/packs/hud/policy.json +84 -84
  200. package/packs/hud/source.md +1 -1
  201. package/packs/hud-ahar/endpoints.json +92 -0
  202. package/packs/hud-ahar/policy.json +88 -0
  203. package/packs/hud-ahar/source.md +72 -0
  204. package/packs/ipums/docs/report_patterns.md +117 -117
  205. package/packs/irs-soi/endpoints.json +114 -114
  206. package/packs/irs-soi/policy.json +107 -107
  207. package/packs/medicaid/endpoints.json +412 -412
  208. package/packs/medicaid/policy.json +108 -108
  209. package/packs/naep/endpoints.json +217 -217
  210. package/packs/naep/policy.json +100 -100
  211. package/packs/naep/source.md +1 -1
  212. package/packs/nces/endpoints.json +101 -0
  213. package/packs/nces/policy.json +99 -0
  214. package/packs/nces/source.md +75 -0
  215. package/packs/nhtsa/endpoints.json +132 -132
  216. package/packs/nhtsa/policy.json +93 -93
  217. package/packs/nhtsa/source.md +1 -1
  218. package/packs/nih/endpoints.json +452 -452
  219. package/packs/nih/policy.json +86 -86
  220. package/packs/noaa/endpoints.json +431 -431
  221. package/packs/noaa/policy.json +95 -95
  222. package/packs/nrel/endpoints.json +327 -327
  223. package/packs/nrel/policy.json +96 -96
  224. package/packs/nsf/endpoints.json +33 -0
  225. package/packs/nsf/policy.json +83 -0
  226. package/packs/nsf/source.md +59 -0
  227. package/packs/nws/endpoints.json +271 -271
  228. package/packs/nws/policy.json +82 -82
  229. package/packs/nyfed/endpoints.json +80 -80
  230. package/packs/nyfed/policy.json +92 -92
  231. package/packs/omb/endpoints.json +460 -460
  232. package/packs/omb/policy.json +106 -106
  233. package/packs/onrr/endpoints.json +198 -0
  234. package/packs/onrr/policy.json +91 -0
  235. package/packs/onrr/source.md +55 -0
  236. package/packs/open-payments/endpoints.json +297 -297
  237. package/packs/open-payments/policy.json +101 -101
  238. package/packs/opm/assets/download_opm.ps1 +125 -125
  239. package/packs/opm/docs/fields.md +184 -184
  240. package/packs/opm/docs/opm_api.md +71 -71
  241. package/packs/opm/docs/report_patterns.md +83 -83
  242. package/packs/opm/lib/data_dictionary.csv +73 -73
  243. package/packs/pbgc/endpoints.json +483 -483
  244. package/packs/pbgc/policy.json +94 -94
  245. package/packs/philfed/endpoints.json +85 -85
  246. package/packs/philfed/policy.json +94 -94
  247. package/packs/port.json +65 -0
  248. package/packs/regulations/endpoints.json +287 -287
  249. package/packs/regulations/policy.json +86 -86
  250. package/packs/sba/endpoints.json +34 -0
  251. package/packs/sba/policy.json +86 -0
  252. package/packs/sba/source.md +60 -0
  253. package/packs/sec/endpoints.json +236 -236
  254. package/packs/sec/policy.json +86 -86
  255. package/packs/senate-lobbying/endpoints.json +553 -553
  256. package/packs/senate-lobbying/policy.json +99 -99
  257. package/packs/snapqc/endpoints.json +1426 -1426
  258. package/packs/snapqc/policy.json +92 -92
  259. package/packs/treasury/catalog.json +2267 -2267
  260. package/packs/treasury/docs/api_guide.md +245 -245
  261. package/packs/treasury/docs/datasets.md +56 -56
  262. package/packs/treasury/docs/report_patterns.md +126 -126
  263. package/packs/treasury/policy.json +159 -159
  264. package/packs/treasury/source.md +170 -165
  265. package/packs/treasury-tic/endpoints.json +77 -0
  266. package/packs/treasury-tic/policy.json +89 -0
  267. package/packs/treasury-tic/source.md +80 -0
  268. package/packs/treasury-yields/endpoints.json +226 -226
  269. package/packs/treasury-yields/policy.json +86 -86
  270. package/packs/usaspending/endpoints.json +1315 -1315
  271. package/packs/usaspending/policy.json +109 -109
  272. package/packs/uscourts/endpoints.json +22 -0
  273. package/packs/uscourts/policy.json +87 -0
  274. package/packs/uscourts/source.md +56 -0
  275. package/packs/usda-fna/policy.json +133 -133
  276. package/packs/usda-fooddata/endpoints.json +180 -180
  277. package/packs/usda-fooddata/policy.json +94 -94
  278. package/packs/usda-nass/endpoints.json +323 -323
  279. package/packs/usda-nass/policy.json +95 -95
  280. package/packs/usda-outlook/endpoints.json +86 -86
  281. package/packs/usda-outlook/policy.json +109 -109
  282. package/packs/usgs/endpoints.json +313 -313
  283. package/packs/usgs/policy.json +95 -95
  284. package/packs/usitc/endpoints.json +74 -74
  285. package/packs/usitc/policy.json +111 -111
  286. package/packs/uspto/endpoints.json +141 -141
  287. package/packs/uspto/policy.json +95 -95
  288. package/packs/uspto/source.md +1 -1
  289. package/packs/va/endpoints.json +414 -414
  290. package/packs/va/policy.json +101 -101
  291. package/packs/world-bank/endpoints.json +122 -122
  292. package/packs/world-bank/policy.json +101 -101
@@ -1,182 +1,182 @@
1
- <#
2
- .SYNOPSIS
3
- Download the BEA data for one analysis: BEA's public files and the BEA Data API replies.
4
- (Windows PowerShell counterpart of download_bea.sh.)
5
-
6
- .DESCRIPTION
7
- - Downloads every file listed in this script into the folder that holds it.
8
- - When some files come from the API, it asks for your BEA API key (hidden input), or uses
9
- $env:BEA_API_KEY when it is set. The key is never shown, saved, or put on a command line, and it is
10
- removed from each API reply before the reply is saved (BEA repeats the key in every reply).
11
- - Run it again after an interruption: files already downloaded are skipped.
12
- - Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
13
-
14
- Get a free key at https://apps.bea.gov/API/signup/ (BEA emails it; click the activation link first).
15
-
16
- .PARAMETER NoPause
17
- Do not wait for Enter before closing the window.
18
-
19
- .EXAMPLE
20
- powershell -NoProfile -ExecutionPolicy Bypass -File .\download_bea.ps1
21
- #>
22
- [CmdletBinding()]
23
- param(
24
- [switch]$NoPause
25
- )
26
-
27
- $ErrorActionPreference = 'Stop'
28
- $ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
29
- try {
30
- [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
31
- } catch { }
32
-
33
- $PauseSeconds = 1 # between API calls: BEA allows 100 a minute per key
34
-
35
- # The files to download, one per line: "bulk NAME URL" (public, no key) or "api NAME URL" (the key is
36
- # added when the script runs). Do not edit them by hand.
37
- $FileList = @'
38
- @FILES@
39
- '@
40
-
41
- $Here = $PSScriptRoot
42
- if (-not $Here) { $Here = (Get-Location).Path }
43
-
44
- # Keep the window open when the script was started with "Run with PowerShell".
45
- function Finish([int]$Code) {
46
- $script:Key = $null
47
- if (-not $NoPause -and -not [Console]::IsInputRedirected) {
48
- Write-Host ''
49
- [void](Read-Host -Prompt 'Press Enter to close')
50
- }
51
- exit $Code
52
- }
53
-
54
- function Fail([string]$Message) {
55
- Write-Host ''
56
- Write-Host "ERROR: $Message" -ForegroundColor Red
57
- Finish 1
58
- }
59
-
60
- $Files = @()
61
- foreach ($line in ($FileList -split "`r?`n")) {
62
- $parts = $line.Trim() -split '\s+'
63
- if ($parts.Count -eq 3) { $Files += [pscustomobject]@{ Kind = $parts[0]; Name = $parts[1]; Url = $parts[2] } }
64
- }
65
- if ($Files.Count -eq 0) { Fail 'The file list is missing from this script.' }
66
-
67
- # ---------------------------------------------------------------------------- API key (only when needed)
68
- $Key = ''
69
- if (@($Files | Where-Object { $_.Kind -eq 'api' }).Count -gt 0) {
70
- $Key = $env:BEA_API_KEY
71
- if ($Key) {
72
- Write-Host 'Using the BEA API key in $env:BEA_API_KEY.'
73
- } elseif ([Console]::IsInputRedirected) {
74
- $Key = [Console]::In.ReadLine()
75
- } else {
76
- $secure = Read-Host -Prompt 'Paste your BEA API key (input is hidden), then press Enter' -AsSecureString
77
- $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
78
- try { $Key = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
79
- finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
80
- Remove-Variable secure
81
- }
82
- if ($null -eq $Key) { $Key = '' }
83
- $Key = $Key -replace '\s', ''
84
- if (-not $Key) { Fail 'No key entered. Get one at https://apps.bea.gov/API/signup/' }
85
- }
86
-
87
- # The reply with the key removed: the key itself, any 36-character key, and the echoed USERID.
88
- function Remove-Key([string]$Text) {
89
- if ($script:Key) { $Text = $Text.Replace($script:Key, '[REDACTED]') }
90
- $Text = $Text -replace '[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}', '[REDACTED]'
91
- return ($Text -replace '(?i)("ParameterName"\s*:\s*"USERID"\s*,\s*"ParameterValue"\s*:\s*")[^"]*', '$1[REDACTED]')
92
- }
93
-
94
- function Get-ApiError([string]$Text) {
95
- $m = [regex]::Match($Text, '"APIErrorCode"\s*:\s*"?([0-9]+)')
96
- if ($m.Success) { return $m.Groups[1].Value }
97
- return ''
98
- }
99
-
100
- # ---------------------------------------------------------------------------- downloads
101
- $utf8 = New-Object Text.UTF8Encoding($false)
102
- $saved = @(); $warn = @()
103
- $total = $Files.Count
104
- for ($i = 0; $i -lt $total; $i++) {
105
- $f = $Files[$i]
106
- $dest = Join-Path $Here $f.Name
107
- $n = $i + 1
108
- if ((Test-Path -LiteralPath $dest) -and (Get-Item -LiteralPath $dest).Length -gt 0 -and
109
- -not (Select-String -LiteralPath $dest -Pattern '"APIErrorCode"' -SimpleMatch -Quiet)) {
110
- Write-Host "[$n/$total] $($f.Name): already downloaded, skipped."
111
- $saved += $dest; continue
112
- }
113
-
114
- if ($f.Kind -eq 'bulk') {
115
- Write-Host "[$n/$total] Downloading $($f.Name) (public file, no key) ..."
116
- $part = "$dest.part"
117
- for ($try = 1; ; $try++) {
118
- try {
119
- Invoke-WebRequest -Uri $f.Url -OutFile $part -UseBasicParsing
120
- break
121
- } catch {
122
- if ($try -ge 3) {
123
- Remove-Item -LiteralPath $part -ErrorAction SilentlyContinue
124
- Fail "The download of $($f.Name) failed ($($_.Exception.Message)). Check the connection and run this script again (finished files are kept)."
125
- }
126
- Start-Sleep -Seconds (10 * $try)
127
- }
128
- }
129
- Move-Item -LiteralPath $part -Destination $dest -Force
130
- $saved += $dest; continue
131
- }
132
-
133
- Write-Host "[$n/$total] Requesting $($f.Name) from the BEA API ..."
134
- $uri = "$($f.Url)&UserID=$([Uri]::EscapeDataString($Key))"
135
- $text = $null; $code = 0
136
- for ($try = 1; ; $try++) {
137
- try {
138
- $resp = Invoke-WebRequest -Uri $uri -UseBasicParsing
139
- $code = [int]$resp.StatusCode
140
- $text = $utf8.GetString($resp.RawContentStream.ToArray())
141
- break
142
- } catch {
143
- $code = 0
144
- if ($_.Exception.Response) { $code = [int]$_.Exception.Response.StatusCode }
145
- if ($code -eq 429) {
146
- Fail 'BEA has blocked the key for about an hour (HTTP 429: too many requests). Wait an hour, then run this script again; it continues where it stopped. Retrying sooner makes the block longer.'
147
- }
148
- if (($code -eq 0 -or $code -ge 500) -and $try -lt 4) {
149
- Start-Sleep -Seconds (10 * $try)
150
- continue
151
- }
152
- if ($code -eq 0) {
153
- Fail "Could not reach BEA (apps.bea.gov): $(Remove-Key $_.Exception.Message) Check the internet connection, VPN and proxy, then run this script again (finished files are kept)."
154
- }
155
- $text = [string]$_.ErrorDetails.Message
156
- break
157
- }
158
- }
159
- $err = Get-ApiError $text
160
- if ($err -eq '1' -or $err -eq '3' -or $err -eq '4') {
161
- Fail "BEA rejected the API key (error $err). Check that it is typed correctly and that you clicked the activation link in BEA's email: https://apps.bea.gov/API/signup/"
162
- }
163
- if (($code -lt 200 -or $code -ge 300) -and -not $err) {
164
- Fail "BEA replied with HTTP $code for $($f.Name). Run this script again later; if it happens again, tell the analyst."
165
- }
166
- [IO.File]::WriteAllText("$dest.part", (Remove-Key $text), $utf8)
167
- Move-Item -LiteralPath "$dest.part" -Destination $dest -Force
168
- if ($err) { $warn += "$($f.Name) (BEA error $err)" }
169
- $saved += $dest
170
- Start-Sleep -Seconds $PauseSeconds
171
- }
172
- $Key = $null
173
-
174
- Write-Host ''
175
- if ($warn.Count -gt 0) {
176
- Write-Host 'BEA answered some requests with an error. Upload those files anyway; the analyst will fix the request:' -ForegroundColor Yellow
177
- foreach ($w in $warn) { Write-Host " $w" }
178
- Write-Host ''
179
- }
180
- Write-Host 'Done. Upload these files to the chat (do not unzip or edit them):' -ForegroundColor Green
181
- foreach ($s in $saved) { Write-Host " $s" }
182
- Finish 0
1
+ <#
2
+ .SYNOPSIS
3
+ Download the BEA data for one analysis: BEA's public files and the BEA Data API replies.
4
+ (Windows PowerShell counterpart of download_bea.sh.)
5
+
6
+ .DESCRIPTION
7
+ - Downloads every file listed in this script into the folder that holds it.
8
+ - When some files come from the API, it asks for your BEA API key (hidden input), or uses
9
+ $env:BEA_API_KEY when it is set. The key is never shown, saved, or put on a command line, and it is
10
+ removed from each API reply before the reply is saved (BEA repeats the key in every reply).
11
+ - Run it again after an interruption: files already downloaded are skipped.
12
+ - Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
13
+
14
+ Get a free key at https://apps.bea.gov/API/signup/ (BEA emails it; click the activation link first).
15
+
16
+ .PARAMETER NoPause
17
+ Do not wait for Enter before closing the window.
18
+
19
+ .EXAMPLE
20
+ powershell -NoProfile -ExecutionPolicy Bypass -File .\download_bea.ps1
21
+ #>
22
+ [CmdletBinding()]
23
+ param(
24
+ [switch]$NoPause
25
+ )
26
+
27
+ $ErrorActionPreference = 'Stop'
28
+ $ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
29
+ try {
30
+ [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
31
+ } catch { }
32
+
33
+ $PauseSeconds = 1 # between API calls: BEA allows 100 a minute per key
34
+
35
+ # The files to download, one per line: "bulk NAME URL" (public, no key) or "api NAME URL" (the key is
36
+ # added when the script runs). Do not edit them by hand.
37
+ $FileList = @'
38
+ @FILES@
39
+ '@
40
+
41
+ $Here = $PSScriptRoot
42
+ if (-not $Here) { $Here = (Get-Location).Path }
43
+
44
+ # Keep the window open when the script was started with "Run with PowerShell".
45
+ function Finish([int]$Code) {
46
+ $script:Key = $null
47
+ if (-not $NoPause -and -not [Console]::IsInputRedirected) {
48
+ Write-Host ''
49
+ [void](Read-Host -Prompt 'Press Enter to close')
50
+ }
51
+ exit $Code
52
+ }
53
+
54
+ function Fail([string]$Message) {
55
+ Write-Host ''
56
+ Write-Host "ERROR: $Message" -ForegroundColor Red
57
+ Finish 1
58
+ }
59
+
60
+ $Files = @()
61
+ foreach ($line in ($FileList -split "`r?`n")) {
62
+ $parts = $line.Trim() -split '\s+'
63
+ if ($parts.Count -eq 3) { $Files += [pscustomobject]@{ Kind = $parts[0]; Name = $parts[1]; Url = $parts[2] } }
64
+ }
65
+ if ($Files.Count -eq 0) { Fail 'The file list is missing from this script.' }
66
+
67
+ # ---------------------------------------------------------------------------- API key (only when needed)
68
+ $Key = ''
69
+ if (@($Files | Where-Object { $_.Kind -eq 'api' }).Count -gt 0) {
70
+ $Key = $env:BEA_API_KEY
71
+ if ($Key) {
72
+ Write-Host 'Using the BEA API key in $env:BEA_API_KEY.'
73
+ } elseif ([Console]::IsInputRedirected) {
74
+ $Key = [Console]::In.ReadLine()
75
+ } else {
76
+ $secure = Read-Host -Prompt 'Paste your BEA API key (input is hidden), then press Enter' -AsSecureString
77
+ $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
78
+ try { $Key = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
79
+ finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
80
+ Remove-Variable secure
81
+ }
82
+ if ($null -eq $Key) { $Key = '' }
83
+ $Key = $Key -replace '\s', ''
84
+ if (-not $Key) { Fail 'No key entered. Get one at https://apps.bea.gov/API/signup/' }
85
+ }
86
+
87
+ # The reply with the key removed: the key itself, any 36-character key, and the echoed USERID.
88
+ function Remove-Key([string]$Text) {
89
+ if ($script:Key) { $Text = $Text.Replace($script:Key, '[REDACTED]') }
90
+ $Text = $Text -replace '[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}', '[REDACTED]'
91
+ return ($Text -replace '(?i)("ParameterName"\s*:\s*"USERID"\s*,\s*"ParameterValue"\s*:\s*")[^"]*', '$1[REDACTED]')
92
+ }
93
+
94
+ function Get-ApiError([string]$Text) {
95
+ $m = [regex]::Match($Text, '"APIErrorCode"\s*:\s*"?([0-9]+)')
96
+ if ($m.Success) { return $m.Groups[1].Value }
97
+ return ''
98
+ }
99
+
100
+ # ---------------------------------------------------------------------------- downloads
101
+ $utf8 = New-Object Text.UTF8Encoding($false)
102
+ $saved = @(); $warn = @()
103
+ $total = $Files.Count
104
+ for ($i = 0; $i -lt $total; $i++) {
105
+ $f = $Files[$i]
106
+ $dest = Join-Path $Here $f.Name
107
+ $n = $i + 1
108
+ if ((Test-Path -LiteralPath $dest) -and (Get-Item -LiteralPath $dest).Length -gt 0 -and
109
+ -not (Select-String -LiteralPath $dest -Pattern '"APIErrorCode"' -SimpleMatch -Quiet)) {
110
+ Write-Host "[$n/$total] $($f.Name): already downloaded, skipped."
111
+ $saved += $dest; continue
112
+ }
113
+
114
+ if ($f.Kind -eq 'bulk') {
115
+ Write-Host "[$n/$total] Downloading $($f.Name) (public file, no key) ..."
116
+ $part = "$dest.part"
117
+ for ($try = 1; ; $try++) {
118
+ try {
119
+ Invoke-WebRequest -Uri $f.Url -OutFile $part -UseBasicParsing
120
+ break
121
+ } catch {
122
+ if ($try -ge 3) {
123
+ Remove-Item -LiteralPath $part -ErrorAction SilentlyContinue
124
+ Fail "The download of $($f.Name) failed ($($_.Exception.Message)). Check the connection and run this script again (finished files are kept)."
125
+ }
126
+ Start-Sleep -Seconds (10 * $try)
127
+ }
128
+ }
129
+ Move-Item -LiteralPath $part -Destination $dest -Force
130
+ $saved += $dest; continue
131
+ }
132
+
133
+ Write-Host "[$n/$total] Requesting $($f.Name) from the BEA API ..."
134
+ $uri = "$($f.Url)&UserID=$([Uri]::EscapeDataString($Key))"
135
+ $text = $null; $code = 0
136
+ for ($try = 1; ; $try++) {
137
+ try {
138
+ $resp = Invoke-WebRequest -Uri $uri -UseBasicParsing
139
+ $code = [int]$resp.StatusCode
140
+ $text = $utf8.GetString($resp.RawContentStream.ToArray())
141
+ break
142
+ } catch {
143
+ $code = 0
144
+ if ($_.Exception.Response) { $code = [int]$_.Exception.Response.StatusCode }
145
+ if ($code -eq 429) {
146
+ Fail 'BEA has blocked the key for about an hour (HTTP 429: too many requests). Wait an hour, then run this script again; it continues where it stopped. Retrying sooner makes the block longer.'
147
+ }
148
+ if (($code -eq 0 -or $code -ge 500) -and $try -lt 4) {
149
+ Start-Sleep -Seconds (10 * $try)
150
+ continue
151
+ }
152
+ if ($code -eq 0) {
153
+ Fail "Could not reach BEA (apps.bea.gov): $(Remove-Key $_.Exception.Message) Check the internet connection, VPN and proxy, then run this script again (finished files are kept)."
154
+ }
155
+ $text = [string]$_.ErrorDetails.Message
156
+ break
157
+ }
158
+ }
159
+ $err = Get-ApiError $text
160
+ if ($err -eq '1' -or $err -eq '3' -or $err -eq '4') {
161
+ Fail "BEA rejected the API key (error $err). Check that it is typed correctly and that you clicked the activation link in BEA's email: https://apps.bea.gov/API/signup/"
162
+ }
163
+ if (($code -lt 200 -or $code -ge 300) -and -not $err) {
164
+ Fail "BEA replied with HTTP $code for $($f.Name). Run this script again later; if it happens again, tell the analyst."
165
+ }
166
+ [IO.File]::WriteAllText("$dest.part", (Remove-Key $text), $utf8)
167
+ Move-Item -LiteralPath "$dest.part" -Destination $dest -Force
168
+ if ($err) { $warn += "$($f.Name) (BEA error $err)" }
169
+ $saved += $dest
170
+ Start-Sleep -Seconds $PauseSeconds
171
+ }
172
+ $Key = $null
173
+
174
+ Write-Host ''
175
+ if ($warn.Count -gt 0) {
176
+ Write-Host 'BEA answered some requests with an error. Upload those files anyway; the analyst will fix the request:' -ForegroundColor Yellow
177
+ foreach ($w in $warn) { Write-Host " $w" }
178
+ Write-Host ''
179
+ }
180
+ Write-Host 'Done. Upload these files to the chat (do not unzip or edit them):' -ForegroundColor Green
181
+ foreach ($s in $saved) { Write-Host " $s" }
182
+ Finish 0