data-analyst-mcp 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/README.md +262 -262
  2. package/dist/report/method_audit.js +7 -1
  3. package/dist/report/method_audit.js.map +1 -1
  4. package/dist/report/scan_secrets.d.ts +4 -1
  5. package/dist/report/scan_secrets.js +36 -5
  6. package/dist/report/scan_secrets.js.map +1 -1
  7. package/dist/server.d.ts +1 -1
  8. package/dist/server.js +3 -2
  9. package/dist/server.js.map +1 -1
  10. package/guide/WORKFLOW.md +134 -134
  11. package/guide/assets/disclaimer.txt +1 -1
  12. package/guide/assets/download_data.ps1 +162 -162
  13. package/guide/assets/report_template.html +597 -597
  14. package/guide/references/routing.md +171 -171
  15. package/guide/references/style_guide.md +398 -398
  16. package/package.json +56 -56
  17. package/packs/acf/endpoints.json +114 -114
  18. package/packs/acf/policy.json +104 -104
  19. package/packs/actuaries/endpoints.json +82 -82
  20. package/packs/actuaries/policy.json +109 -109
  21. package/packs/bea/assets/download_bea.ps1 +182 -182
  22. package/packs/bea/docs/bea_api.md +461 -461
  23. package/packs/bea/docs/datasets.md +157 -157
  24. package/packs/bea/docs/report_patterns.md +112 -112
  25. package/packs/bea/policy.json +170 -170
  26. package/packs/bea/source.md +152 -152
  27. package/packs/bls/assets/download_bls.ps1 +152 -152
  28. package/packs/bls/docs/analysis_patterns.md +108 -108
  29. package/packs/bls/docs/bls_api.md +73 -73
  30. package/packs/bls/docs/series_ids.md +149 -149
  31. package/packs/bls/policy.json +238 -238
  32. package/packs/bts/endpoints.json +346 -346
  33. package/packs/bts/policy.json +94 -94
  34. package/packs/cdc/endpoints.json +197 -197
  35. package/packs/cdc/policy.json +83 -83
  36. package/packs/census/endpoints.json +99 -99
  37. package/packs/cfpb/endpoints.json +326 -326
  38. package/packs/cfpb/policy.json +119 -119
  39. package/packs/clinical-trials/endpoints.json +166 -166
  40. package/packs/clinical-trials/policy.json +84 -84
  41. package/packs/cms/endpoints.json +69 -69
  42. package/packs/cms/policy.json +96 -96
  43. package/packs/college-scorecard/endpoints.json +252 -252
  44. package/packs/college-scorecard/policy.json +106 -106
  45. package/packs/congress/endpoints.json +1701 -1701
  46. package/packs/congress/policy.json +117 -117
  47. package/packs/dhs-ohss/endpoints.json +66 -66
  48. package/packs/dhs-ohss/policy.json +107 -107
  49. package/packs/doj-news/endpoints.json +59 -59
  50. package/packs/doj-news/policy.json +96 -96
  51. package/packs/dol/endpoints.json +817 -817
  52. package/packs/dol/policy.json +135 -135
  53. package/packs/eia/endpoints.json +717 -717
  54. package/packs/eia/policy.json +112 -112
  55. package/packs/epa/endpoints.json +291 -291
  56. package/packs/epa/policy.json +99 -99
  57. package/packs/epa-aqs/endpoints.json +173 -173
  58. package/packs/epa-aqs/policy.json +96 -96
  59. package/packs/fbi/endpoints.json +1841 -1841
  60. package/packs/fbi/policy.json +118 -118
  61. package/packs/fda/endpoints.json +237 -237
  62. package/packs/fda/policy.json +121 -121
  63. package/packs/fdic/endpoints.json +416 -416
  64. package/packs/fdic/policy.json +83 -83
  65. package/packs/fec/endpoints.json +707 -707
  66. package/packs/fec/policy.json +93 -93
  67. package/packs/fed-board/endpoints.json +89 -89
  68. package/packs/fed-board/policy.json +136 -136
  69. package/packs/federal-register/endpoints.json +143 -143
  70. package/packs/federal-register/policy.json +86 -86
  71. package/packs/fema/endpoints.json +645 -645
  72. package/packs/fema/policy.json +103 -103
  73. package/packs/fhfa/endpoints.json +105 -105
  74. package/packs/fhfa/policy.json +109 -109
  75. package/packs/fred/assets/download_fred.ps1 +122 -122
  76. package/packs/fred/docs/analysis_patterns.md +124 -124
  77. package/packs/fred/docs/fred_api.md +94 -94
  78. package/packs/fred/docs/series.md +188 -188
  79. package/packs/fred/policy.json +253 -253
  80. package/packs/fsa/endpoints.json +150 -150
  81. package/packs/fsa/policy.json +99 -99
  82. package/packs/govinfo/endpoints.json +519 -519
  83. package/packs/gsa-calc/endpoints.json +93 -93
  84. package/packs/gsa-calc/policy.json +97 -97
  85. package/packs/hud/endpoints.json +146 -146
  86. package/packs/hud/policy.json +84 -84
  87. package/packs/ipums/docs/report_patterns.md +117 -117
  88. package/packs/irs-soi/endpoints.json +114 -114
  89. package/packs/irs-soi/policy.json +107 -107
  90. package/packs/medicaid/endpoints.json +412 -412
  91. package/packs/medicaid/policy.json +108 -108
  92. package/packs/naep/endpoints.json +217 -217
  93. package/packs/naep/policy.json +100 -100
  94. package/packs/nhtsa/endpoints.json +132 -132
  95. package/packs/nhtsa/policy.json +93 -93
  96. package/packs/nih/endpoints.json +452 -452
  97. package/packs/nih/policy.json +86 -86
  98. package/packs/noaa/endpoints.json +431 -431
  99. package/packs/noaa/policy.json +95 -95
  100. package/packs/nrel/endpoints.json +327 -327
  101. package/packs/nrel/policy.json +96 -96
  102. package/packs/nws/endpoints.json +271 -271
  103. package/packs/nws/policy.json +82 -82
  104. package/packs/nyfed/endpoints.json +80 -80
  105. package/packs/nyfed/policy.json +92 -92
  106. package/packs/omb/endpoints.json +460 -460
  107. package/packs/omb/policy.json +106 -106
  108. package/packs/open-payments/endpoints.json +297 -297
  109. package/packs/open-payments/policy.json +101 -101
  110. package/packs/opm/assets/download_opm.ps1 +125 -125
  111. package/packs/opm/docs/fields.md +184 -184
  112. package/packs/opm/docs/opm_api.md +71 -71
  113. package/packs/opm/docs/report_patterns.md +83 -83
  114. package/packs/opm/lib/data_dictionary.csv +73 -73
  115. package/packs/pbgc/endpoints.json +483 -483
  116. package/packs/pbgc/policy.json +94 -94
  117. package/packs/philfed/endpoints.json +85 -85
  118. package/packs/philfed/policy.json +94 -94
  119. package/packs/regulations/endpoints.json +287 -287
  120. package/packs/regulations/policy.json +86 -86
  121. package/packs/sec/endpoints.json +236 -236
  122. package/packs/sec/policy.json +86 -86
  123. package/packs/senate-lobbying/endpoints.json +553 -553
  124. package/packs/senate-lobbying/policy.json +99 -99
  125. package/packs/snapqc/endpoints.json +1426 -1426
  126. package/packs/snapqc/policy.json +92 -92
  127. package/packs/treasury/catalog.json +2267 -2267
  128. package/packs/treasury/docs/api_guide.md +245 -245
  129. package/packs/treasury/docs/datasets.md +56 -56
  130. package/packs/treasury/docs/report_patterns.md +126 -126
  131. package/packs/treasury/policy.json +159 -159
  132. package/packs/treasury/source.md +170 -165
  133. package/packs/treasury-yields/endpoints.json +226 -226
  134. package/packs/treasury-yields/policy.json +86 -86
  135. package/packs/usaspending/endpoints.json +1315 -1315
  136. package/packs/usaspending/policy.json +109 -109
  137. package/packs/usda-fna/policy.json +133 -133
  138. package/packs/usda-fooddata/endpoints.json +180 -180
  139. package/packs/usda-fooddata/policy.json +94 -94
  140. package/packs/usda-nass/endpoints.json +323 -323
  141. package/packs/usda-nass/policy.json +95 -95
  142. package/packs/usda-outlook/endpoints.json +86 -86
  143. package/packs/usda-outlook/policy.json +109 -109
  144. package/packs/usgs/endpoints.json +313 -313
  145. package/packs/usgs/policy.json +95 -95
  146. package/packs/usitc/endpoints.json +74 -74
  147. package/packs/usitc/policy.json +111 -111
  148. package/packs/uspto/endpoints.json +141 -141
  149. package/packs/uspto/policy.json +95 -95
  150. package/packs/va/endpoints.json +414 -414
  151. package/packs/va/policy.json +101 -101
  152. package/packs/world-bank/endpoints.json +122 -122
  153. package/packs/world-bank/policy.json +101 -101
@@ -1,182 +1,182 @@
1
- <#
2
- .SYNOPSIS
3
- Download the BEA data for one analysis: BEA's public files and the BEA Data API replies.
4
- (Windows PowerShell counterpart of download_bea.sh.)
5
-
6
- .DESCRIPTION
7
- - Downloads every file listed in this script into the folder that holds it.
8
- - When some files come from the API, it asks for your BEA API key (hidden input), or uses
9
- $env:BEA_API_KEY when it is set. The key is never shown, saved, or put on a command line, and it is
10
- removed from each API reply before the reply is saved (BEA repeats the key in every reply).
11
- - Run it again after an interruption: files already downloaded are skipped.
12
- - Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
13
-
14
- Get a free key at https://apps.bea.gov/API/signup/ (BEA emails it; click the activation link first).
15
-
16
- .PARAMETER NoPause
17
- Do not wait for Enter before closing the window.
18
-
19
- .EXAMPLE
20
- powershell -NoProfile -ExecutionPolicy Bypass -File .\download_bea.ps1
21
- #>
22
- [CmdletBinding()]
23
- param(
24
- [switch]$NoPause
25
- )
26
-
27
- $ErrorActionPreference = 'Stop'
28
- $ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
29
- try {
30
- [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
31
- } catch { }
32
-
33
- $PauseSeconds = 1 # between API calls: BEA allows 100 a minute per key
34
-
35
- # The files to download, one per line: "bulk NAME URL" (public, no key) or "api NAME URL" (the key is
36
- # added when the script runs). Do not edit them by hand.
37
- $FileList = @'
38
- @FILES@
39
- '@
40
-
41
- $Here = $PSScriptRoot
42
- if (-not $Here) { $Here = (Get-Location).Path }
43
-
44
- # Keep the window open when the script was started with "Run with PowerShell".
45
- function Finish([int]$Code) {
46
- $script:Key = $null
47
- if (-not $NoPause -and -not [Console]::IsInputRedirected) {
48
- Write-Host ''
49
- [void](Read-Host -Prompt 'Press Enter to close')
50
- }
51
- exit $Code
52
- }
53
-
54
- function Fail([string]$Message) {
55
- Write-Host ''
56
- Write-Host "ERROR: $Message" -ForegroundColor Red
57
- Finish 1
58
- }
59
-
60
- $Files = @()
61
- foreach ($line in ($FileList -split "`r?`n")) {
62
- $parts = $line.Trim() -split '\s+'
63
- if ($parts.Count -eq 3) { $Files += [pscustomobject]@{ Kind = $parts[0]; Name = $parts[1]; Url = $parts[2] } }
64
- }
65
- if ($Files.Count -eq 0) { Fail 'The file list is missing from this script.' }
66
-
67
- # ---------------------------------------------------------------------------- API key (only when needed)
68
- $Key = ''
69
- if (@($Files | Where-Object { $_.Kind -eq 'api' }).Count -gt 0) {
70
- $Key = $env:BEA_API_KEY
71
- if ($Key) {
72
- Write-Host 'Using the BEA API key in $env:BEA_API_KEY.'
73
- } elseif ([Console]::IsInputRedirected) {
74
- $Key = [Console]::In.ReadLine()
75
- } else {
76
- $secure = Read-Host -Prompt 'Paste your BEA API key (input is hidden), then press Enter' -AsSecureString
77
- $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
78
- try { $Key = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
79
- finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
80
- Remove-Variable secure
81
- }
82
- if ($null -eq $Key) { $Key = '' }
83
- $Key = $Key -replace '\s', ''
84
- if (-not $Key) { Fail 'No key entered. Get one at https://apps.bea.gov/API/signup/' }
85
- }
86
-
87
- # The reply with the key removed: the key itself, any 36-character key, and the echoed USERID.
88
- function Remove-Key([string]$Text) {
89
- if ($script:Key) { $Text = $Text.Replace($script:Key, '[REDACTED]') }
90
- $Text = $Text -replace '[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}', '[REDACTED]'
91
- return ($Text -replace '(?i)("ParameterName"\s*:\s*"USERID"\s*,\s*"ParameterValue"\s*:\s*")[^"]*', '$1[REDACTED]')
92
- }
93
-
94
- function Get-ApiError([string]$Text) {
95
- $m = [regex]::Match($Text, '"APIErrorCode"\s*:\s*"?([0-9]+)')
96
- if ($m.Success) { return $m.Groups[1].Value }
97
- return ''
98
- }
99
-
100
- # ---------------------------------------------------------------------------- downloads
101
- $utf8 = New-Object Text.UTF8Encoding($false)
102
- $saved = @(); $warn = @()
103
- $total = $Files.Count
104
- for ($i = 0; $i -lt $total; $i++) {
105
- $f = $Files[$i]
106
- $dest = Join-Path $Here $f.Name
107
- $n = $i + 1
108
- if ((Test-Path -LiteralPath $dest) -and (Get-Item -LiteralPath $dest).Length -gt 0 -and
109
- -not (Select-String -LiteralPath $dest -Pattern '"APIErrorCode"' -SimpleMatch -Quiet)) {
110
- Write-Host "[$n/$total] $($f.Name): already downloaded, skipped."
111
- $saved += $dest; continue
112
- }
113
-
114
- if ($f.Kind -eq 'bulk') {
115
- Write-Host "[$n/$total] Downloading $($f.Name) (public file, no key) ..."
116
- $part = "$dest.part"
117
- for ($try = 1; ; $try++) {
118
- try {
119
- Invoke-WebRequest -Uri $f.Url -OutFile $part -UseBasicParsing
120
- break
121
- } catch {
122
- if ($try -ge 3) {
123
- Remove-Item -LiteralPath $part -ErrorAction SilentlyContinue
124
- Fail "The download of $($f.Name) failed ($($_.Exception.Message)). Check the connection and run this script again (finished files are kept)."
125
- }
126
- Start-Sleep -Seconds (10 * $try)
127
- }
128
- }
129
- Move-Item -LiteralPath $part -Destination $dest -Force
130
- $saved += $dest; continue
131
- }
132
-
133
- Write-Host "[$n/$total] Requesting $($f.Name) from the BEA API ..."
134
- $uri = "$($f.Url)&UserID=$([Uri]::EscapeDataString($Key))"
135
- $text = $null; $code = 0
136
- for ($try = 1; ; $try++) {
137
- try {
138
- $resp = Invoke-WebRequest -Uri $uri -UseBasicParsing
139
- $code = [int]$resp.StatusCode
140
- $text = $utf8.GetString($resp.RawContentStream.ToArray())
141
- break
142
- } catch {
143
- $code = 0
144
- if ($_.Exception.Response) { $code = [int]$_.Exception.Response.StatusCode }
145
- if ($code -eq 429) {
146
- Fail 'BEA has blocked the key for about an hour (HTTP 429: too many requests). Wait an hour, then run this script again; it continues where it stopped. Retrying sooner makes the block longer.'
147
- }
148
- if (($code -eq 0 -or $code -ge 500) -and $try -lt 4) {
149
- Start-Sleep -Seconds (10 * $try)
150
- continue
151
- }
152
- if ($code -eq 0) {
153
- Fail "Could not reach BEA (apps.bea.gov): $(Remove-Key $_.Exception.Message) Check the internet connection, VPN and proxy, then run this script again (finished files are kept)."
154
- }
155
- $text = [string]$_.ErrorDetails.Message
156
- break
157
- }
158
- }
159
- $err = Get-ApiError $text
160
- if ($err -eq '1' -or $err -eq '3' -or $err -eq '4') {
161
- Fail "BEA rejected the API key (error $err). Check that it is typed correctly and that you clicked the activation link in BEA's email: https://apps.bea.gov/API/signup/"
162
- }
163
- if (($code -lt 200 -or $code -ge 300) -and -not $err) {
164
- Fail "BEA replied with HTTP $code for $($f.Name). Run this script again later; if it happens again, tell the analyst."
165
- }
166
- [IO.File]::WriteAllText("$dest.part", (Remove-Key $text), $utf8)
167
- Move-Item -LiteralPath "$dest.part" -Destination $dest -Force
168
- if ($err) { $warn += "$($f.Name) (BEA error $err)" }
169
- $saved += $dest
170
- Start-Sleep -Seconds $PauseSeconds
171
- }
172
- $Key = $null
173
-
174
- Write-Host ''
175
- if ($warn.Count -gt 0) {
176
- Write-Host 'BEA answered some requests with an error. Upload those files anyway; the analyst will fix the request:' -ForegroundColor Yellow
177
- foreach ($w in $warn) { Write-Host " $w" }
178
- Write-Host ''
179
- }
180
- Write-Host 'Done. Upload these files to the chat (do not unzip or edit them):' -ForegroundColor Green
181
- foreach ($s in $saved) { Write-Host " $s" }
182
- Finish 0
1
+ <#
2
+ .SYNOPSIS
3
+ Download the BEA data for one analysis: BEA's public files and the BEA Data API replies.
4
+ (Windows PowerShell counterpart of download_bea.sh.)
5
+
6
+ .DESCRIPTION
7
+ - Downloads every file listed in this script into the folder that holds it.
8
+ - When some files come from the API, it asks for your BEA API key (hidden input), or uses
9
+ $env:BEA_API_KEY when it is set. The key is never shown, saved, or put on a command line, and it is
10
+ removed from each API reply before the reply is saved (BEA repeats the key in every reply).
11
+ - Run it again after an interruption: files already downloaded are skipped.
12
+ - Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
13
+
14
+ Get a free key at https://apps.bea.gov/API/signup/ (BEA emails it; click the activation link first).
15
+
16
+ .PARAMETER NoPause
17
+ Do not wait for Enter before closing the window.
18
+
19
+ .EXAMPLE
20
+ powershell -NoProfile -ExecutionPolicy Bypass -File .\download_bea.ps1
21
+ #>
22
+ [CmdletBinding()]
23
+ param(
24
+ [switch]$NoPause
25
+ )
26
+
27
+ $ErrorActionPreference = 'Stop'
28
+ $ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
29
+ try {
30
+ [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
31
+ } catch { }
32
+
33
+ $PauseSeconds = 1 # between API calls: BEA allows 100 a minute per key
34
+
35
+ # The files to download, one per line: "bulk NAME URL" (public, no key) or "api NAME URL" (the key is
36
+ # added when the script runs). Do not edit them by hand.
37
+ $FileList = @'
38
+ @FILES@
39
+ '@
40
+
41
+ $Here = $PSScriptRoot
42
+ if (-not $Here) { $Here = (Get-Location).Path }
43
+
44
+ # Keep the window open when the script was started with "Run with PowerShell".
45
+ function Finish([int]$Code) {
46
+ $script:Key = $null
47
+ if (-not $NoPause -and -not [Console]::IsInputRedirected) {
48
+ Write-Host ''
49
+ [void](Read-Host -Prompt 'Press Enter to close')
50
+ }
51
+ exit $Code
52
+ }
53
+
54
+ function Fail([string]$Message) {
55
+ Write-Host ''
56
+ Write-Host "ERROR: $Message" -ForegroundColor Red
57
+ Finish 1
58
+ }
59
+
60
+ $Files = @()
61
+ foreach ($line in ($FileList -split "`r?`n")) {
62
+ $parts = $line.Trim() -split '\s+'
63
+ if ($parts.Count -eq 3) { $Files += [pscustomobject]@{ Kind = $parts[0]; Name = $parts[1]; Url = $parts[2] } }
64
+ }
65
+ if ($Files.Count -eq 0) { Fail 'The file list is missing from this script.' }
66
+
67
+ # ---------------------------------------------------------------------------- API key (only when needed)
68
+ $Key = ''
69
+ if (@($Files | Where-Object { $_.Kind -eq 'api' }).Count -gt 0) {
70
+ $Key = $env:BEA_API_KEY
71
+ if ($Key) {
72
+ Write-Host 'Using the BEA API key in $env:BEA_API_KEY.'
73
+ } elseif ([Console]::IsInputRedirected) {
74
+ $Key = [Console]::In.ReadLine()
75
+ } else {
76
+ $secure = Read-Host -Prompt 'Paste your BEA API key (input is hidden), then press Enter' -AsSecureString
77
+ $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
78
+ try { $Key = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
79
+ finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
80
+ Remove-Variable secure
81
+ }
82
+ if ($null -eq $Key) { $Key = '' }
83
+ $Key = $Key -replace '\s', ''
84
+ if (-not $Key) { Fail 'No key entered. Get one at https://apps.bea.gov/API/signup/' }
85
+ }
86
+
87
+ # The reply with the key removed: the key itself, any 36-character key, and the echoed USERID.
88
+ function Remove-Key([string]$Text) {
89
+ if ($script:Key) { $Text = $Text.Replace($script:Key, '[REDACTED]') }
90
+ $Text = $Text -replace '[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}', '[REDACTED]'
91
+ return ($Text -replace '(?i)("ParameterName"\s*:\s*"USERID"\s*,\s*"ParameterValue"\s*:\s*")[^"]*', '$1[REDACTED]')
92
+ }
93
+
94
+ function Get-ApiError([string]$Text) {
95
+ $m = [regex]::Match($Text, '"APIErrorCode"\s*:\s*"?([0-9]+)')
96
+ if ($m.Success) { return $m.Groups[1].Value }
97
+ return ''
98
+ }
99
+
100
+ # ---------------------------------------------------------------------------- downloads
101
+ $utf8 = New-Object Text.UTF8Encoding($false)
102
+ $saved = @(); $warn = @()
103
+ $total = $Files.Count
104
+ for ($i = 0; $i -lt $total; $i++) {
105
+ $f = $Files[$i]
106
+ $dest = Join-Path $Here $f.Name
107
+ $n = $i + 1
108
+ if ((Test-Path -LiteralPath $dest) -and (Get-Item -LiteralPath $dest).Length -gt 0 -and
109
+ -not (Select-String -LiteralPath $dest -Pattern '"APIErrorCode"' -SimpleMatch -Quiet)) {
110
+ Write-Host "[$n/$total] $($f.Name): already downloaded, skipped."
111
+ $saved += $dest; continue
112
+ }
113
+
114
+ if ($f.Kind -eq 'bulk') {
115
+ Write-Host "[$n/$total] Downloading $($f.Name) (public file, no key) ..."
116
+ $part = "$dest.part"
117
+ for ($try = 1; ; $try++) {
118
+ try {
119
+ Invoke-WebRequest -Uri $f.Url -OutFile $part -UseBasicParsing
120
+ break
121
+ } catch {
122
+ if ($try -ge 3) {
123
+ Remove-Item -LiteralPath $part -ErrorAction SilentlyContinue
124
+ Fail "The download of $($f.Name) failed ($($_.Exception.Message)). Check the connection and run this script again (finished files are kept)."
125
+ }
126
+ Start-Sleep -Seconds (10 * $try)
127
+ }
128
+ }
129
+ Move-Item -LiteralPath $part -Destination $dest -Force
130
+ $saved += $dest; continue
131
+ }
132
+
133
+ Write-Host "[$n/$total] Requesting $($f.Name) from the BEA API ..."
134
+ $uri = "$($f.Url)&UserID=$([Uri]::EscapeDataString($Key))"
135
+ $text = $null; $code = 0
136
+ for ($try = 1; ; $try++) {
137
+ try {
138
+ $resp = Invoke-WebRequest -Uri $uri -UseBasicParsing
139
+ $code = [int]$resp.StatusCode
140
+ $text = $utf8.GetString($resp.RawContentStream.ToArray())
141
+ break
142
+ } catch {
143
+ $code = 0
144
+ if ($_.Exception.Response) { $code = [int]$_.Exception.Response.StatusCode }
145
+ if ($code -eq 429) {
146
+ Fail 'BEA has blocked the key for about an hour (HTTP 429: too many requests). Wait an hour, then run this script again; it continues where it stopped. Retrying sooner makes the block longer.'
147
+ }
148
+ if (($code -eq 0 -or $code -ge 500) -and $try -lt 4) {
149
+ Start-Sleep -Seconds (10 * $try)
150
+ continue
151
+ }
152
+ if ($code -eq 0) {
153
+ Fail "Could not reach BEA (apps.bea.gov): $(Remove-Key $_.Exception.Message) Check the internet connection, VPN and proxy, then run this script again (finished files are kept)."
154
+ }
155
+ $text = [string]$_.ErrorDetails.Message
156
+ break
157
+ }
158
+ }
159
+ $err = Get-ApiError $text
160
+ if ($err -eq '1' -or $err -eq '3' -or $err -eq '4') {
161
+ Fail "BEA rejected the API key (error $err). Check that it is typed correctly and that you clicked the activation link in BEA's email: https://apps.bea.gov/API/signup/"
162
+ }
163
+ if (($code -lt 200 -or $code -ge 300) -and -not $err) {
164
+ Fail "BEA replied with HTTP $code for $($f.Name). Run this script again later; if it happens again, tell the analyst."
165
+ }
166
+ [IO.File]::WriteAllText("$dest.part", (Remove-Key $text), $utf8)
167
+ Move-Item -LiteralPath "$dest.part" -Destination $dest -Force
168
+ if ($err) { $warn += "$($f.Name) (BEA error $err)" }
169
+ $saved += $dest
170
+ Start-Sleep -Seconds $PauseSeconds
171
+ }
172
+ $Key = $null
173
+
174
+ Write-Host ''
175
+ if ($warn.Count -gt 0) {
176
+ Write-Host 'BEA answered some requests with an error. Upload those files anyway; the analyst will fix the request:' -ForegroundColor Yellow
177
+ foreach ($w in $warn) { Write-Host " $w" }
178
+ Write-Host ''
179
+ }
180
+ Write-Host 'Done. Upload these files to the chat (do not unzip or edit them):' -ForegroundColor Green
181
+ foreach ($s in $saved) { Write-Host " $s" }
182
+ Finish 0