data-analyst-mcp 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +262 -262
- package/dist/report/method_audit.js +7 -1
- package/dist/report/method_audit.js.map +1 -1
- package/dist/report/scan_secrets.d.ts +4 -1
- package/dist/report/scan_secrets.js +36 -5
- package/dist/report/scan_secrets.js.map +1 -1
- package/dist/server.d.ts +1 -1
- package/dist/server.js +3 -2
- package/dist/server.js.map +1 -1
- package/guide/WORKFLOW.md +134 -134
- package/guide/assets/disclaimer.txt +1 -1
- package/guide/assets/download_data.ps1 +162 -162
- package/guide/assets/report_template.html +597 -597
- package/guide/references/routing.md +171 -171
- package/guide/references/style_guide.md +398 -398
- package/package.json +56 -56
- package/packs/acf/endpoints.json +114 -114
- package/packs/acf/policy.json +104 -104
- package/packs/actuaries/endpoints.json +82 -82
- package/packs/actuaries/policy.json +109 -109
- package/packs/bea/assets/download_bea.ps1 +182 -182
- package/packs/bea/docs/bea_api.md +461 -461
- package/packs/bea/docs/datasets.md +157 -157
- package/packs/bea/docs/report_patterns.md +112 -112
- package/packs/bea/policy.json +170 -170
- package/packs/bea/source.md +152 -152
- package/packs/bls/assets/download_bls.ps1 +152 -152
- package/packs/bls/docs/analysis_patterns.md +108 -108
- package/packs/bls/docs/bls_api.md +73 -73
- package/packs/bls/docs/series_ids.md +149 -149
- package/packs/bls/policy.json +238 -238
- package/packs/bts/endpoints.json +346 -346
- package/packs/bts/policy.json +94 -94
- package/packs/cdc/endpoints.json +197 -197
- package/packs/cdc/policy.json +83 -83
- package/packs/census/endpoints.json +99 -99
- package/packs/cfpb/endpoints.json +326 -326
- package/packs/cfpb/policy.json +119 -119
- package/packs/clinical-trials/endpoints.json +166 -166
- package/packs/clinical-trials/policy.json +84 -84
- package/packs/cms/endpoints.json +69 -69
- package/packs/cms/policy.json +96 -96
- package/packs/college-scorecard/endpoints.json +252 -252
- package/packs/college-scorecard/policy.json +106 -106
- package/packs/congress/endpoints.json +1701 -1701
- package/packs/congress/policy.json +117 -117
- package/packs/dhs-ohss/endpoints.json +66 -66
- package/packs/dhs-ohss/policy.json +107 -107
- package/packs/doj-news/endpoints.json +59 -59
- package/packs/doj-news/policy.json +96 -96
- package/packs/dol/endpoints.json +817 -817
- package/packs/dol/policy.json +135 -135
- package/packs/eia/endpoints.json +717 -717
- package/packs/eia/policy.json +112 -112
- package/packs/epa/endpoints.json +291 -291
- package/packs/epa/policy.json +99 -99
- package/packs/epa-aqs/endpoints.json +173 -173
- package/packs/epa-aqs/policy.json +96 -96
- package/packs/fbi/endpoints.json +1841 -1841
- package/packs/fbi/policy.json +118 -118
- package/packs/fda/endpoints.json +237 -237
- package/packs/fda/policy.json +121 -121
- package/packs/fdic/endpoints.json +416 -416
- package/packs/fdic/policy.json +83 -83
- package/packs/fec/endpoints.json +707 -707
- package/packs/fec/policy.json +93 -93
- package/packs/fed-board/endpoints.json +89 -89
- package/packs/fed-board/policy.json +136 -136
- package/packs/federal-register/endpoints.json +143 -143
- package/packs/federal-register/policy.json +86 -86
- package/packs/fema/endpoints.json +645 -645
- package/packs/fema/policy.json +103 -103
- package/packs/fhfa/endpoints.json +105 -105
- package/packs/fhfa/policy.json +109 -109
- package/packs/fred/assets/download_fred.ps1 +122 -122
- package/packs/fred/docs/analysis_patterns.md +124 -124
- package/packs/fred/docs/fred_api.md +94 -94
- package/packs/fred/docs/series.md +188 -188
- package/packs/fred/policy.json +253 -253
- package/packs/fsa/endpoints.json +150 -150
- package/packs/fsa/policy.json +99 -99
- package/packs/govinfo/endpoints.json +519 -519
- package/packs/gsa-calc/endpoints.json +93 -93
- package/packs/gsa-calc/policy.json +97 -97
- package/packs/hud/endpoints.json +146 -146
- package/packs/hud/policy.json +84 -84
- package/packs/ipums/docs/report_patterns.md +117 -117
- package/packs/irs-soi/endpoints.json +114 -114
- package/packs/irs-soi/policy.json +107 -107
- package/packs/medicaid/endpoints.json +412 -412
- package/packs/medicaid/policy.json +108 -108
- package/packs/naep/endpoints.json +217 -217
- package/packs/naep/policy.json +100 -100
- package/packs/nhtsa/endpoints.json +132 -132
- package/packs/nhtsa/policy.json +93 -93
- package/packs/nih/endpoints.json +452 -452
- package/packs/nih/policy.json +86 -86
- package/packs/noaa/endpoints.json +431 -431
- package/packs/noaa/policy.json +95 -95
- package/packs/nrel/endpoints.json +327 -327
- package/packs/nrel/policy.json +96 -96
- package/packs/nws/endpoints.json +271 -271
- package/packs/nws/policy.json +82 -82
- package/packs/nyfed/endpoints.json +80 -80
- package/packs/nyfed/policy.json +92 -92
- package/packs/omb/endpoints.json +460 -460
- package/packs/omb/policy.json +106 -106
- package/packs/open-payments/endpoints.json +297 -297
- package/packs/open-payments/policy.json +101 -101
- package/packs/opm/assets/download_opm.ps1 +125 -125
- package/packs/opm/docs/fields.md +184 -184
- package/packs/opm/docs/opm_api.md +71 -71
- package/packs/opm/docs/report_patterns.md +83 -83
- package/packs/opm/lib/data_dictionary.csv +73 -73
- package/packs/pbgc/endpoints.json +483 -483
- package/packs/pbgc/policy.json +94 -94
- package/packs/philfed/endpoints.json +85 -85
- package/packs/philfed/policy.json +94 -94
- package/packs/regulations/endpoints.json +287 -287
- package/packs/regulations/policy.json +86 -86
- package/packs/sec/endpoints.json +236 -236
- package/packs/sec/policy.json +86 -86
- package/packs/senate-lobbying/endpoints.json +553 -553
- package/packs/senate-lobbying/policy.json +99 -99
- package/packs/snapqc/endpoints.json +1426 -1426
- package/packs/snapqc/policy.json +92 -92
- package/packs/treasury/catalog.json +2267 -2267
- package/packs/treasury/docs/api_guide.md +245 -245
- package/packs/treasury/docs/datasets.md +56 -56
- package/packs/treasury/docs/report_patterns.md +126 -126
- package/packs/treasury/policy.json +159 -159
- package/packs/treasury/source.md +170 -165
- package/packs/treasury-yields/endpoints.json +226 -226
- package/packs/treasury-yields/policy.json +86 -86
- package/packs/usaspending/endpoints.json +1315 -1315
- package/packs/usaspending/policy.json +109 -109
- package/packs/usda-fna/policy.json +133 -133
- package/packs/usda-fooddata/endpoints.json +180 -180
- package/packs/usda-fooddata/policy.json +94 -94
- package/packs/usda-nass/endpoints.json +323 -323
- package/packs/usda-nass/policy.json +95 -95
- package/packs/usda-outlook/endpoints.json +86 -86
- package/packs/usda-outlook/policy.json +109 -109
- package/packs/usgs/endpoints.json +313 -313
- package/packs/usgs/policy.json +95 -95
- package/packs/usitc/endpoints.json +74 -74
- package/packs/usitc/policy.json +111 -111
- package/packs/uspto/endpoints.json +141 -141
- package/packs/uspto/policy.json +95 -95
- package/packs/va/endpoints.json +414 -414
- package/packs/va/policy.json +101 -101
- package/packs/world-bank/endpoints.json +122 -122
- package/packs/world-bank/policy.json +101 -101
|
@@ -1,182 +1,182 @@
|
|
|
1
|
-
<#
|
|
2
|
-
.SYNOPSIS
|
|
3
|
-
Download the BEA data for one analysis: BEA's public files and the BEA Data API replies.
|
|
4
|
-
(Windows PowerShell counterpart of download_bea.sh.)
|
|
5
|
-
|
|
6
|
-
.DESCRIPTION
|
|
7
|
-
- Downloads every file listed in this script into the folder that holds it.
|
|
8
|
-
- When some files come from the API, it asks for your BEA API key (hidden input), or uses
|
|
9
|
-
$env:BEA_API_KEY when it is set. The key is never shown, saved, or put on a command line, and it is
|
|
10
|
-
removed from each API reply before the reply is saved (BEA repeats the key in every reply).
|
|
11
|
-
- Run it again after an interruption: files already downloaded are skipped.
|
|
12
|
-
- Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
|
|
13
|
-
|
|
14
|
-
Get a free key at https://apps.bea.gov/API/signup/ (BEA emails it; click the activation link first).
|
|
15
|
-
|
|
16
|
-
.PARAMETER NoPause
|
|
17
|
-
Do not wait for Enter before closing the window.
|
|
18
|
-
|
|
19
|
-
.EXAMPLE
|
|
20
|
-
powershell -NoProfile -ExecutionPolicy Bypass -File .\download_bea.ps1
|
|
21
|
-
#>
|
|
22
|
-
[CmdletBinding()]
|
|
23
|
-
param(
|
|
24
|
-
[switch]$NoPause
|
|
25
|
-
)
|
|
26
|
-
|
|
27
|
-
$ErrorActionPreference = 'Stop'
|
|
28
|
-
$ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
|
|
29
|
-
try {
|
|
30
|
-
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
|
31
|
-
} catch { }
|
|
32
|
-
|
|
33
|
-
$PauseSeconds = 1 # between API calls: BEA allows 100 a minute per key
|
|
34
|
-
|
|
35
|
-
# The files to download, one per line: "bulk NAME URL" (public, no key) or "api NAME URL" (the key is
|
|
36
|
-
# added when the script runs). Do not edit them by hand.
|
|
37
|
-
$FileList = @'
|
|
38
|
-
@FILES@
|
|
39
|
-
'@
|
|
40
|
-
|
|
41
|
-
$Here = $PSScriptRoot
|
|
42
|
-
if (-not $Here) { $Here = (Get-Location).Path }
|
|
43
|
-
|
|
44
|
-
# Keep the window open when the script was started with "Run with PowerShell".
|
|
45
|
-
function Finish([int]$Code) {
|
|
46
|
-
$script:Key = $null
|
|
47
|
-
if (-not $NoPause -and -not [Console]::IsInputRedirected) {
|
|
48
|
-
Write-Host ''
|
|
49
|
-
[void](Read-Host -Prompt 'Press Enter to close')
|
|
50
|
-
}
|
|
51
|
-
exit $Code
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
function Fail([string]$Message) {
|
|
55
|
-
Write-Host ''
|
|
56
|
-
Write-Host "ERROR: $Message" -ForegroundColor Red
|
|
57
|
-
Finish 1
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
$Files = @()
|
|
61
|
-
foreach ($line in ($FileList -split "`r?`n")) {
|
|
62
|
-
$parts = $line.Trim() -split '\s+'
|
|
63
|
-
if ($parts.Count -eq 3) { $Files += [pscustomobject]@{ Kind = $parts[0]; Name = $parts[1]; Url = $parts[2] } }
|
|
64
|
-
}
|
|
65
|
-
if ($Files.Count -eq 0) { Fail 'The file list is missing from this script.' }
|
|
66
|
-
|
|
67
|
-
# ---------------------------------------------------------------------------- API key (only when needed)
|
|
68
|
-
$Key = ''
|
|
69
|
-
if (@($Files | Where-Object { $_.Kind -eq 'api' }).Count -gt 0) {
|
|
70
|
-
$Key = $env:BEA_API_KEY
|
|
71
|
-
if ($Key) {
|
|
72
|
-
Write-Host 'Using the BEA API key in $env:BEA_API_KEY.'
|
|
73
|
-
} elseif ([Console]::IsInputRedirected) {
|
|
74
|
-
$Key = [Console]::In.ReadLine()
|
|
75
|
-
} else {
|
|
76
|
-
$secure = Read-Host -Prompt 'Paste your BEA API key (input is hidden), then press Enter' -AsSecureString
|
|
77
|
-
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
|
78
|
-
try { $Key = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
|
|
79
|
-
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
|
|
80
|
-
Remove-Variable secure
|
|
81
|
-
}
|
|
82
|
-
if ($null -eq $Key) { $Key = '' }
|
|
83
|
-
$Key = $Key -replace '\s', ''
|
|
84
|
-
if (-not $Key) { Fail 'No key entered. Get one at https://apps.bea.gov/API/signup/' }
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
# The reply with the key removed: the key itself, any 36-character key, and the echoed USERID.
|
|
88
|
-
function Remove-Key([string]$Text) {
|
|
89
|
-
if ($script:Key) { $Text = $Text.Replace($script:Key, '[REDACTED]') }
|
|
90
|
-
$Text = $Text -replace '[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}', '[REDACTED]'
|
|
91
|
-
return ($Text -replace '(?i)("ParameterName"\s*:\s*"USERID"\s*,\s*"ParameterValue"\s*:\s*")[^"]*', '$1[REDACTED]')
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
function Get-ApiError([string]$Text) {
|
|
95
|
-
$m = [regex]::Match($Text, '"APIErrorCode"\s*:\s*"?([0-9]+)')
|
|
96
|
-
if ($m.Success) { return $m.Groups[1].Value }
|
|
97
|
-
return ''
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
# ---------------------------------------------------------------------------- downloads
|
|
101
|
-
$utf8 = New-Object Text.UTF8Encoding($false)
|
|
102
|
-
$saved = @(); $warn = @()
|
|
103
|
-
$total = $Files.Count
|
|
104
|
-
for ($i = 0; $i -lt $total; $i++) {
|
|
105
|
-
$f = $Files[$i]
|
|
106
|
-
$dest = Join-Path $Here $f.Name
|
|
107
|
-
$n = $i + 1
|
|
108
|
-
if ((Test-Path -LiteralPath $dest) -and (Get-Item -LiteralPath $dest).Length -gt 0 -and
|
|
109
|
-
-not (Select-String -LiteralPath $dest -Pattern '"APIErrorCode"' -SimpleMatch -Quiet)) {
|
|
110
|
-
Write-Host "[$n/$total] $($f.Name): already downloaded, skipped."
|
|
111
|
-
$saved += $dest; continue
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
if ($f.Kind -eq 'bulk') {
|
|
115
|
-
Write-Host "[$n/$total] Downloading $($f.Name) (public file, no key) ..."
|
|
116
|
-
$part = "$dest.part"
|
|
117
|
-
for ($try = 1; ; $try++) {
|
|
118
|
-
try {
|
|
119
|
-
Invoke-WebRequest -Uri $f.Url -OutFile $part -UseBasicParsing
|
|
120
|
-
break
|
|
121
|
-
} catch {
|
|
122
|
-
if ($try -ge 3) {
|
|
123
|
-
Remove-Item -LiteralPath $part -ErrorAction SilentlyContinue
|
|
124
|
-
Fail "The download of $($f.Name) failed ($($_.Exception.Message)). Check the connection and run this script again (finished files are kept)."
|
|
125
|
-
}
|
|
126
|
-
Start-Sleep -Seconds (10 * $try)
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
Move-Item -LiteralPath $part -Destination $dest -Force
|
|
130
|
-
$saved += $dest; continue
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
Write-Host "[$n/$total] Requesting $($f.Name) from the BEA API ..."
|
|
134
|
-
$uri = "$($f.Url)&UserID=$([Uri]::EscapeDataString($Key))"
|
|
135
|
-
$text = $null; $code = 0
|
|
136
|
-
for ($try = 1; ; $try++) {
|
|
137
|
-
try {
|
|
138
|
-
$resp = Invoke-WebRequest -Uri $uri -UseBasicParsing
|
|
139
|
-
$code = [int]$resp.StatusCode
|
|
140
|
-
$text = $utf8.GetString($resp.RawContentStream.ToArray())
|
|
141
|
-
break
|
|
142
|
-
} catch {
|
|
143
|
-
$code = 0
|
|
144
|
-
if ($_.Exception.Response) { $code = [int]$_.Exception.Response.StatusCode }
|
|
145
|
-
if ($code -eq 429) {
|
|
146
|
-
Fail 'BEA has blocked the key for about an hour (HTTP 429: too many requests). Wait an hour, then run this script again; it continues where it stopped. Retrying sooner makes the block longer.'
|
|
147
|
-
}
|
|
148
|
-
if (($code -eq 0 -or $code -ge 500) -and $try -lt 4) {
|
|
149
|
-
Start-Sleep -Seconds (10 * $try)
|
|
150
|
-
continue
|
|
151
|
-
}
|
|
152
|
-
if ($code -eq 0) {
|
|
153
|
-
Fail "Could not reach BEA (apps.bea.gov): $(Remove-Key $_.Exception.Message) Check the internet connection, VPN and proxy, then run this script again (finished files are kept)."
|
|
154
|
-
}
|
|
155
|
-
$text = [string]$_.ErrorDetails.Message
|
|
156
|
-
break
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
$err = Get-ApiError $text
|
|
160
|
-
if ($err -eq '1' -or $err -eq '3' -or $err -eq '4') {
|
|
161
|
-
Fail "BEA rejected the API key (error $err). Check that it is typed correctly and that you clicked the activation link in BEA's email: https://apps.bea.gov/API/signup/"
|
|
162
|
-
}
|
|
163
|
-
if (($code -lt 200 -or $code -ge 300) -and -not $err) {
|
|
164
|
-
Fail "BEA replied with HTTP $code for $($f.Name). Run this script again later; if it happens again, tell the analyst."
|
|
165
|
-
}
|
|
166
|
-
[IO.File]::WriteAllText("$dest.part", (Remove-Key $text), $utf8)
|
|
167
|
-
Move-Item -LiteralPath "$dest.part" -Destination $dest -Force
|
|
168
|
-
if ($err) { $warn += "$($f.Name) (BEA error $err)" }
|
|
169
|
-
$saved += $dest
|
|
170
|
-
Start-Sleep -Seconds $PauseSeconds
|
|
171
|
-
}
|
|
172
|
-
$Key = $null
|
|
173
|
-
|
|
174
|
-
Write-Host ''
|
|
175
|
-
if ($warn.Count -gt 0) {
|
|
176
|
-
Write-Host 'BEA answered some requests with an error. Upload those files anyway; the analyst will fix the request:' -ForegroundColor Yellow
|
|
177
|
-
foreach ($w in $warn) { Write-Host " $w" }
|
|
178
|
-
Write-Host ''
|
|
179
|
-
}
|
|
180
|
-
Write-Host 'Done. Upload these files to the chat (do not unzip or edit them):' -ForegroundColor Green
|
|
181
|
-
foreach ($s in $saved) { Write-Host " $s" }
|
|
182
|
-
Finish 0
|
|
1
|
+
<#
|
|
2
|
+
.SYNOPSIS
|
|
3
|
+
Download the BEA data for one analysis: BEA's public files and the BEA Data API replies.
|
|
4
|
+
(Windows PowerShell counterpart of download_bea.sh.)
|
|
5
|
+
|
|
6
|
+
.DESCRIPTION
|
|
7
|
+
- Downloads every file listed in this script into the folder that holds it.
|
|
8
|
+
- When some files come from the API, it asks for your BEA API key (hidden input), or uses
|
|
9
|
+
$env:BEA_API_KEY when it is set. The key is never shown, saved, or put on a command line, and it is
|
|
10
|
+
removed from each API reply before the reply is saved (BEA repeats the key in every reply).
|
|
11
|
+
- Run it again after an interruption: files already downloaded are skipped.
|
|
12
|
+
- Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
|
|
13
|
+
|
|
14
|
+
Get a free key at https://apps.bea.gov/API/signup/ (BEA emails it; click the activation link first).
|
|
15
|
+
|
|
16
|
+
.PARAMETER NoPause
|
|
17
|
+
Do not wait for Enter before closing the window.
|
|
18
|
+
|
|
19
|
+
.EXAMPLE
|
|
20
|
+
powershell -NoProfile -ExecutionPolicy Bypass -File .\download_bea.ps1
|
|
21
|
+
#>
|
|
22
|
+
[CmdletBinding()]
|
|
23
|
+
param(
|
|
24
|
+
[switch]$NoPause
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
$ErrorActionPreference = 'Stop'
|
|
28
|
+
$ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
|
|
29
|
+
try {
|
|
30
|
+
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
|
31
|
+
} catch { }
|
|
32
|
+
|
|
33
|
+
$PauseSeconds = 1 # between API calls: BEA allows 100 a minute per key
|
|
34
|
+
|
|
35
|
+
# The files to download, one per line: "bulk NAME URL" (public, no key) or "api NAME URL" (the key is
|
|
36
|
+
# added when the script runs). Do not edit them by hand.
|
|
37
|
+
$FileList = @'
|
|
38
|
+
@FILES@
|
|
39
|
+
'@
|
|
40
|
+
|
|
41
|
+
$Here = $PSScriptRoot
|
|
42
|
+
if (-not $Here) { $Here = (Get-Location).Path }
|
|
43
|
+
|
|
44
|
+
# Keep the window open when the script was started with "Run with PowerShell".
|
|
45
|
+
function Finish([int]$Code) {
|
|
46
|
+
$script:Key = $null
|
|
47
|
+
if (-not $NoPause -and -not [Console]::IsInputRedirected) {
|
|
48
|
+
Write-Host ''
|
|
49
|
+
[void](Read-Host -Prompt 'Press Enter to close')
|
|
50
|
+
}
|
|
51
|
+
exit $Code
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function Fail([string]$Message) {
|
|
55
|
+
Write-Host ''
|
|
56
|
+
Write-Host "ERROR: $Message" -ForegroundColor Red
|
|
57
|
+
Finish 1
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
$Files = @()
|
|
61
|
+
foreach ($line in ($FileList -split "`r?`n")) {
|
|
62
|
+
$parts = $line.Trim() -split '\s+'
|
|
63
|
+
if ($parts.Count -eq 3) { $Files += [pscustomobject]@{ Kind = $parts[0]; Name = $parts[1]; Url = $parts[2] } }
|
|
64
|
+
}
|
|
65
|
+
if ($Files.Count -eq 0) { Fail 'The file list is missing from this script.' }
|
|
66
|
+
|
|
67
|
+
# ---------------------------------------------------------------------------- API key (only when needed)
|
|
68
|
+
$Key = ''
|
|
69
|
+
if (@($Files | Where-Object { $_.Kind -eq 'api' }).Count -gt 0) {
|
|
70
|
+
$Key = $env:BEA_API_KEY
|
|
71
|
+
if ($Key) {
|
|
72
|
+
Write-Host 'Using the BEA API key in $env:BEA_API_KEY.'
|
|
73
|
+
} elseif ([Console]::IsInputRedirected) {
|
|
74
|
+
$Key = [Console]::In.ReadLine()
|
|
75
|
+
} else {
|
|
76
|
+
$secure = Read-Host -Prompt 'Paste your BEA API key (input is hidden), then press Enter' -AsSecureString
|
|
77
|
+
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
|
78
|
+
try { $Key = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
|
|
79
|
+
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
|
|
80
|
+
Remove-Variable secure
|
|
81
|
+
}
|
|
82
|
+
if ($null -eq $Key) { $Key = '' }
|
|
83
|
+
$Key = $Key -replace '\s', ''
|
|
84
|
+
if (-not $Key) { Fail 'No key entered. Get one at https://apps.bea.gov/API/signup/' }
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
# The reply with the key removed: the key itself, any 36-character key, and the echoed USERID.
|
|
88
|
+
function Remove-Key([string]$Text) {
|
|
89
|
+
if ($script:Key) { $Text = $Text.Replace($script:Key, '[REDACTED]') }
|
|
90
|
+
$Text = $Text -replace '[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}', '[REDACTED]'
|
|
91
|
+
return ($Text -replace '(?i)("ParameterName"\s*:\s*"USERID"\s*,\s*"ParameterValue"\s*:\s*")[^"]*', '$1[REDACTED]')
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function Get-ApiError([string]$Text) {
|
|
95
|
+
$m = [regex]::Match($Text, '"APIErrorCode"\s*:\s*"?([0-9]+)')
|
|
96
|
+
if ($m.Success) { return $m.Groups[1].Value }
|
|
97
|
+
return ''
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
# ---------------------------------------------------------------------------- downloads
|
|
101
|
+
$utf8 = New-Object Text.UTF8Encoding($false)
|
|
102
|
+
$saved = @(); $warn = @()
|
|
103
|
+
$total = $Files.Count
|
|
104
|
+
for ($i = 0; $i -lt $total; $i++) {
|
|
105
|
+
$f = $Files[$i]
|
|
106
|
+
$dest = Join-Path $Here $f.Name
|
|
107
|
+
$n = $i + 1
|
|
108
|
+
if ((Test-Path -LiteralPath $dest) -and (Get-Item -LiteralPath $dest).Length -gt 0 -and
|
|
109
|
+
-not (Select-String -LiteralPath $dest -Pattern '"APIErrorCode"' -SimpleMatch -Quiet)) {
|
|
110
|
+
Write-Host "[$n/$total] $($f.Name): already downloaded, skipped."
|
|
111
|
+
$saved += $dest; continue
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
if ($f.Kind -eq 'bulk') {
|
|
115
|
+
Write-Host "[$n/$total] Downloading $($f.Name) (public file, no key) ..."
|
|
116
|
+
$part = "$dest.part"
|
|
117
|
+
for ($try = 1; ; $try++) {
|
|
118
|
+
try {
|
|
119
|
+
Invoke-WebRequest -Uri $f.Url -OutFile $part -UseBasicParsing
|
|
120
|
+
break
|
|
121
|
+
} catch {
|
|
122
|
+
if ($try -ge 3) {
|
|
123
|
+
Remove-Item -LiteralPath $part -ErrorAction SilentlyContinue
|
|
124
|
+
Fail "The download of $($f.Name) failed ($($_.Exception.Message)). Check the connection and run this script again (finished files are kept)."
|
|
125
|
+
}
|
|
126
|
+
Start-Sleep -Seconds (10 * $try)
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
Move-Item -LiteralPath $part -Destination $dest -Force
|
|
130
|
+
$saved += $dest; continue
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
Write-Host "[$n/$total] Requesting $($f.Name) from the BEA API ..."
|
|
134
|
+
$uri = "$($f.Url)&UserID=$([Uri]::EscapeDataString($Key))"
|
|
135
|
+
$text = $null; $code = 0
|
|
136
|
+
for ($try = 1; ; $try++) {
|
|
137
|
+
try {
|
|
138
|
+
$resp = Invoke-WebRequest -Uri $uri -UseBasicParsing
|
|
139
|
+
$code = [int]$resp.StatusCode
|
|
140
|
+
$text = $utf8.GetString($resp.RawContentStream.ToArray())
|
|
141
|
+
break
|
|
142
|
+
} catch {
|
|
143
|
+
$code = 0
|
|
144
|
+
if ($_.Exception.Response) { $code = [int]$_.Exception.Response.StatusCode }
|
|
145
|
+
if ($code -eq 429) {
|
|
146
|
+
Fail 'BEA has blocked the key for about an hour (HTTP 429: too many requests). Wait an hour, then run this script again; it continues where it stopped. Retrying sooner makes the block longer.'
|
|
147
|
+
}
|
|
148
|
+
if (($code -eq 0 -or $code -ge 500) -and $try -lt 4) {
|
|
149
|
+
Start-Sleep -Seconds (10 * $try)
|
|
150
|
+
continue
|
|
151
|
+
}
|
|
152
|
+
if ($code -eq 0) {
|
|
153
|
+
Fail "Could not reach BEA (apps.bea.gov): $(Remove-Key $_.Exception.Message) Check the internet connection, VPN and proxy, then run this script again (finished files are kept)."
|
|
154
|
+
}
|
|
155
|
+
$text = [string]$_.ErrorDetails.Message
|
|
156
|
+
break
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
$err = Get-ApiError $text
|
|
160
|
+
if ($err -eq '1' -or $err -eq '3' -or $err -eq '4') {
|
|
161
|
+
Fail "BEA rejected the API key (error $err). Check that it is typed correctly and that you clicked the activation link in BEA's email: https://apps.bea.gov/API/signup/"
|
|
162
|
+
}
|
|
163
|
+
if (($code -lt 200 -or $code -ge 300) -and -not $err) {
|
|
164
|
+
Fail "BEA replied with HTTP $code for $($f.Name). Run this script again later; if it happens again, tell the analyst."
|
|
165
|
+
}
|
|
166
|
+
[IO.File]::WriteAllText("$dest.part", (Remove-Key $text), $utf8)
|
|
167
|
+
Move-Item -LiteralPath "$dest.part" -Destination $dest -Force
|
|
168
|
+
if ($err) { $warn += "$($f.Name) (BEA error $err)" }
|
|
169
|
+
$saved += $dest
|
|
170
|
+
Start-Sleep -Seconds $PauseSeconds
|
|
171
|
+
}
|
|
172
|
+
$Key = $null
|
|
173
|
+
|
|
174
|
+
Write-Host ''
|
|
175
|
+
if ($warn.Count -gt 0) {
|
|
176
|
+
Write-Host 'BEA answered some requests with an error. Upload those files anyway; the analyst will fix the request:' -ForegroundColor Yellow
|
|
177
|
+
foreach ($w in $warn) { Write-Host " $w" }
|
|
178
|
+
Write-Host ''
|
|
179
|
+
}
|
|
180
|
+
Write-Host 'Done. Upload these files to the chat (do not unzip or edit them):' -ForegroundColor Green
|
|
181
|
+
foreach ($s in $saved) { Write-Host " $s" }
|
|
182
|
+
Finish 0
|