data-analyst-mcp 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +286 -262
- package/dist/adapters/bjs.d.ts +13 -0
- package/dist/adapters/bjs.js +28 -0
- package/dist/adapters/bjs.js.map +1 -0
- package/dist/adapters/cbp.d.ts +5 -0
- package/dist/adapters/cbp.js +64 -0
- package/dist/adapters/cbp.js.map +1 -0
- package/dist/adapters/cdc-wonder.d.ts +5 -0
- package/dist/adapters/cdc-wonder.js +68 -0
- package/dist/adapters/cdc-wonder.js.map +1 -0
- package/dist/adapters/cpsc.d.ts +4 -0
- package/dist/adapters/cpsc.js +125 -0
- package/dist/adapters/cpsc.js.map +1 -0
- package/dist/adapters/fhwa.d.ts +9 -0
- package/dist/adapters/fhwa.js +16 -0
- package/dist/adapters/fhwa.js.map +1 -0
- package/dist/adapters/foreign-assistance.d.ts +9 -0
- package/dist/adapters/foreign-assistance.js +42 -0
- package/dist/adapters/foreign-assistance.js.map +1 -0
- package/dist/adapters/hud-ahar.d.ts +3 -0
- package/dist/adapters/hud-ahar.js +80 -0
- package/dist/adapters/hud-ahar.js.map +1 -0
- package/dist/adapters/nces.d.ts +10 -0
- package/dist/adapters/nces.js +32 -0
- package/dist/adapters/nces.js.map +1 -0
- package/dist/adapters/nsf.d.ts +11 -0
- package/dist/adapters/nsf.js +25 -0
- package/dist/adapters/nsf.js.map +1 -0
- package/dist/adapters/onrr.d.ts +9 -0
- package/dist/adapters/onrr.js +35 -0
- package/dist/adapters/onrr.js.map +1 -0
- package/dist/adapters/sba.d.ts +10 -0
- package/dist/adapters/sba.js +134 -0
- package/dist/adapters/sba.js.map +1 -0
- package/dist/adapters/treasury-tic.d.ts +30 -0
- package/dist/adapters/treasury-tic.js +149 -0
- package/dist/adapters/treasury-tic.js.map +1 -0
- package/dist/adapters/uscourts.d.ts +8 -0
- package/dist/adapters/uscourts.js +8 -0
- package/dist/adapters/uscourts.js.map +1 -0
- package/dist/analysis/cite.d.ts +9 -1
- package/dist/analysis/cite.js +9 -2
- package/dist/analysis/cite.js.map +1 -1
- package/dist/bin/cli.d.ts +2 -0
- package/dist/bin/cli.js +17 -0
- package/dist/bin/cli.js.map +1 -1
- package/dist/core/generic-rest.js +42 -17
- package/dist/core/generic-rest.js.map +1 -1
- package/dist/core/guard.d.ts +32 -0
- package/dist/core/guard.js +176 -0
- package/dist/core/guard.js.map +1 -0
- package/dist/core/http.d.ts +10 -0
- package/dist/core/http.js +36 -7
- package/dist/core/http.js.map +1 -1
- package/dist/core/keys.d.ts +13 -3
- package/dist/core/keys.js +67 -7
- package/dist/core/keys.js.map +1 -1
- package/dist/core/package.d.ts +12 -3
- package/dist/core/package.js +56 -12
- package/dist/core/package.js.map +1 -1
- package/dist/core/secret_patterns.d.ts +6 -0
- package/dist/core/secret_patterns.js +20 -0
- package/dist/core/secret_patterns.js.map +1 -0
- package/dist/core/tabular.d.ts +43 -1
- package/dist/core/tabular.js +149 -6
- package/dist/core/tabular.js.map +1 -1
- package/dist/envelope.js +3 -1
- package/dist/envelope.js.map +1 -1
- package/dist/index.d.ts +29 -22
- package/dist/index.js +45 -46
- package/dist/index.js.map +1 -1
- package/dist/remote/app.js +87 -31
- package/dist/remote/app.js.map +1 -1
- package/dist/remote/oauth.js +8 -1
- package/dist/remote/oauth.js.map +1 -1
- package/dist/remote/pages.d.ts +3 -7
- package/dist/remote/pages.js +49 -19
- package/dist/remote/pages.js.map +1 -1
- package/dist/remote/sign.d.ts +7 -1
- package/dist/remote/sign.js +33 -7
- package/dist/remote/sign.js.map +1 -1
- package/dist/remote/store.js +14 -5
- package/dist/remote/store.js.map +1 -1
- package/dist/report/audit_report.js +29 -7
- package/dist/report/audit_report.js.map +1 -1
- package/dist/report/build_report.js +34 -4
- package/dist/report/build_report.js.map +1 -1
- package/dist/report/method_audit.js +10 -1
- package/dist/report/method_audit.js.map +1 -1
- package/dist/report/scan_secrets.d.ts +4 -1
- package/dist/report/scan_secrets.js +38 -18
- package/dist/report/scan_secrets.js.map +1 -1
- package/dist/server.d.ts +3 -1
- package/dist/server.js +46 -13
- package/dist/server.js.map +1 -1
- package/dist/tools.js +27 -6
- package/dist/tools.js.map +1 -1
- package/guide/WORKFLOW.md +134 -134
- package/guide/assets/disclaimer.txt +1 -1
- package/guide/assets/download_data.ps1 +164 -162
- package/guide/assets/download_data.sh +8 -2
- package/guide/assets/report_template.html +597 -597
- package/guide/references/routing.md +198 -171
- package/guide/references/style_guide.md +398 -398
- package/package.json +56 -56
- package/packs/acf/endpoints.json +114 -114
- package/packs/acf/policy.json +104 -104
- package/packs/actuaries/endpoints.json +82 -82
- package/packs/actuaries/policy.json +109 -109
- package/packs/bea/assets/download_bea.ps1 +182 -182
- package/packs/bea/docs/bea_api.md +461 -461
- package/packs/bea/docs/datasets.md +157 -157
- package/packs/bea/docs/report_patterns.md +112 -112
- package/packs/bea/policy.json +170 -170
- package/packs/bea/source.md +152 -152
- package/packs/bjs/endpoints.json +150 -0
- package/packs/bjs/policy.json +114 -0
- package/packs/bjs/source.md +98 -0
- package/packs/bls/assets/download_bls.ps1 +152 -152
- package/packs/bls/docs/analysis_patterns.md +108 -108
- package/packs/bls/docs/bls_api.md +73 -73
- package/packs/bls/docs/series_ids.md +149 -149
- package/packs/bls/policy.json +238 -238
- package/packs/bts/endpoints.json +346 -346
- package/packs/bts/policy.json +94 -94
- package/packs/cbp/endpoints.json +31 -0
- package/packs/cbp/policy.json +85 -0
- package/packs/cbp/source.md +57 -0
- package/packs/cdc/endpoints.json +197 -197
- package/packs/cdc/policy.json +83 -83
- package/packs/cdc/source.md +1 -1
- package/packs/cdc-wonder/endpoints.json +26 -0
- package/packs/cdc-wonder/policy.json +84 -0
- package/packs/cdc-wonder/source.md +53 -0
- package/packs/census/endpoints.json +99 -99
- package/packs/cfpb/endpoints.json +326 -326
- package/packs/cfpb/policy.json +119 -119
- package/packs/clinical-trials/endpoints.json +166 -166
- package/packs/clinical-trials/policy.json +84 -84
- package/packs/cms/endpoints.json +69 -69
- package/packs/cms/policy.json +96 -96
- package/packs/college-scorecard/endpoints.json +252 -252
- package/packs/college-scorecard/policy.json +106 -106
- package/packs/college-scorecard/source.md +1 -1
- package/packs/congress/endpoints.json +1701 -1701
- package/packs/congress/policy.json +117 -117
- package/packs/cpsc/endpoints.json +91 -0
- package/packs/cpsc/policy.json +91 -0
- package/packs/cpsc/source.md +76 -0
- package/packs/dhs-ohss/endpoints.json +66 -66
- package/packs/dhs-ohss/policy.json +107 -107
- package/packs/dhs-ohss/source.md +1 -1
- package/packs/doj-news/endpoints.json +59 -59
- package/packs/doj-news/policy.json +96 -96
- package/packs/doj-news/source.md +1 -1
- package/packs/dol/endpoints.json +817 -817
- package/packs/dol/policy.json +135 -135
- package/packs/eia/endpoints.json +717 -717
- package/packs/eia/policy.json +112 -112
- package/packs/epa/endpoints.json +291 -291
- package/packs/epa/policy.json +99 -99
- package/packs/epa-aqs/endpoints.json +173 -173
- package/packs/epa-aqs/policy.json +96 -96
- package/packs/fbi/endpoints.json +1841 -1841
- package/packs/fbi/policy.json +118 -118
- package/packs/fbi/source.md +3 -3
- package/packs/fda/endpoints.json +237 -237
- package/packs/fda/policy.json +121 -121
- package/packs/fdic/endpoints.json +416 -416
- package/packs/fdic/policy.json +83 -83
- package/packs/fec/endpoints.json +707 -707
- package/packs/fec/policy.json +93 -93
- package/packs/fed-board/endpoints.json +89 -89
- package/packs/fed-board/policy.json +136 -136
- package/packs/federal-register/endpoints.json +143 -143
- package/packs/federal-register/policy.json +86 -86
- package/packs/fema/endpoints.json +645 -645
- package/packs/fema/policy.json +103 -103
- package/packs/fhfa/endpoints.json +105 -105
- package/packs/fhfa/policy.json +109 -109
- package/packs/fhwa/endpoints.json +48 -0
- package/packs/fhwa/policy.json +91 -0
- package/packs/fhwa/source.md +59 -0
- package/packs/foreign-assistance/endpoints.json +166 -0
- package/packs/foreign-assistance/policy.json +80 -0
- package/packs/foreign-assistance/source.md +76 -0
- package/packs/fred/assets/download_fred.ps1 +122 -122
- package/packs/fred/docs/analysis_patterns.md +124 -124
- package/packs/fred/docs/fred_api.md +94 -94
- package/packs/fred/docs/series.md +188 -188
- package/packs/fred/policy.json +253 -253
- package/packs/fsa/endpoints.json +150 -150
- package/packs/fsa/policy.json +99 -99
- package/packs/fsa/source.md +1 -1
- package/packs/govinfo/endpoints.json +519 -519
- package/packs/gsa-calc/endpoints.json +93 -93
- package/packs/gsa-calc/policy.json +97 -97
- package/packs/hud/endpoints.json +146 -146
- package/packs/hud/policy.json +84 -84
- package/packs/hud/source.md +1 -1
- package/packs/hud-ahar/endpoints.json +92 -0
- package/packs/hud-ahar/policy.json +88 -0
- package/packs/hud-ahar/source.md +72 -0
- package/packs/ipums/docs/report_patterns.md +117 -117
- package/packs/irs-soi/endpoints.json +114 -114
- package/packs/irs-soi/policy.json +107 -107
- package/packs/medicaid/endpoints.json +412 -412
- package/packs/medicaid/policy.json +108 -108
- package/packs/naep/endpoints.json +217 -217
- package/packs/naep/policy.json +100 -100
- package/packs/naep/source.md +1 -1
- package/packs/nces/endpoints.json +101 -0
- package/packs/nces/policy.json +99 -0
- package/packs/nces/source.md +75 -0
- package/packs/nhtsa/endpoints.json +132 -132
- package/packs/nhtsa/policy.json +93 -93
- package/packs/nhtsa/source.md +1 -1
- package/packs/nih/endpoints.json +452 -452
- package/packs/nih/policy.json +86 -86
- package/packs/noaa/endpoints.json +431 -431
- package/packs/noaa/policy.json +95 -95
- package/packs/nrel/endpoints.json +327 -327
- package/packs/nrel/policy.json +96 -96
- package/packs/nsf/endpoints.json +33 -0
- package/packs/nsf/policy.json +83 -0
- package/packs/nsf/source.md +59 -0
- package/packs/nws/endpoints.json +271 -271
- package/packs/nws/policy.json +82 -82
- package/packs/nyfed/endpoints.json +80 -80
- package/packs/nyfed/policy.json +92 -92
- package/packs/omb/endpoints.json +460 -460
- package/packs/omb/policy.json +106 -106
- package/packs/onrr/endpoints.json +198 -0
- package/packs/onrr/policy.json +91 -0
- package/packs/onrr/source.md +55 -0
- package/packs/open-payments/endpoints.json +297 -297
- package/packs/open-payments/policy.json +101 -101
- package/packs/opm/assets/download_opm.ps1 +125 -125
- package/packs/opm/docs/fields.md +184 -184
- package/packs/opm/docs/opm_api.md +71 -71
- package/packs/opm/docs/report_patterns.md +83 -83
- package/packs/opm/lib/data_dictionary.csv +73 -73
- package/packs/pbgc/endpoints.json +483 -483
- package/packs/pbgc/policy.json +94 -94
- package/packs/philfed/endpoints.json +85 -85
- package/packs/philfed/policy.json +94 -94
- package/packs/port.json +65 -0
- package/packs/regulations/endpoints.json +287 -287
- package/packs/regulations/policy.json +86 -86
- package/packs/sba/endpoints.json +34 -0
- package/packs/sba/policy.json +86 -0
- package/packs/sba/source.md +60 -0
- package/packs/sec/endpoints.json +236 -236
- package/packs/sec/policy.json +86 -86
- package/packs/senate-lobbying/endpoints.json +553 -553
- package/packs/senate-lobbying/policy.json +99 -99
- package/packs/snapqc/endpoints.json +1426 -1426
- package/packs/snapqc/policy.json +92 -92
- package/packs/treasury/catalog.json +2267 -2267
- package/packs/treasury/docs/api_guide.md +245 -245
- package/packs/treasury/docs/datasets.md +56 -56
- package/packs/treasury/docs/report_patterns.md +126 -126
- package/packs/treasury/policy.json +159 -159
- package/packs/treasury/source.md +170 -165
- package/packs/treasury-tic/endpoints.json +77 -0
- package/packs/treasury-tic/policy.json +89 -0
- package/packs/treasury-tic/source.md +80 -0
- package/packs/treasury-yields/endpoints.json +226 -226
- package/packs/treasury-yields/policy.json +86 -86
- package/packs/usaspending/endpoints.json +1315 -1315
- package/packs/usaspending/policy.json +109 -109
- package/packs/uscourts/endpoints.json +22 -0
- package/packs/uscourts/policy.json +87 -0
- package/packs/uscourts/source.md +56 -0
- package/packs/usda-fna/policy.json +133 -133
- package/packs/usda-fooddata/endpoints.json +180 -180
- package/packs/usda-fooddata/policy.json +94 -94
- package/packs/usda-nass/endpoints.json +323 -323
- package/packs/usda-nass/policy.json +95 -95
- package/packs/usda-outlook/endpoints.json +86 -86
- package/packs/usda-outlook/policy.json +109 -109
- package/packs/usgs/endpoints.json +313 -313
- package/packs/usgs/policy.json +95 -95
- package/packs/usitc/endpoints.json +74 -74
- package/packs/usitc/policy.json +111 -111
- package/packs/uspto/endpoints.json +141 -141
- package/packs/uspto/policy.json +95 -95
- package/packs/uspto/source.md +1 -1
- package/packs/va/endpoints.json +414 -414
- package/packs/va/policy.json +101 -101
- package/packs/world-bank/endpoints.json +122 -122
- package/packs/world-bank/policy.json +101 -101
|
@@ -1,162 +1,164 @@
|
|
|
1
|
-
<#
|
|
2
|
-
.SYNOPSIS
|
|
3
|
-
Download the files for this analysis into the folder that holds this script. (Windows PowerShell
|
|
4
|
-
counterpart of @STEM@.sh.)
|
|
5
|
-
|
|
6
|
-
.DESCRIPTION
|
|
7
|
-
- The requests are built into this script (written by data-analyst-mcp; do not edit them).
|
|
8
|
-
- Files that need an API key: the script uses the key's environment variable when it is set, otherwise
|
|
9
|
-
asks for the key with hidden input (press Enter to skip those files). The key is never shown or saved.
|
|
10
|
-
- Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
|
|
11
|
-
- At the end it prints the files to upload to the chat.
|
|
12
|
-
|
|
13
|
-
.PARAMETER NoPause
|
|
14
|
-
Do not wait for Enter before closing the window.
|
|
15
|
-
|
|
16
|
-
.EXAMPLE
|
|
17
|
-
powershell -NoProfile -ExecutionPolicy Bypass -File .\@STEM@.ps1
|
|
18
|
-
#>
|
|
19
|
-
[CmdletBinding()]
|
|
20
|
-
param([switch]$NoPause)
|
|
21
|
-
|
|
22
|
-
$ErrorActionPreference = 'Stop'
|
|
23
|
-
$ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
|
|
24
|
-
try {
|
|
25
|
-
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
|
26
|
-
} catch { }
|
|
27
|
-
|
|
28
|
-
# Values the requests need. Columns, tab-separated: variable, kind (secret = an API key, asked with
|
|
29
|
-
# hidden input; plain = asked with visible input), where to get it.
|
|
30
|
-
$Values = @'
|
|
31
|
-
@VALUES@
|
|
32
|
-
'@
|
|
33
|
-
|
|
34
|
-
# The requests. Columns, tab-separated ("-" = empty): file, method, optional (1 = may fail), key
|
|
35
|
-
# variable, key placement (query, header, body or none), header or body field name, header prefix,
|
|
36
|
-
# other variables (comma-separated), URL, JSON body.
|
|
37
|
-
$Items = @'
|
|
38
|
-
@ITEMS@
|
|
39
|
-
'@
|
|
40
|
-
|
|
41
|
-
$Here = $PSScriptRoot
|
|
42
|
-
if (-not $Here) { $Here = (Get-Location).Path }
|
|
43
|
-
|
|
44
|
-
# Keep the window open when the script was started with "Run with PowerShell".
|
|
45
|
-
function Finish([int]$Code) {
|
|
46
|
-
$script:Vals = $null
|
|
47
|
-
if (-not $NoPause -and -not [Console]::IsInputRedirected) {
|
|
48
|
-
Write-Host ''
|
|
49
|
-
[void](Read-Host -Prompt 'Press Enter to close')
|
|
50
|
-
}
|
|
51
|
-
exit $Code
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
function Lines([string]$Text) { $Text -split "`r?`n" | Where-Object { $_.Trim() } }
|
|
55
|
-
|
|
56
|
-
# ---------------------------------------------------------------------------- keys and other values
|
|
57
|
-
$Vals = @{}
|
|
58
|
-
foreach ($line in (Lines $Values)) {
|
|
59
|
-
$var, $kind, $where = $line -split "`t", 3
|
|
60
|
-
$v = [Environment]::GetEnvironmentVariable($var)
|
|
61
|
-
if ($v) {
|
|
62
|
-
Write-Host "Using `$env:$var."
|
|
63
|
-
} elseif ([Console]::IsInputRedirected) {
|
|
64
|
-
$v = [Console]::In.ReadLine()
|
|
65
|
-
} elseif ($kind -eq 'secret') {
|
|
66
|
-
Write-Host "Some files need your $var ($where). Press Enter to skip them."
|
|
67
|
-
$secure = Read-Host -Prompt "$var (input is hidden)" -AsSecureString
|
|
68
|
-
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
|
69
|
-
try { $v = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
|
|
70
|
-
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
|
|
71
|
-
Remove-Variable secure
|
|
72
|
-
} else {
|
|
73
|
-
$v = Read-Host -Prompt "$var ($where)"
|
|
74
|
-
}
|
|
75
|
-
if ($null -eq $v) { $v = '' }
|
|
76
|
-
if ($kind -eq 'secret') { $v = $v -replace '\s', '' } else { $v = $v.Trim() }
|
|
77
|
-
$Vals[$var] = $v
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
function Get-Code($Err) {
|
|
81
|
-
if ($Err.Exception.Response) { return [int]$Err.Exception.Response.StatusCode }
|
|
82
|
-
return 0
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
# ---------------------------------------------------------------------------- download
|
|
86
|
-
$saved = @(); $failed = 0; $skipped = 0
|
|
87
|
-
foreach ($line in (Lines $Items)) {
|
|
88
|
-
$f = @($line -split "`t", 10 | ForEach-Object { if ($_ -eq '-') { '' } else { $_ } })
|
|
89
|
-
$file, $method, $optional, $kvar, $place, $kname, $kprefix, $extras, $url, $body = $f
|
|
90
|
-
$key = ''
|
|
91
|
-
if ($kvar) {
|
|
92
|
-
$key = $Vals[$kvar]
|
|
93
|
-
if (-not $key) { Write-Host "skipped (needs $kvar): $file"; $skipped++; continue }
|
|
94
|
-
}
|
|
95
|
-
$miss = ''
|
|
96
|
-
if ($extras) {
|
|
97
|
-
foreach ($x in $extras.Split(',')) {
|
|
98
|
-
if (-not $Vals[$x]) { $miss = $x; break }
|
|
99
|
-
$url = $url.Replace("{$x}", $Vals[$x])
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
if ($miss) { Write-Host "skipped (needs $miss): $file"; $skipped++; continue }
|
|
103
|
-
if ($place -eq 'query') { $url = $url.Replace('{API_KEY}', $key) }
|
|
104
|
-
|
|
105
|
-
$params = @{ Uri = $url; Method = $method; UseBasicParsing = $true; UserAgent = 'data-analyst/1.0'
|
|
106
|
-
OutFile = (Join-Path $Here "$file.part") }
|
|
107
|
-
if ($place -eq 'header') {
|
|
108
|
-
$params.Headers = @{ $kname = "$kprefix$key" }
|
|
109
|
-
$params.MaximumRedirection = 0 # a redirect would carry the key header to another host
|
|
110
|
-
}
|
|
111
|
-
if ($body) {
|
|
112
|
-
if ($place -eq 'body') { $body = $body.Replace('{API_KEY}', ($key -replace '\\', '\\' -replace '"', '\"')) }
|
|
113
|
-
$params.Body = [Text.Encoding]::UTF8.GetBytes($body)
|
|
114
|
-
$params.ContentType = 'application/json'
|
|
115
|
-
}
|
|
116
|
-
$code = 0; $msg = ''; $ok = $false
|
|
117
|
-
for ($try = 1; $try -le 4; $try++) {
|
|
118
|
-
try {
|
|
119
|
-
Invoke-WebRequest @params
|
|
120
|
-
$ok = $true
|
|
121
|
-
break
|
|
122
|
-
} catch {
|
|
123
|
-
$code = Get-Code $_
|
|
124
|
-
$msg = [string]$_.ErrorDetails.Message
|
|
125
|
-
if (-not $msg) { $msg = $_.Exception.Message }
|
|
126
|
-
if ($try -lt 4 -and ($code -eq 0 -or $code -eq 429 -or $code -ge 500)) { Start-Sleep -Seconds (10 * $try); continue }
|
|
127
|
-
break
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
$part = Join-Path $Here "$file.part"
|
|
131
|
-
$dest = Join-Path $Here $file
|
|
132
|
-
if ($ok) {
|
|
133
|
-
Move-Item -LiteralPath $part -Destination $dest -Force
|
|
134
|
-
$saved += $dest
|
|
135
|
-
Write-Host "saved $file"
|
|
136
|
-
} else {
|
|
137
|
-
if (Test-Path -LiteralPath $part) { Remove-Item -LiteralPath $part -Force }
|
|
138
|
-
|
|
139
|
-
if ($
|
|
140
|
-
if ($
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
Write-Host "
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
Finish
|
|
1
|
+
<#
|
|
2
|
+
.SYNOPSIS
|
|
3
|
+
Download the files for this analysis into the folder that holds this script. (Windows PowerShell
|
|
4
|
+
counterpart of @STEM@.sh.)
|
|
5
|
+
|
|
6
|
+
.DESCRIPTION
|
|
7
|
+
- The requests are built into this script (written by data-analyst-mcp; do not edit them).
|
|
8
|
+
- Files that need an API key: the script uses the key's environment variable when it is set, otherwise
|
|
9
|
+
asks for the key with hidden input (press Enter to skip those files). The key is never shown or saved.
|
|
10
|
+
- Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
|
|
11
|
+
- At the end it prints the files to upload to the chat.
|
|
12
|
+
|
|
13
|
+
.PARAMETER NoPause
|
|
14
|
+
Do not wait for Enter before closing the window.
|
|
15
|
+
|
|
16
|
+
.EXAMPLE
|
|
17
|
+
powershell -NoProfile -ExecutionPolicy Bypass -File .\@STEM@.ps1
|
|
18
|
+
#>
|
|
19
|
+
[CmdletBinding()]
|
|
20
|
+
param([switch]$NoPause)
|
|
21
|
+
|
|
22
|
+
$ErrorActionPreference = 'Stop'
|
|
23
|
+
$ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
|
|
24
|
+
try {
|
|
25
|
+
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
|
26
|
+
} catch { }
|
|
27
|
+
|
|
28
|
+
# Values the requests need. Columns, tab-separated: variable, kind (secret = an API key, asked with
|
|
29
|
+
# hidden input; plain = asked with visible input), where to get it.
|
|
30
|
+
$Values = @'
|
|
31
|
+
@VALUES@
|
|
32
|
+
'@
|
|
33
|
+
|
|
34
|
+
# The requests. Columns, tab-separated ("-" = empty): file, method, optional (1 = may fail), key
|
|
35
|
+
# variable, key placement (query, header, body or none), header or body field name, header prefix,
|
|
36
|
+
# other variables (comma-separated), URL, JSON body.
|
|
37
|
+
$Items = @'
|
|
38
|
+
@ITEMS@
|
|
39
|
+
'@
|
|
40
|
+
|
|
41
|
+
$Here = $PSScriptRoot
|
|
42
|
+
if (-not $Here) { $Here = (Get-Location).Path }
|
|
43
|
+
|
|
44
|
+
# Keep the window open when the script was started with "Run with PowerShell".
|
|
45
|
+
function Finish([int]$Code) {
|
|
46
|
+
$script:Vals = $null
|
|
47
|
+
if (-not $NoPause -and -not [Console]::IsInputRedirected) {
|
|
48
|
+
Write-Host ''
|
|
49
|
+
[void](Read-Host -Prompt 'Press Enter to close')
|
|
50
|
+
}
|
|
51
|
+
exit $Code
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function Lines([string]$Text) { $Text -split "`r?`n" | Where-Object { $_.Trim() } }
|
|
55
|
+
|
|
56
|
+
# ---------------------------------------------------------------------------- keys and other values
|
|
57
|
+
$Vals = @{}
|
|
58
|
+
foreach ($line in (Lines $Values)) {
|
|
59
|
+
$var, $kind, $where = $line -split "`t", 3
|
|
60
|
+
$v = [Environment]::GetEnvironmentVariable($var)
|
|
61
|
+
if ($v) {
|
|
62
|
+
Write-Host "Using `$env:$var."
|
|
63
|
+
} elseif ([Console]::IsInputRedirected) {
|
|
64
|
+
$v = [Console]::In.ReadLine()
|
|
65
|
+
} elseif ($kind -eq 'secret') {
|
|
66
|
+
Write-Host "Some files need your $var ($where). Press Enter to skip them."
|
|
67
|
+
$secure = Read-Host -Prompt "$var (input is hidden)" -AsSecureString
|
|
68
|
+
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
|
69
|
+
try { $v = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
|
|
70
|
+
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
|
|
71
|
+
Remove-Variable secure
|
|
72
|
+
} else {
|
|
73
|
+
$v = Read-Host -Prompt "$var ($where)"
|
|
74
|
+
}
|
|
75
|
+
if ($null -eq $v) { $v = '' }
|
|
76
|
+
if ($kind -eq 'secret') { $v = $v -replace '\s', '' } else { $v = $v.Trim() }
|
|
77
|
+
$Vals[$var] = $v
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function Get-Code($Err) {
|
|
81
|
+
if ($Err.Exception.Response) { return [int]$Err.Exception.Response.StatusCode }
|
|
82
|
+
return 0
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
# ---------------------------------------------------------------------------- download
|
|
86
|
+
$saved = @(); $failed = 0; $skipped = 0
|
|
87
|
+
foreach ($line in (Lines $Items)) {
|
|
88
|
+
$f = @($line -split "`t", 10 | ForEach-Object { if ($_ -eq '-') { '' } else { $_ } })
|
|
89
|
+
$file, $method, $optional, $kvar, $place, $kname, $kprefix, $extras, $url, $body = $f
|
|
90
|
+
$key = ''
|
|
91
|
+
if ($kvar) {
|
|
92
|
+
$key = $Vals[$kvar]
|
|
93
|
+
if (-not $key) { Write-Host "skipped (needs $kvar): $file"; $skipped++; continue }
|
|
94
|
+
}
|
|
95
|
+
$miss = ''
|
|
96
|
+
if ($extras) {
|
|
97
|
+
foreach ($x in $extras.Split(',')) {
|
|
98
|
+
if (-not $Vals[$x]) { $miss = $x; break }
|
|
99
|
+
$url = $url.Replace("{$x}", $Vals[$x])
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if ($miss) { Write-Host "skipped (needs $miss): $file"; $skipped++; continue }
|
|
103
|
+
if ($place -eq 'query') { $url = $url.Replace('{API_KEY}', $key) }
|
|
104
|
+
|
|
105
|
+
$params = @{ Uri = $url; Method = $method; UseBasicParsing = $true; UserAgent = 'data-analyst/1.0'
|
|
106
|
+
OutFile = (Join-Path $Here "$file.part") }
|
|
107
|
+
if ($place -eq 'header') {
|
|
108
|
+
$params.Headers = @{ $kname = "$kprefix$key" }
|
|
109
|
+
$params.MaximumRedirection = 0 # a redirect would carry the key header to another host
|
|
110
|
+
}
|
|
111
|
+
if ($body) {
|
|
112
|
+
if ($place -eq 'body') { $body = $body.Replace('{API_KEY}', ($key -replace '\\', '\\' -replace '"', '\"')) }
|
|
113
|
+
$params.Body = [Text.Encoding]::UTF8.GetBytes($body)
|
|
114
|
+
$params.ContentType = 'application/json'
|
|
115
|
+
}
|
|
116
|
+
$code = 0; $msg = ''; $ok = $false
|
|
117
|
+
for ($try = 1; $try -le 4; $try++) {
|
|
118
|
+
try {
|
|
119
|
+
Invoke-WebRequest @params
|
|
120
|
+
$ok = $true
|
|
121
|
+
break
|
|
122
|
+
} catch {
|
|
123
|
+
$code = Get-Code $_
|
|
124
|
+
$msg = [string]$_.ErrorDetails.Message
|
|
125
|
+
if (-not $msg) { $msg = $_.Exception.Message }
|
|
126
|
+
if ($try -lt 4 -and ($code -eq 0 -or $code -eq 429 -or $code -ge 500)) { Start-Sleep -Seconds (10 * $try); continue }
|
|
127
|
+
break
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
$part = Join-Path $Here "$file.part"
|
|
131
|
+
$dest = Join-Path $Here $file
|
|
132
|
+
if ($ok) {
|
|
133
|
+
Move-Item -LiteralPath $part -Destination $dest -Force
|
|
134
|
+
$saved += $dest
|
|
135
|
+
Write-Host "saved $file"
|
|
136
|
+
} else {
|
|
137
|
+
if (Test-Path -LiteralPath $part) { Remove-Item -LiteralPath $part -Force }
|
|
138
|
+
# masked before it is shortened, so no part of a key or an extra value (an email) is left at the cut
|
|
139
|
+
if ($key) { $msg = $msg.Replace($key, '[KEY]') }
|
|
140
|
+
if ($extras) { foreach ($x in $extras.Split(',')) { if ($Vals[$x]) { $msg = $msg.Replace($Vals[$x], "[$x]") } } }
|
|
141
|
+
if ($msg.Length -gt 300) { $msg = $msg.Substring(0, 300) }
|
|
142
|
+
if ($optional -eq '1') {
|
|
143
|
+
Write-Host "not available (optional, HTTP $code): $file"
|
|
144
|
+
} elseif ($code -eq 0) {
|
|
145
|
+
Write-Host "FAILED $($file): could not reach the server ($msg) Check the connection, VPN and proxy." -ForegroundColor Red
|
|
146
|
+
$failed++
|
|
147
|
+
} else {
|
|
148
|
+
Write-Host "FAILED $($file): HTTP $code $msg" -ForegroundColor Red
|
|
149
|
+
$failed++
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
Write-Host ''
|
|
155
|
+
if ($saved.Count -gt 0) {
|
|
156
|
+
Write-Host 'Done. Upload these files to the chat (do not rename or edit them):' -ForegroundColor Green
|
|
157
|
+
foreach ($p in $saved) { Write-Host " $p" }
|
|
158
|
+
}
|
|
159
|
+
if ($skipped -gt 0) { Write-Host "$skipped file(s) skipped: run again and enter the value asked for." }
|
|
160
|
+
if ($failed -gt 0) {
|
|
161
|
+
Write-Host "$failed file(s) failed: send the messages above to the analyst." -ForegroundColor Red
|
|
162
|
+
Finish 1
|
|
163
|
+
}
|
|
164
|
+
Finish 0
|
|
@@ -95,7 +95,8 @@ while IFS=$'\t' read -r file method optional kvar place kname kprefix extras url
|
|
|
95
95
|
return 0
|
|
96
96
|
}
|
|
97
97
|
args=(-sS -K - -A "data-analyst/1.0" -o "$HERE/$file.part" -w '%{http_code}')
|
|
98
|
-
# curl -L forwards custom headers (the key) to any host a redirect names: follow redirects only without one
|
|
98
|
+
# curl -L forwards custom headers (the key) to any host a redirect names: follow redirects only without one. A key
|
|
99
|
+
# in the POST body follows a 307/308 redirect
|
|
99
100
|
[ "$place" = header ] || args+=(-L)
|
|
100
101
|
if [ "$body" != "-" ]; then
|
|
101
102
|
[ "$place" = body ] && body="$(subst "$body" "{API_KEY}" "$(esc "$key")")"
|
|
@@ -120,8 +121,13 @@ while IFS=$'\t' read -r file method optional kvar place kname kprefix extras url
|
|
|
120
121
|
echo "saved $file"
|
|
121
122
|
else
|
|
122
123
|
msg=""
|
|
123
|
-
[ -f "$HERE/$file.part" ] && msg="$(head -c
|
|
124
|
+
[ -f "$HERE/$file.part" ] && msg="$(head -c 4000 "$HERE/$file.part" | tr -d '\r\n')"
|
|
125
|
+
# masked before it is shortened, so no part of a key or an extra value (an email) is left at the cut
|
|
124
126
|
[ -n "$key" ] && msg="$(subst "$msg" "$key" "[KEY]")"
|
|
127
|
+
if [ "$extras" != "-" ]; then
|
|
128
|
+
for x in ${extras//,/ }; do v="V_$x"; [ -n "${!v:-}" ] && msg="$(subst "$msg" "${!v}" "[$x]")"; done
|
|
129
|
+
fi
|
|
130
|
+
msg="${msg:0:300}"
|
|
125
131
|
rm -f "$HERE/$file.part"
|
|
126
132
|
if [ "$optional" = 1 ]; then
|
|
127
133
|
echo "not available (optional, HTTP ${code:-none}): $file"
|