data-analyst-mcp 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (292) hide show
  1. package/README.md +286 -262
  2. package/dist/adapters/bjs.d.ts +13 -0
  3. package/dist/adapters/bjs.js +28 -0
  4. package/dist/adapters/bjs.js.map +1 -0
  5. package/dist/adapters/cbp.d.ts +5 -0
  6. package/dist/adapters/cbp.js +64 -0
  7. package/dist/adapters/cbp.js.map +1 -0
  8. package/dist/adapters/cdc-wonder.d.ts +5 -0
  9. package/dist/adapters/cdc-wonder.js +68 -0
  10. package/dist/adapters/cdc-wonder.js.map +1 -0
  11. package/dist/adapters/cpsc.d.ts +4 -0
  12. package/dist/adapters/cpsc.js +125 -0
  13. package/dist/adapters/cpsc.js.map +1 -0
  14. package/dist/adapters/fhwa.d.ts +9 -0
  15. package/dist/adapters/fhwa.js +16 -0
  16. package/dist/adapters/fhwa.js.map +1 -0
  17. package/dist/adapters/foreign-assistance.d.ts +9 -0
  18. package/dist/adapters/foreign-assistance.js +42 -0
  19. package/dist/adapters/foreign-assistance.js.map +1 -0
  20. package/dist/adapters/hud-ahar.d.ts +3 -0
  21. package/dist/adapters/hud-ahar.js +80 -0
  22. package/dist/adapters/hud-ahar.js.map +1 -0
  23. package/dist/adapters/nces.d.ts +10 -0
  24. package/dist/adapters/nces.js +32 -0
  25. package/dist/adapters/nces.js.map +1 -0
  26. package/dist/adapters/nsf.d.ts +11 -0
  27. package/dist/adapters/nsf.js +25 -0
  28. package/dist/adapters/nsf.js.map +1 -0
  29. package/dist/adapters/onrr.d.ts +9 -0
  30. package/dist/adapters/onrr.js +35 -0
  31. package/dist/adapters/onrr.js.map +1 -0
  32. package/dist/adapters/sba.d.ts +10 -0
  33. package/dist/adapters/sba.js +134 -0
  34. package/dist/adapters/sba.js.map +1 -0
  35. package/dist/adapters/treasury-tic.d.ts +30 -0
  36. package/dist/adapters/treasury-tic.js +149 -0
  37. package/dist/adapters/treasury-tic.js.map +1 -0
  38. package/dist/adapters/uscourts.d.ts +8 -0
  39. package/dist/adapters/uscourts.js +8 -0
  40. package/dist/adapters/uscourts.js.map +1 -0
  41. package/dist/analysis/cite.d.ts +9 -1
  42. package/dist/analysis/cite.js +9 -2
  43. package/dist/analysis/cite.js.map +1 -1
  44. package/dist/bin/cli.d.ts +2 -0
  45. package/dist/bin/cli.js +17 -0
  46. package/dist/bin/cli.js.map +1 -1
  47. package/dist/core/generic-rest.js +42 -17
  48. package/dist/core/generic-rest.js.map +1 -1
  49. package/dist/core/guard.d.ts +32 -0
  50. package/dist/core/guard.js +176 -0
  51. package/dist/core/guard.js.map +1 -0
  52. package/dist/core/http.d.ts +10 -0
  53. package/dist/core/http.js +36 -7
  54. package/dist/core/http.js.map +1 -1
  55. package/dist/core/keys.d.ts +13 -3
  56. package/dist/core/keys.js +67 -7
  57. package/dist/core/keys.js.map +1 -1
  58. package/dist/core/package.d.ts +12 -3
  59. package/dist/core/package.js +56 -12
  60. package/dist/core/package.js.map +1 -1
  61. package/dist/core/secret_patterns.d.ts +6 -0
  62. package/dist/core/secret_patterns.js +20 -0
  63. package/dist/core/secret_patterns.js.map +1 -0
  64. package/dist/core/tabular.d.ts +43 -1
  65. package/dist/core/tabular.js +149 -6
  66. package/dist/core/tabular.js.map +1 -1
  67. package/dist/envelope.js +3 -1
  68. package/dist/envelope.js.map +1 -1
  69. package/dist/index.d.ts +29 -22
  70. package/dist/index.js +45 -46
  71. package/dist/index.js.map +1 -1
  72. package/dist/remote/app.js +87 -31
  73. package/dist/remote/app.js.map +1 -1
  74. package/dist/remote/oauth.js +8 -1
  75. package/dist/remote/oauth.js.map +1 -1
  76. package/dist/remote/pages.d.ts +3 -7
  77. package/dist/remote/pages.js +49 -19
  78. package/dist/remote/pages.js.map +1 -1
  79. package/dist/remote/sign.d.ts +7 -1
  80. package/dist/remote/sign.js +33 -7
  81. package/dist/remote/sign.js.map +1 -1
  82. package/dist/remote/store.js +14 -5
  83. package/dist/remote/store.js.map +1 -1
  84. package/dist/report/audit_report.js +29 -7
  85. package/dist/report/audit_report.js.map +1 -1
  86. package/dist/report/build_report.js +34 -4
  87. package/dist/report/build_report.js.map +1 -1
  88. package/dist/report/method_audit.js +10 -1
  89. package/dist/report/method_audit.js.map +1 -1
  90. package/dist/report/scan_secrets.d.ts +4 -1
  91. package/dist/report/scan_secrets.js +38 -18
  92. package/dist/report/scan_secrets.js.map +1 -1
  93. package/dist/server.d.ts +3 -1
  94. package/dist/server.js +46 -13
  95. package/dist/server.js.map +1 -1
  96. package/dist/tools.js +27 -6
  97. package/dist/tools.js.map +1 -1
  98. package/guide/WORKFLOW.md +134 -134
  99. package/guide/assets/disclaimer.txt +1 -1
  100. package/guide/assets/download_data.ps1 +164 -162
  101. package/guide/assets/download_data.sh +8 -2
  102. package/guide/assets/report_template.html +597 -597
  103. package/guide/references/routing.md +198 -171
  104. package/guide/references/style_guide.md +398 -398
  105. package/package.json +56 -56
  106. package/packs/acf/endpoints.json +114 -114
  107. package/packs/acf/policy.json +104 -104
  108. package/packs/actuaries/endpoints.json +82 -82
  109. package/packs/actuaries/policy.json +109 -109
  110. package/packs/bea/assets/download_bea.ps1 +182 -182
  111. package/packs/bea/docs/bea_api.md +461 -461
  112. package/packs/bea/docs/datasets.md +157 -157
  113. package/packs/bea/docs/report_patterns.md +112 -112
  114. package/packs/bea/policy.json +170 -170
  115. package/packs/bea/source.md +152 -152
  116. package/packs/bjs/endpoints.json +150 -0
  117. package/packs/bjs/policy.json +114 -0
  118. package/packs/bjs/source.md +98 -0
  119. package/packs/bls/assets/download_bls.ps1 +152 -152
  120. package/packs/bls/docs/analysis_patterns.md +108 -108
  121. package/packs/bls/docs/bls_api.md +73 -73
  122. package/packs/bls/docs/series_ids.md +149 -149
  123. package/packs/bls/policy.json +238 -238
  124. package/packs/bts/endpoints.json +346 -346
  125. package/packs/bts/policy.json +94 -94
  126. package/packs/cbp/endpoints.json +31 -0
  127. package/packs/cbp/policy.json +85 -0
  128. package/packs/cbp/source.md +57 -0
  129. package/packs/cdc/endpoints.json +197 -197
  130. package/packs/cdc/policy.json +83 -83
  131. package/packs/cdc/source.md +1 -1
  132. package/packs/cdc-wonder/endpoints.json +26 -0
  133. package/packs/cdc-wonder/policy.json +84 -0
  134. package/packs/cdc-wonder/source.md +53 -0
  135. package/packs/census/endpoints.json +99 -99
  136. package/packs/cfpb/endpoints.json +326 -326
  137. package/packs/cfpb/policy.json +119 -119
  138. package/packs/clinical-trials/endpoints.json +166 -166
  139. package/packs/clinical-trials/policy.json +84 -84
  140. package/packs/cms/endpoints.json +69 -69
  141. package/packs/cms/policy.json +96 -96
  142. package/packs/college-scorecard/endpoints.json +252 -252
  143. package/packs/college-scorecard/policy.json +106 -106
  144. package/packs/college-scorecard/source.md +1 -1
  145. package/packs/congress/endpoints.json +1701 -1701
  146. package/packs/congress/policy.json +117 -117
  147. package/packs/cpsc/endpoints.json +91 -0
  148. package/packs/cpsc/policy.json +91 -0
  149. package/packs/cpsc/source.md +76 -0
  150. package/packs/dhs-ohss/endpoints.json +66 -66
  151. package/packs/dhs-ohss/policy.json +107 -107
  152. package/packs/dhs-ohss/source.md +1 -1
  153. package/packs/doj-news/endpoints.json +59 -59
  154. package/packs/doj-news/policy.json +96 -96
  155. package/packs/doj-news/source.md +1 -1
  156. package/packs/dol/endpoints.json +817 -817
  157. package/packs/dol/policy.json +135 -135
  158. package/packs/eia/endpoints.json +717 -717
  159. package/packs/eia/policy.json +112 -112
  160. package/packs/epa/endpoints.json +291 -291
  161. package/packs/epa/policy.json +99 -99
  162. package/packs/epa-aqs/endpoints.json +173 -173
  163. package/packs/epa-aqs/policy.json +96 -96
  164. package/packs/fbi/endpoints.json +1841 -1841
  165. package/packs/fbi/policy.json +118 -118
  166. package/packs/fbi/source.md +3 -3
  167. package/packs/fda/endpoints.json +237 -237
  168. package/packs/fda/policy.json +121 -121
  169. package/packs/fdic/endpoints.json +416 -416
  170. package/packs/fdic/policy.json +83 -83
  171. package/packs/fec/endpoints.json +707 -707
  172. package/packs/fec/policy.json +93 -93
  173. package/packs/fed-board/endpoints.json +89 -89
  174. package/packs/fed-board/policy.json +136 -136
  175. package/packs/federal-register/endpoints.json +143 -143
  176. package/packs/federal-register/policy.json +86 -86
  177. package/packs/fema/endpoints.json +645 -645
  178. package/packs/fema/policy.json +103 -103
  179. package/packs/fhfa/endpoints.json +105 -105
  180. package/packs/fhfa/policy.json +109 -109
  181. package/packs/fhwa/endpoints.json +48 -0
  182. package/packs/fhwa/policy.json +91 -0
  183. package/packs/fhwa/source.md +59 -0
  184. package/packs/foreign-assistance/endpoints.json +166 -0
  185. package/packs/foreign-assistance/policy.json +80 -0
  186. package/packs/foreign-assistance/source.md +76 -0
  187. package/packs/fred/assets/download_fred.ps1 +122 -122
  188. package/packs/fred/docs/analysis_patterns.md +124 -124
  189. package/packs/fred/docs/fred_api.md +94 -94
  190. package/packs/fred/docs/series.md +188 -188
  191. package/packs/fred/policy.json +253 -253
  192. package/packs/fsa/endpoints.json +150 -150
  193. package/packs/fsa/policy.json +99 -99
  194. package/packs/fsa/source.md +1 -1
  195. package/packs/govinfo/endpoints.json +519 -519
  196. package/packs/gsa-calc/endpoints.json +93 -93
  197. package/packs/gsa-calc/policy.json +97 -97
  198. package/packs/hud/endpoints.json +146 -146
  199. package/packs/hud/policy.json +84 -84
  200. package/packs/hud/source.md +1 -1
  201. package/packs/hud-ahar/endpoints.json +92 -0
  202. package/packs/hud-ahar/policy.json +88 -0
  203. package/packs/hud-ahar/source.md +72 -0
  204. package/packs/ipums/docs/report_patterns.md +117 -117
  205. package/packs/irs-soi/endpoints.json +114 -114
  206. package/packs/irs-soi/policy.json +107 -107
  207. package/packs/medicaid/endpoints.json +412 -412
  208. package/packs/medicaid/policy.json +108 -108
  209. package/packs/naep/endpoints.json +217 -217
  210. package/packs/naep/policy.json +100 -100
  211. package/packs/naep/source.md +1 -1
  212. package/packs/nces/endpoints.json +101 -0
  213. package/packs/nces/policy.json +99 -0
  214. package/packs/nces/source.md +75 -0
  215. package/packs/nhtsa/endpoints.json +132 -132
  216. package/packs/nhtsa/policy.json +93 -93
  217. package/packs/nhtsa/source.md +1 -1
  218. package/packs/nih/endpoints.json +452 -452
  219. package/packs/nih/policy.json +86 -86
  220. package/packs/noaa/endpoints.json +431 -431
  221. package/packs/noaa/policy.json +95 -95
  222. package/packs/nrel/endpoints.json +327 -327
  223. package/packs/nrel/policy.json +96 -96
  224. package/packs/nsf/endpoints.json +33 -0
  225. package/packs/nsf/policy.json +83 -0
  226. package/packs/nsf/source.md +59 -0
  227. package/packs/nws/endpoints.json +271 -271
  228. package/packs/nws/policy.json +82 -82
  229. package/packs/nyfed/endpoints.json +80 -80
  230. package/packs/nyfed/policy.json +92 -92
  231. package/packs/omb/endpoints.json +460 -460
  232. package/packs/omb/policy.json +106 -106
  233. package/packs/onrr/endpoints.json +198 -0
  234. package/packs/onrr/policy.json +91 -0
  235. package/packs/onrr/source.md +55 -0
  236. package/packs/open-payments/endpoints.json +297 -297
  237. package/packs/open-payments/policy.json +101 -101
  238. package/packs/opm/assets/download_opm.ps1 +125 -125
  239. package/packs/opm/docs/fields.md +184 -184
  240. package/packs/opm/docs/opm_api.md +71 -71
  241. package/packs/opm/docs/report_patterns.md +83 -83
  242. package/packs/opm/lib/data_dictionary.csv +73 -73
  243. package/packs/pbgc/endpoints.json +483 -483
  244. package/packs/pbgc/policy.json +94 -94
  245. package/packs/philfed/endpoints.json +85 -85
  246. package/packs/philfed/policy.json +94 -94
  247. package/packs/port.json +65 -0
  248. package/packs/regulations/endpoints.json +287 -287
  249. package/packs/regulations/policy.json +86 -86
  250. package/packs/sba/endpoints.json +34 -0
  251. package/packs/sba/policy.json +86 -0
  252. package/packs/sba/source.md +60 -0
  253. package/packs/sec/endpoints.json +236 -236
  254. package/packs/sec/policy.json +86 -86
  255. package/packs/senate-lobbying/endpoints.json +553 -553
  256. package/packs/senate-lobbying/policy.json +99 -99
  257. package/packs/snapqc/endpoints.json +1426 -1426
  258. package/packs/snapqc/policy.json +92 -92
  259. package/packs/treasury/catalog.json +2267 -2267
  260. package/packs/treasury/docs/api_guide.md +245 -245
  261. package/packs/treasury/docs/datasets.md +56 -56
  262. package/packs/treasury/docs/report_patterns.md +126 -126
  263. package/packs/treasury/policy.json +159 -159
  264. package/packs/treasury/source.md +170 -165
  265. package/packs/treasury-tic/endpoints.json +77 -0
  266. package/packs/treasury-tic/policy.json +89 -0
  267. package/packs/treasury-tic/source.md +80 -0
  268. package/packs/treasury-yields/endpoints.json +226 -226
  269. package/packs/treasury-yields/policy.json +86 -86
  270. package/packs/usaspending/endpoints.json +1315 -1315
  271. package/packs/usaspending/policy.json +109 -109
  272. package/packs/uscourts/endpoints.json +22 -0
  273. package/packs/uscourts/policy.json +87 -0
  274. package/packs/uscourts/source.md +56 -0
  275. package/packs/usda-fna/policy.json +133 -133
  276. package/packs/usda-fooddata/endpoints.json +180 -180
  277. package/packs/usda-fooddata/policy.json +94 -94
  278. package/packs/usda-nass/endpoints.json +323 -323
  279. package/packs/usda-nass/policy.json +95 -95
  280. package/packs/usda-outlook/endpoints.json +86 -86
  281. package/packs/usda-outlook/policy.json +109 -109
  282. package/packs/usgs/endpoints.json +313 -313
  283. package/packs/usgs/policy.json +95 -95
  284. package/packs/usitc/endpoints.json +74 -74
  285. package/packs/usitc/policy.json +111 -111
  286. package/packs/uspto/endpoints.json +141 -141
  287. package/packs/uspto/policy.json +95 -95
  288. package/packs/uspto/source.md +1 -1
  289. package/packs/va/endpoints.json +414 -414
  290. package/packs/va/policy.json +101 -101
  291. package/packs/world-bank/endpoints.json +122 -122
  292. package/packs/world-bank/policy.json +101 -101
@@ -1,162 +1,164 @@
1
- <#
2
- .SYNOPSIS
3
- Download the files for this analysis into the folder that holds this script. (Windows PowerShell
4
- counterpart of @STEM@.sh.)
5
-
6
- .DESCRIPTION
7
- - The requests are built into this script (written by data-analyst-mcp; do not edit them).
8
- - Files that need an API key: the script uses the key's environment variable when it is set, otherwise
9
- asks for the key with hidden input (press Enter to skip those files). The key is never shown or saved.
10
- - Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
11
- - At the end it prints the files to upload to the chat.
12
-
13
- .PARAMETER NoPause
14
- Do not wait for Enter before closing the window.
15
-
16
- .EXAMPLE
17
- powershell -NoProfile -ExecutionPolicy Bypass -File .\@STEM@.ps1
18
- #>
19
- [CmdletBinding()]
20
- param([switch]$NoPause)
21
-
22
- $ErrorActionPreference = 'Stop'
23
- $ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
24
- try {
25
- [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
26
- } catch { }
27
-
28
- # Values the requests need. Columns, tab-separated: variable, kind (secret = an API key, asked with
29
- # hidden input; plain = asked with visible input), where to get it.
30
- $Values = @'
31
- @VALUES@
32
- '@
33
-
34
- # The requests. Columns, tab-separated ("-" = empty): file, method, optional (1 = may fail), key
35
- # variable, key placement (query, header, body or none), header or body field name, header prefix,
36
- # other variables (comma-separated), URL, JSON body.
37
- $Items = @'
38
- @ITEMS@
39
- '@
40
-
41
- $Here = $PSScriptRoot
42
- if (-not $Here) { $Here = (Get-Location).Path }
43
-
44
- # Keep the window open when the script was started with "Run with PowerShell".
45
- function Finish([int]$Code) {
46
- $script:Vals = $null
47
- if (-not $NoPause -and -not [Console]::IsInputRedirected) {
48
- Write-Host ''
49
- [void](Read-Host -Prompt 'Press Enter to close')
50
- }
51
- exit $Code
52
- }
53
-
54
- function Lines([string]$Text) { $Text -split "`r?`n" | Where-Object { $_.Trim() } }
55
-
56
- # ---------------------------------------------------------------------------- keys and other values
57
- $Vals = @{}
58
- foreach ($line in (Lines $Values)) {
59
- $var, $kind, $where = $line -split "`t", 3
60
- $v = [Environment]::GetEnvironmentVariable($var)
61
- if ($v) {
62
- Write-Host "Using `$env:$var."
63
- } elseif ([Console]::IsInputRedirected) {
64
- $v = [Console]::In.ReadLine()
65
- } elseif ($kind -eq 'secret') {
66
- Write-Host "Some files need your $var ($where). Press Enter to skip them."
67
- $secure = Read-Host -Prompt "$var (input is hidden)" -AsSecureString
68
- $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
69
- try { $v = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
70
- finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
71
- Remove-Variable secure
72
- } else {
73
- $v = Read-Host -Prompt "$var ($where)"
74
- }
75
- if ($null -eq $v) { $v = '' }
76
- if ($kind -eq 'secret') { $v = $v -replace '\s', '' } else { $v = $v.Trim() }
77
- $Vals[$var] = $v
78
- }
79
-
80
- function Get-Code($Err) {
81
- if ($Err.Exception.Response) { return [int]$Err.Exception.Response.StatusCode }
82
- return 0
83
- }
84
-
85
- # ---------------------------------------------------------------------------- download
86
- $saved = @(); $failed = 0; $skipped = 0
87
- foreach ($line in (Lines $Items)) {
88
- $f = @($line -split "`t", 10 | ForEach-Object { if ($_ -eq '-') { '' } else { $_ } })
89
- $file, $method, $optional, $kvar, $place, $kname, $kprefix, $extras, $url, $body = $f
90
- $key = ''
91
- if ($kvar) {
92
- $key = $Vals[$kvar]
93
- if (-not $key) { Write-Host "skipped (needs $kvar): $file"; $skipped++; continue }
94
- }
95
- $miss = ''
96
- if ($extras) {
97
- foreach ($x in $extras.Split(',')) {
98
- if (-not $Vals[$x]) { $miss = $x; break }
99
- $url = $url.Replace("{$x}", $Vals[$x])
100
- }
101
- }
102
- if ($miss) { Write-Host "skipped (needs $miss): $file"; $skipped++; continue }
103
- if ($place -eq 'query') { $url = $url.Replace('{API_KEY}', $key) }
104
-
105
- $params = @{ Uri = $url; Method = $method; UseBasicParsing = $true; UserAgent = 'data-analyst/1.0'
106
- OutFile = (Join-Path $Here "$file.part") }
107
- if ($place -eq 'header') {
108
- $params.Headers = @{ $kname = "$kprefix$key" }
109
- $params.MaximumRedirection = 0 # a redirect would carry the key header to another host
110
- }
111
- if ($body) {
112
- if ($place -eq 'body') { $body = $body.Replace('{API_KEY}', ($key -replace '\\', '\\' -replace '"', '\"')) }
113
- $params.Body = [Text.Encoding]::UTF8.GetBytes($body)
114
- $params.ContentType = 'application/json'
115
- }
116
- $code = 0; $msg = ''; $ok = $false
117
- for ($try = 1; $try -le 4; $try++) {
118
- try {
119
- Invoke-WebRequest @params
120
- $ok = $true
121
- break
122
- } catch {
123
- $code = Get-Code $_
124
- $msg = [string]$_.ErrorDetails.Message
125
- if (-not $msg) { $msg = $_.Exception.Message }
126
- if ($try -lt 4 -and ($code -eq 0 -or $code -eq 429 -or $code -ge 500)) { Start-Sleep -Seconds (10 * $try); continue }
127
- break
128
- }
129
- }
130
- $part = Join-Path $Here "$file.part"
131
- $dest = Join-Path $Here $file
132
- if ($ok) {
133
- Move-Item -LiteralPath $part -Destination $dest -Force
134
- $saved += $dest
135
- Write-Host "saved $file"
136
- } else {
137
- if (Test-Path -LiteralPath $part) { Remove-Item -LiteralPath $part -Force }
138
- if ($key) { $msg = $msg.Replace($key, '[KEY]') }
139
- if ($msg.Length -gt 300) { $msg = $msg.Substring(0, 300) }
140
- if ($optional -eq '1') {
141
- Write-Host "not available (optional, HTTP $code): $file"
142
- } elseif ($code -eq 0) {
143
- Write-Host "FAILED $($file): could not reach the server ($msg) Check the connection, VPN and proxy." -ForegroundColor Red
144
- $failed++
145
- } else {
146
- Write-Host "FAILED $($file): HTTP $code $msg" -ForegroundColor Red
147
- $failed++
148
- }
149
- }
150
- }
151
-
152
- Write-Host ''
153
- if ($saved.Count -gt 0) {
154
- Write-Host 'Done. Upload these files to the chat (do not rename or edit them):' -ForegroundColor Green
155
- foreach ($p in $saved) { Write-Host " $p" }
156
- }
157
- if ($skipped -gt 0) { Write-Host "$skipped file(s) skipped: run again and enter the value asked for." }
158
- if ($failed -gt 0) {
159
- Write-Host "$failed file(s) failed: send the messages above to the analyst." -ForegroundColor Red
160
- Finish 1
161
- }
162
- Finish 0
1
+ <#
2
+ .SYNOPSIS
3
+ Download the files for this analysis into the folder that holds this script. (Windows PowerShell
4
+ counterpart of @STEM@.sh.)
5
+
6
+ .DESCRIPTION
7
+ - The requests are built into this script (written by data-analyst-mcp; do not edit them).
8
+ - Files that need an API key: the script uses the key's environment variable when it is set, otherwise
9
+ asks for the key with hidden input (press Enter to skip those files). The key is never shown or saved.
10
+ - Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
11
+ - At the end it prints the files to upload to the chat.
12
+
13
+ .PARAMETER NoPause
14
+ Do not wait for Enter before closing the window.
15
+
16
+ .EXAMPLE
17
+ powershell -NoProfile -ExecutionPolicy Bypass -File .\@STEM@.ps1
18
+ #>
19
+ [CmdletBinding()]
20
+ param([switch]$NoPause)
21
+
22
+ $ErrorActionPreference = 'Stop'
23
+ $ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
24
+ try {
25
+ [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
26
+ } catch { }
27
+
28
+ # Values the requests need. Columns, tab-separated: variable, kind (secret = an API key, asked with
29
+ # hidden input; plain = asked with visible input), where to get it.
30
+ $Values = @'
31
+ @VALUES@
32
+ '@
33
+
34
+ # The requests. Columns, tab-separated ("-" = empty): file, method, optional (1 = may fail), key
35
+ # variable, key placement (query, header, body or none), header or body field name, header prefix,
36
+ # other variables (comma-separated), URL, JSON body.
37
+ $Items = @'
38
+ @ITEMS@
39
+ '@
40
+
41
+ $Here = $PSScriptRoot
42
+ if (-not $Here) { $Here = (Get-Location).Path }
43
+
44
+ # Keep the window open when the script was started with "Run with PowerShell".
45
+ function Finish([int]$Code) {
46
+ $script:Vals = $null
47
+ if (-not $NoPause -and -not [Console]::IsInputRedirected) {
48
+ Write-Host ''
49
+ [void](Read-Host -Prompt 'Press Enter to close')
50
+ }
51
+ exit $Code
52
+ }
53
+
54
+ function Lines([string]$Text) { $Text -split "`r?`n" | Where-Object { $_.Trim() } }
55
+
56
+ # ---------------------------------------------------------------------------- keys and other values
57
+ $Vals = @{}
58
+ foreach ($line in (Lines $Values)) {
59
+ $var, $kind, $where = $line -split "`t", 3
60
+ $v = [Environment]::GetEnvironmentVariable($var)
61
+ if ($v) {
62
+ Write-Host "Using `$env:$var."
63
+ } elseif ([Console]::IsInputRedirected) {
64
+ $v = [Console]::In.ReadLine()
65
+ } elseif ($kind -eq 'secret') {
66
+ Write-Host "Some files need your $var ($where). Press Enter to skip them."
67
+ $secure = Read-Host -Prompt "$var (input is hidden)" -AsSecureString
68
+ $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
69
+ try { $v = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
70
+ finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
71
+ Remove-Variable secure
72
+ } else {
73
+ $v = Read-Host -Prompt "$var ($where)"
74
+ }
75
+ if ($null -eq $v) { $v = '' }
76
+ if ($kind -eq 'secret') { $v = $v -replace '\s', '' } else { $v = $v.Trim() }
77
+ $Vals[$var] = $v
78
+ }
79
+
80
+ function Get-Code($Err) {
81
+ if ($Err.Exception.Response) { return [int]$Err.Exception.Response.StatusCode }
82
+ return 0
83
+ }
84
+
85
+ # ---------------------------------------------------------------------------- download
86
+ $saved = @(); $failed = 0; $skipped = 0
87
+ foreach ($line in (Lines $Items)) {
88
+ $f = @($line -split "`t", 10 | ForEach-Object { if ($_ -eq '-') { '' } else { $_ } })
89
+ $file, $method, $optional, $kvar, $place, $kname, $kprefix, $extras, $url, $body = $f
90
+ $key = ''
91
+ if ($kvar) {
92
+ $key = $Vals[$kvar]
93
+ if (-not $key) { Write-Host "skipped (needs $kvar): $file"; $skipped++; continue }
94
+ }
95
+ $miss = ''
96
+ if ($extras) {
97
+ foreach ($x in $extras.Split(',')) {
98
+ if (-not $Vals[$x]) { $miss = $x; break }
99
+ $url = $url.Replace("{$x}", $Vals[$x])
100
+ }
101
+ }
102
+ if ($miss) { Write-Host "skipped (needs $miss): $file"; $skipped++; continue }
103
+ if ($place -eq 'query') { $url = $url.Replace('{API_KEY}', $key) }
104
+
105
+ $params = @{ Uri = $url; Method = $method; UseBasicParsing = $true; UserAgent = 'data-analyst/1.0'
106
+ OutFile = (Join-Path $Here "$file.part") }
107
+ if ($place -eq 'header') {
108
+ $params.Headers = @{ $kname = "$kprefix$key" }
109
+ $params.MaximumRedirection = 0 # a redirect would carry the key header to another host
110
+ }
111
+ if ($body) {
112
+ if ($place -eq 'body') { $body = $body.Replace('{API_KEY}', ($key -replace '\\', '\\' -replace '"', '\"')) }
113
+ $params.Body = [Text.Encoding]::UTF8.GetBytes($body)
114
+ $params.ContentType = 'application/json'
115
+ }
116
+ $code = 0; $msg = ''; $ok = $false
117
+ for ($try = 1; $try -le 4; $try++) {
118
+ try {
119
+ Invoke-WebRequest @params
120
+ $ok = $true
121
+ break
122
+ } catch {
123
+ $code = Get-Code $_
124
+ $msg = [string]$_.ErrorDetails.Message
125
+ if (-not $msg) { $msg = $_.Exception.Message }
126
+ if ($try -lt 4 -and ($code -eq 0 -or $code -eq 429 -or $code -ge 500)) { Start-Sleep -Seconds (10 * $try); continue }
127
+ break
128
+ }
129
+ }
130
+ $part = Join-Path $Here "$file.part"
131
+ $dest = Join-Path $Here $file
132
+ if ($ok) {
133
+ Move-Item -LiteralPath $part -Destination $dest -Force
134
+ $saved += $dest
135
+ Write-Host "saved $file"
136
+ } else {
137
+ if (Test-Path -LiteralPath $part) { Remove-Item -LiteralPath $part -Force }
138
+ # masked before it is shortened, so no part of a key or an extra value (an email) is left at the cut
139
+ if ($key) { $msg = $msg.Replace($key, '[KEY]') }
140
+ if ($extras) { foreach ($x in $extras.Split(',')) { if ($Vals[$x]) { $msg = $msg.Replace($Vals[$x], "[$x]") } } }
141
+ if ($msg.Length -gt 300) { $msg = $msg.Substring(0, 300) }
142
+ if ($optional -eq '1') {
143
+ Write-Host "not available (optional, HTTP $code): $file"
144
+ } elseif ($code -eq 0) {
145
+ Write-Host "FAILED $($file): could not reach the server ($msg) Check the connection, VPN and proxy." -ForegroundColor Red
146
+ $failed++
147
+ } else {
148
+ Write-Host "FAILED $($file): HTTP $code $msg" -ForegroundColor Red
149
+ $failed++
150
+ }
151
+ }
152
+ }
153
+
154
+ Write-Host ''
155
+ if ($saved.Count -gt 0) {
156
+ Write-Host 'Done. Upload these files to the chat (do not rename or edit them):' -ForegroundColor Green
157
+ foreach ($p in $saved) { Write-Host " $p" }
158
+ }
159
+ if ($skipped -gt 0) { Write-Host "$skipped file(s) skipped: run again and enter the value asked for." }
160
+ if ($failed -gt 0) {
161
+ Write-Host "$failed file(s) failed: send the messages above to the analyst." -ForegroundColor Red
162
+ Finish 1
163
+ }
164
+ Finish 0
@@ -95,7 +95,8 @@ while IFS=$'\t' read -r file method optional kvar place kname kprefix extras url
95
95
  return 0
96
96
  }
97
97
  args=(-sS -K - -A "data-analyst/1.0" -o "$HERE/$file.part" -w '%{http_code}')
98
- # curl -L forwards custom headers (the key) to any host a redirect names: follow redirects only without one
98
+ # curl -L forwards custom headers (the key) to any host a redirect names: follow redirects only without one. A key
99
+ # in the POST body follows a 307/308 redirect
99
100
  [ "$place" = header ] || args+=(-L)
100
101
  if [ "$body" != "-" ]; then
101
102
  [ "$place" = body ] && body="$(subst "$body" "{API_KEY}" "$(esc "$key")")"
@@ -120,8 +121,13 @@ while IFS=$'\t' read -r file method optional kvar place kname kprefix extras url
120
121
  echo "saved $file"
121
122
  else
122
123
  msg=""
123
- [ -f "$HERE/$file.part" ] && msg="$(head -c 300 "$HERE/$file.part" | tr -d '\r\n')"
124
+ [ -f "$HERE/$file.part" ] && msg="$(head -c 4000 "$HERE/$file.part" | tr -d '\r\n')"
125
+ # masked before it is shortened, so no part of a key or an extra value (an email) is left at the cut
124
126
  [ -n "$key" ] && msg="$(subst "$msg" "$key" "[KEY]")"
127
+ if [ "$extras" != "-" ]; then
128
+ for x in ${extras//,/ }; do v="V_$x"; [ -n "${!v:-}" ] && msg="$(subst "$msg" "${!v}" "[$x]")"; done
129
+ fi
130
+ msg="${msg:0:300}"
125
131
  rm -f "$HERE/$file.part"
126
132
  if [ "$optional" = 1 ]; then
127
133
  echo "not available (optional, HTTP ${code:-none}): $file"