data-analyst-mcp 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/README.md +262 -262
  2. package/dist/report/method_audit.js +7 -1
  3. package/dist/report/method_audit.js.map +1 -1
  4. package/dist/report/scan_secrets.d.ts +4 -1
  5. package/dist/report/scan_secrets.js +36 -5
  6. package/dist/report/scan_secrets.js.map +1 -1
  7. package/dist/server.d.ts +1 -1
  8. package/dist/server.js +3 -2
  9. package/dist/server.js.map +1 -1
  10. package/guide/WORKFLOW.md +134 -134
  11. package/guide/assets/disclaimer.txt +1 -1
  12. package/guide/assets/download_data.ps1 +162 -162
  13. package/guide/assets/report_template.html +597 -597
  14. package/guide/references/routing.md +171 -171
  15. package/guide/references/style_guide.md +398 -398
  16. package/package.json +56 -56
  17. package/packs/acf/endpoints.json +114 -114
  18. package/packs/acf/policy.json +104 -104
  19. package/packs/actuaries/endpoints.json +82 -82
  20. package/packs/actuaries/policy.json +109 -109
  21. package/packs/bea/assets/download_bea.ps1 +182 -182
  22. package/packs/bea/docs/bea_api.md +461 -461
  23. package/packs/bea/docs/datasets.md +157 -157
  24. package/packs/bea/docs/report_patterns.md +112 -112
  25. package/packs/bea/policy.json +170 -170
  26. package/packs/bea/source.md +152 -152
  27. package/packs/bls/assets/download_bls.ps1 +152 -152
  28. package/packs/bls/docs/analysis_patterns.md +108 -108
  29. package/packs/bls/docs/bls_api.md +73 -73
  30. package/packs/bls/docs/series_ids.md +149 -149
  31. package/packs/bls/policy.json +238 -238
  32. package/packs/bts/endpoints.json +346 -346
  33. package/packs/bts/policy.json +94 -94
  34. package/packs/cdc/endpoints.json +197 -197
  35. package/packs/cdc/policy.json +83 -83
  36. package/packs/census/endpoints.json +99 -99
  37. package/packs/cfpb/endpoints.json +326 -326
  38. package/packs/cfpb/policy.json +119 -119
  39. package/packs/clinical-trials/endpoints.json +166 -166
  40. package/packs/clinical-trials/policy.json +84 -84
  41. package/packs/cms/endpoints.json +69 -69
  42. package/packs/cms/policy.json +96 -96
  43. package/packs/college-scorecard/endpoints.json +252 -252
  44. package/packs/college-scorecard/policy.json +106 -106
  45. package/packs/congress/endpoints.json +1701 -1701
  46. package/packs/congress/policy.json +117 -117
  47. package/packs/dhs-ohss/endpoints.json +66 -66
  48. package/packs/dhs-ohss/policy.json +107 -107
  49. package/packs/doj-news/endpoints.json +59 -59
  50. package/packs/doj-news/policy.json +96 -96
  51. package/packs/dol/endpoints.json +817 -817
  52. package/packs/dol/policy.json +135 -135
  53. package/packs/eia/endpoints.json +717 -717
  54. package/packs/eia/policy.json +112 -112
  55. package/packs/epa/endpoints.json +291 -291
  56. package/packs/epa/policy.json +99 -99
  57. package/packs/epa-aqs/endpoints.json +173 -173
  58. package/packs/epa-aqs/policy.json +96 -96
  59. package/packs/fbi/endpoints.json +1841 -1841
  60. package/packs/fbi/policy.json +118 -118
  61. package/packs/fda/endpoints.json +237 -237
  62. package/packs/fda/policy.json +121 -121
  63. package/packs/fdic/endpoints.json +416 -416
  64. package/packs/fdic/policy.json +83 -83
  65. package/packs/fec/endpoints.json +707 -707
  66. package/packs/fec/policy.json +93 -93
  67. package/packs/fed-board/endpoints.json +89 -89
  68. package/packs/fed-board/policy.json +136 -136
  69. package/packs/federal-register/endpoints.json +143 -143
  70. package/packs/federal-register/policy.json +86 -86
  71. package/packs/fema/endpoints.json +645 -645
  72. package/packs/fema/policy.json +103 -103
  73. package/packs/fhfa/endpoints.json +105 -105
  74. package/packs/fhfa/policy.json +109 -109
  75. package/packs/fred/assets/download_fred.ps1 +122 -122
  76. package/packs/fred/docs/analysis_patterns.md +124 -124
  77. package/packs/fred/docs/fred_api.md +94 -94
  78. package/packs/fred/docs/series.md +188 -188
  79. package/packs/fred/policy.json +253 -253
  80. package/packs/fsa/endpoints.json +150 -150
  81. package/packs/fsa/policy.json +99 -99
  82. package/packs/govinfo/endpoints.json +519 -519
  83. package/packs/gsa-calc/endpoints.json +93 -93
  84. package/packs/gsa-calc/policy.json +97 -97
  85. package/packs/hud/endpoints.json +146 -146
  86. package/packs/hud/policy.json +84 -84
  87. package/packs/ipums/docs/report_patterns.md +117 -117
  88. package/packs/irs-soi/endpoints.json +114 -114
  89. package/packs/irs-soi/policy.json +107 -107
  90. package/packs/medicaid/endpoints.json +412 -412
  91. package/packs/medicaid/policy.json +108 -108
  92. package/packs/naep/endpoints.json +217 -217
  93. package/packs/naep/policy.json +100 -100
  94. package/packs/nhtsa/endpoints.json +132 -132
  95. package/packs/nhtsa/policy.json +93 -93
  96. package/packs/nih/endpoints.json +452 -452
  97. package/packs/nih/policy.json +86 -86
  98. package/packs/noaa/endpoints.json +431 -431
  99. package/packs/noaa/policy.json +95 -95
  100. package/packs/nrel/endpoints.json +327 -327
  101. package/packs/nrel/policy.json +96 -96
  102. package/packs/nws/endpoints.json +271 -271
  103. package/packs/nws/policy.json +82 -82
  104. package/packs/nyfed/endpoints.json +80 -80
  105. package/packs/nyfed/policy.json +92 -92
  106. package/packs/omb/endpoints.json +460 -460
  107. package/packs/omb/policy.json +106 -106
  108. package/packs/open-payments/endpoints.json +297 -297
  109. package/packs/open-payments/policy.json +101 -101
  110. package/packs/opm/assets/download_opm.ps1 +125 -125
  111. package/packs/opm/docs/fields.md +184 -184
  112. package/packs/opm/docs/opm_api.md +71 -71
  113. package/packs/opm/docs/report_patterns.md +83 -83
  114. package/packs/opm/lib/data_dictionary.csv +73 -73
  115. package/packs/pbgc/endpoints.json +483 -483
  116. package/packs/pbgc/policy.json +94 -94
  117. package/packs/philfed/endpoints.json +85 -85
  118. package/packs/philfed/policy.json +94 -94
  119. package/packs/regulations/endpoints.json +287 -287
  120. package/packs/regulations/policy.json +86 -86
  121. package/packs/sec/endpoints.json +236 -236
  122. package/packs/sec/policy.json +86 -86
  123. package/packs/senate-lobbying/endpoints.json +553 -553
  124. package/packs/senate-lobbying/policy.json +99 -99
  125. package/packs/snapqc/endpoints.json +1426 -1426
  126. package/packs/snapqc/policy.json +92 -92
  127. package/packs/treasury/catalog.json +2267 -2267
  128. package/packs/treasury/docs/api_guide.md +245 -245
  129. package/packs/treasury/docs/datasets.md +56 -56
  130. package/packs/treasury/docs/report_patterns.md +126 -126
  131. package/packs/treasury/policy.json +159 -159
  132. package/packs/treasury/source.md +170 -165
  133. package/packs/treasury-yields/endpoints.json +226 -226
  134. package/packs/treasury-yields/policy.json +86 -86
  135. package/packs/usaspending/endpoints.json +1315 -1315
  136. package/packs/usaspending/policy.json +109 -109
  137. package/packs/usda-fna/policy.json +133 -133
  138. package/packs/usda-fooddata/endpoints.json +180 -180
  139. package/packs/usda-fooddata/policy.json +94 -94
  140. package/packs/usda-nass/endpoints.json +323 -323
  141. package/packs/usda-nass/policy.json +95 -95
  142. package/packs/usda-outlook/endpoints.json +86 -86
  143. package/packs/usda-outlook/policy.json +109 -109
  144. package/packs/usgs/endpoints.json +313 -313
  145. package/packs/usgs/policy.json +95 -95
  146. package/packs/usitc/endpoints.json +74 -74
  147. package/packs/usitc/policy.json +111 -111
  148. package/packs/uspto/endpoints.json +141 -141
  149. package/packs/uspto/policy.json +95 -95
  150. package/packs/va/endpoints.json +414 -414
  151. package/packs/va/policy.json +101 -101
  152. package/packs/world-bank/endpoints.json +122 -122
  153. package/packs/world-bank/policy.json +101 -101
@@ -1,109 +1,109 @@
1
- {
2
- "id": "fhfa",
3
- "title": "FHFA House Price Index, conforming loan limits and Public Use Databases (fhfa.gov files)",
4
- "agency": "Federal Housing Finance Agency",
5
- "depth": "standard",
6
- "domains": [
7
- "housing",
8
- "house prices",
9
- "mortgage finance"
10
- ],
11
- "keywords": [
12
- "fhfa",
13
- "house price index",
14
- "hpi",
15
- "home prices",
16
- "house prices",
17
- "home values",
18
- "purchase-only index",
19
- "all-transactions index",
20
- "conforming loan limit",
21
- "loan limit",
22
- "jumbo",
23
- "high-cost area",
24
- "fannie mae",
25
- "freddie mac",
26
- "enterprises",
27
- "gse",
28
- "federal home loan bank",
29
- "fhlbank",
30
- "public use database",
31
- "pudb",
32
- "mortgage"
33
- ],
34
- "questions": [
35
- "latest FHFA house price index for the U.S., seasonally adjusted",
36
- "house price growth by state over the last year",
37
- "home price appreciation in a metro area since 2020",
38
- "annual house price index by county",
39
- "2026 conforming loan limit for a one-unit home in a county",
40
- "which counties have high-cost loan limits"
41
- ],
42
- "homepage": "https://www.fhfa.gov/data",
43
- "terms_url": "https://www.fhfa.gov/about/fhfa-policies",
44
- "auth": {
45
- "env": null,
46
- "required": "none",
47
- "placement": "none",
48
- "param": null,
49
- "prefix": "",
50
- "home": null,
51
- "signup_url": null,
52
- "echoes_key": false,
53
- "key_only": [],
54
- "extra": {},
55
- "demo_key": null
56
- },
57
- "limits": {
58
- "rps": 1,
59
- "burst": 2,
60
- "daily_quota": null,
61
- "errors_per_min": null,
62
- "bytes_per_min": null,
63
- "max_rows": 400000
64
- },
65
- "routes": {
66
- "api": {
67
- "base": "https://www.fhfa.gov",
68
- "probe": "https://www.fhfa.gov/hpi/download/quarterly_datasets/hpi_at_state.csv",
69
- "hosts": [
70
- "www.fhfa.gov"
71
- ]
72
- },
73
- "manual": {
74
- "how": "Open each link in manual_download/INSTRUCTIONS.md in a browser; the file downloads (hpi_master.csv is 17 MB). Upload it under the name shown; the pack applies the request's filters to the uploaded file. Without the links: HPI files are listed on https://www.fhfa.gov/data/hpi/datasets, conforming loan limits on https://www.fhfa.gov/data/conforming-loan-limit, Public Use Databases on https://www.fhfa.gov/data/pudb."
75
- }
76
- },
77
- "cache": {
78
- "mode": "run_only",
79
- "ttl_hours": null,
80
- "why": "Files are overwritten in place at each release (monthly HPI, quarterly, annual); data/ is reused only within a run."
81
- },
82
- "redistribution": {
83
- "mode": "allowed",
84
- "why": "FHFA data are works of a U.S. government agency, in the public domain (17 U.S.C. 105); cite FHFA as the source."
85
- },
86
- "notices": [],
87
- "retention": null,
88
- "citation": "Federal Housing Finance Agency, {dataset}, www.fhfa.gov, retrieved {retrieved_long}.",
89
- "audit_hooks": [],
90
- "clarify": [
91
- "Which index: purchase-only (the flagship, sales only) or all-transactions (adds refinance appraisals, longer history)? Default: purchase-only.",
92
- "Seasonally adjusted or not? Default: seasonally adjusted for month-to-month or quarter-to-quarter changes, not adjusted for year-over-year changes.",
93
- "Which loan limit: the baseline (most counties) or a specific county's limit (high-cost areas are higher), and for how many units? Default: one-unit."
94
- ],
95
- "formats": [
96
- "csv",
97
- "xlsx"
98
- ],
99
- "use_instead": [
100
- {
101
- "question": "One FHFA house price series for the U.S. or a state, with no index flavor choice",
102
- "source": "fred",
103
- "terms": [
104
- "USSTHPI"
105
- ],
106
- "why": "FRED carries the all-transactions and purchase-only HPI series under stable IDs."
107
- }
108
- ]
109
- }
1
+ {
2
+ "id": "fhfa",
3
+ "title": "FHFA House Price Index, conforming loan limits and Public Use Databases (fhfa.gov files)",
4
+ "agency": "Federal Housing Finance Agency",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "housing",
8
+ "house prices",
9
+ "mortgage finance"
10
+ ],
11
+ "keywords": [
12
+ "fhfa",
13
+ "house price index",
14
+ "hpi",
15
+ "home prices",
16
+ "house prices",
17
+ "home values",
18
+ "purchase-only index",
19
+ "all-transactions index",
20
+ "conforming loan limit",
21
+ "loan limit",
22
+ "jumbo",
23
+ "high-cost area",
24
+ "fannie mae",
25
+ "freddie mac",
26
+ "enterprises",
27
+ "gse",
28
+ "federal home loan bank",
29
+ "fhlbank",
30
+ "public use database",
31
+ "pudb",
32
+ "mortgage"
33
+ ],
34
+ "questions": [
35
+ "latest FHFA house price index for the U.S., seasonally adjusted",
36
+ "house price growth by state over the last year",
37
+ "home price appreciation in a metro area since 2020",
38
+ "annual house price index by county",
39
+ "2026 conforming loan limit for a one-unit home in a county",
40
+ "which counties have high-cost loan limits"
41
+ ],
42
+ "homepage": "https://www.fhfa.gov/data",
43
+ "terms_url": "https://www.fhfa.gov/about/fhfa-policies",
44
+ "auth": {
45
+ "env": null,
46
+ "required": "none",
47
+ "placement": "none",
48
+ "param": null,
49
+ "prefix": "",
50
+ "home": null,
51
+ "signup_url": null,
52
+ "echoes_key": false,
53
+ "key_only": [],
54
+ "extra": {},
55
+ "demo_key": null
56
+ },
57
+ "limits": {
58
+ "rps": 1,
59
+ "burst": 2,
60
+ "daily_quota": null,
61
+ "errors_per_min": null,
62
+ "bytes_per_min": null,
63
+ "max_rows": 400000
64
+ },
65
+ "routes": {
66
+ "api": {
67
+ "base": "https://www.fhfa.gov",
68
+ "probe": "https://www.fhfa.gov/hpi/download/quarterly_datasets/hpi_at_state.csv",
69
+ "hosts": [
70
+ "www.fhfa.gov"
71
+ ]
72
+ },
73
+ "manual": {
74
+ "how": "Open each link in manual_download/INSTRUCTIONS.md in a browser; the file downloads (hpi_master.csv is 17 MB). Upload it under the name shown; the pack applies the request's filters to the uploaded file. Without the links: HPI files are listed on https://www.fhfa.gov/data/hpi/datasets, conforming loan limits on https://www.fhfa.gov/data/conforming-loan-limit, Public Use Databases on https://www.fhfa.gov/data/pudb."
75
+ }
76
+ },
77
+ "cache": {
78
+ "mode": "run_only",
79
+ "ttl_hours": null,
80
+ "why": "Files are overwritten in place at each release (monthly HPI, quarterly, annual); data/ is reused only within a run."
81
+ },
82
+ "redistribution": {
83
+ "mode": "allowed",
84
+ "why": "FHFA data are works of a U.S. government agency, in the public domain (17 U.S.C. 105); cite FHFA as the source."
85
+ },
86
+ "notices": [],
87
+ "retention": null,
88
+ "citation": "Federal Housing Finance Agency, {dataset}, www.fhfa.gov, retrieved {retrieved_long}.",
89
+ "audit_hooks": [],
90
+ "clarify": [
91
+ "Which index: purchase-only (the flagship, sales only) or all-transactions (adds refinance appraisals, longer history)? Default: purchase-only.",
92
+ "Seasonally adjusted or not? Default: seasonally adjusted for month-to-month or quarter-to-quarter changes, not adjusted for year-over-year changes.",
93
+ "Which loan limit: the baseline (most counties) or a specific county's limit (high-cost areas are higher), and for how many units? Default: one-unit."
94
+ ],
95
+ "formats": [
96
+ "csv",
97
+ "xlsx"
98
+ ],
99
+ "use_instead": [
100
+ {
101
+ "question": "One FHFA house price series for the U.S. or a state, with no index flavor choice",
102
+ "source": "fred",
103
+ "terms": [
104
+ "USSTHPI"
105
+ ],
106
+ "why": "FRED carries the all-transactions and purchase-only HPI series under stable IDs."
107
+ }
108
+ ]
109
+ }
@@ -1,122 +1,122 @@
1
- <#
2
- .SYNOPSIS
3
- Download the FRED files for one data-analyst request into the folder that holds this script.
4
-
5
- .DESCRIPTION
6
- Files marked key, bearer or group come from the FRED API and need your own free FRED API key
7
- (https://fredaccount.stlouisfed.org/apikeys). The script uses $env:FRED_API_KEY when it is set,
8
- otherwise asks for it with hidden input; press Enter to skip those files. The key is never printed
9
- and never written into the saved files.
10
-
11
- .EXAMPLE
12
- powershell -NoProfile -ExecutionPolicy Bypass -File "$HOME\Downloads\download_fred.ps1"
13
- #>
14
- [CmdletBinding()]
15
- param([switch]$NoPause)
16
- $ErrorActionPreference = 'Stop'
17
- [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
18
- $here = Split-Path -Parent $MyInvocation.MyCommand.Path
19
-
20
- # file <TAB> kind <TAB> pages <TAB> url <TAB> then_file <TAB> then_url ("-" = empty, pages 0 = unpaged)
21
- # kind: plain (no key), key (key in the query), bearer (FRED API v2, key in a header, paged),
22
- # group (GeoFRED series group; its fields fill the __GROUP_*__ placeholders of then_url)
23
- $manifest = @'
24
- @MANIFEST@
25
- '@
26
- $items = $manifest -split "`n" | ForEach-Object { $_.TrimEnd("`r") } | Where-Object { $_ } | ForEach-Object {
27
- $f = $_ -split "`t"
28
- [pscustomobject]@{ File = $f[0]; Kind = $f[1]; Pages = [int]$f[2]; Url = $f[3]; ThenFile = $f[4]; ThenUrl = $f[5] }
29
- }
30
-
31
- $key = $env:FRED_API_KEY
32
- if (-not $key -and ($items | Where-Object { $_.Kind -ne 'plain' })) {
33
- Write-Host 'Some files need a FRED API key (https://fredaccount.stlouisfed.org/apikeys). Press Enter to skip them.'
34
- if ([Console]::IsInputRedirected) { $key = [Console]::In.ReadLine() }
35
- else {
36
- $secure = Read-Host -Prompt 'FRED API key (hidden)' -AsSecureString
37
- $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
38
- try { $key = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
39
- finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
40
- }
41
- }
42
- if ($null -eq $key) { $key = '' }
43
- $key = $key -replace '\s', ''
44
-
45
- $utf8 = New-Object System.Text.UTF8Encoding($false)
46
- $saved = New-Object System.Collections.ArrayList
47
-
48
- # Download one keyed file, wrap it with the request URL (placeholder, not the key) and save it.
49
- # Returns the reply text, or $null when the request failed.
50
- function Save-Keyed([string]$file, [string]$url, [bool]$bearer) {
51
- try {
52
- if ($bearer) {
53
- $r = Invoke-WebRequest -UseBasicParsing -UserAgent 'data-analyst/1.0' -Uri $url `
54
- -Headers @{ Authorization = "Bearer $key" }
55
- } else {
56
- $r = Invoke-WebRequest -UseBasicParsing -UserAgent 'data-analyst/1.0' -Uri ($url.Replace('__FRED_KEY__', $key))
57
- }
58
- } catch {
59
- $msg = "$($_.Exception.Message) $($_.ErrorDetails.Message)"
60
- if ($key) { $msg = $msg.Replace($key, '[KEY]') }
61
- Write-Host "${file}: download failed ($msg)"
62
- return $null
63
- }
64
- $text = $utf8.GetString($r.RawContentStream.ToArray())
65
- $wrapped = '{"request": "' + $url + '", "response": ' + $text + '}'
66
- $dest = Join-Path $here $file
67
- [IO.File]::WriteAllText($dest, $wrapped, $utf8)
68
- [void]$saved.Add($dest)
69
- Write-Host "saved $file"
70
- return $text
71
- }
72
-
73
- foreach ($it in $items) {
74
- if ($it.Kind -eq 'plain') {
75
- $dest = Join-Path $here $it.File
76
- try {
77
- Invoke-WebRequest -UseBasicParsing -UserAgent 'data-analyst/1.0' -Uri $it.Url -OutFile $dest
78
- [void]$saved.Add($dest)
79
- Write-Host "saved $($it.File)"
80
- } catch { Write-Host "$($it.File): download failed ($($_.Exception.Message))" }
81
- Start-Sleep -Milliseconds 600
82
- continue
83
- }
84
- if (-not $key) { Write-Host "skipped (needs a key): $($it.File)"; continue }
85
- switch ($it.Kind) {
86
- 'key' { [void](Save-Keyed $it.File $it.Url $false) }
87
- 'bearer' {
88
- # FRED API v2 release/observations: follow next_cursor up to Pages pages
89
- $n = 1; $u = $it.Url; $f = $it.File
90
- while ($true) {
91
- $body = Save-Keyed $f $u $true
92
- if ($null -eq $body -or $body -notmatch '"has_more"\s*:\s*true') { break }
93
- if ($body -notmatch '"next_cursor"\s*:\s*"([^"]*)"' -or $n -ge $it.Pages) { break }
94
- $n++
95
- $u = $it.Url + '&next_cursor=' + [Uri]::EscapeDataString($Matches[1])
96
- $f = $it.File -replace '\.json$', ('__p{0:D2}.json' -f $n)
97
- Start-Sleep -Milliseconds 600
98
- }
99
- }
100
- 'group' {
101
- # GeoFRED: the series group first, then the cross-section it names
102
- $body = Save-Keyed $it.File $it.Url $false
103
- if ($null -ne $body) {
104
- $g = (ConvertFrom-Json $body).series_group
105
- $u = $it.ThenUrl
106
- foreach ($k in 'series_group', 'region_type', 'season', 'units', 'frequency', 'max_date') {
107
- $v = "$($g.$k)"
108
- if ($k -eq 'frequency') { $v = $v.Substring(0, [Math]::Min(1, $v.Length)).ToLower() }
109
- $u = $u.Replace('__GROUP_' + $k.ToUpper() + '__', [Uri]::EscapeDataString($v))
110
- }
111
- Start-Sleep -Milliseconds 600
112
- [void](Save-Keyed $it.ThenFile $u $false)
113
- }
114
- }
115
- }
116
- Start-Sleep -Milliseconds 600
117
- }
118
- $key = $null
119
- Write-Host ''
120
- Write-Host 'Done. Upload these files to the chat:'
121
- $saved | ForEach-Object { Write-Host " $_" }
122
- if (-not $NoPause -and -not [Console]::IsInputRedirected) { Read-Host 'Press Enter to close' | Out-Null }
1
+ <#
2
+ .SYNOPSIS
3
+ Download the FRED files for one data-analyst request into the folder that holds this script.
4
+
5
+ .DESCRIPTION
6
+ Files marked key, bearer or group come from the FRED API and need your own free FRED API key
7
+ (https://fredaccount.stlouisfed.org/apikeys). The script uses $env:FRED_API_KEY when it is set,
8
+ otherwise asks for it with hidden input; press Enter to skip those files. The key is never printed
9
+ and never written into the saved files.
10
+
11
+ .EXAMPLE
12
+ powershell -NoProfile -ExecutionPolicy Bypass -File "$HOME\Downloads\download_fred.ps1"
13
+ #>
14
+ [CmdletBinding()]
15
+ param([switch]$NoPause)
16
+ $ErrorActionPreference = 'Stop'
17
+ [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
18
+ $here = Split-Path -Parent $MyInvocation.MyCommand.Path
19
+
20
+ # file <TAB> kind <TAB> pages <TAB> url <TAB> then_file <TAB> then_url ("-" = empty, pages 0 = unpaged)
21
+ # kind: plain (no key), key (key in the query), bearer (FRED API v2, key in a header, paged),
22
+ # group (GeoFRED series group; its fields fill the __GROUP_*__ placeholders of then_url)
23
+ $manifest = @'
24
+ @MANIFEST@
25
+ '@
26
+ $items = $manifest -split "`n" | ForEach-Object { $_.TrimEnd("`r") } | Where-Object { $_ } | ForEach-Object {
27
+ $f = $_ -split "`t"
28
+ [pscustomobject]@{ File = $f[0]; Kind = $f[1]; Pages = [int]$f[2]; Url = $f[3]; ThenFile = $f[4]; ThenUrl = $f[5] }
29
+ }
30
+
31
+ $key = $env:FRED_API_KEY
32
+ if (-not $key -and ($items | Where-Object { $_.Kind -ne 'plain' })) {
33
+ Write-Host 'Some files need a FRED API key (https://fredaccount.stlouisfed.org/apikeys). Press Enter to skip them.'
34
+ if ([Console]::IsInputRedirected) { $key = [Console]::In.ReadLine() }
35
+ else {
36
+ $secure = Read-Host -Prompt 'FRED API key (hidden)' -AsSecureString
37
+ $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
38
+ try { $key = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
39
+ finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
40
+ }
41
+ }
42
+ if ($null -eq $key) { $key = '' }
43
+ $key = $key -replace '\s', ''
44
+
45
+ $utf8 = New-Object System.Text.UTF8Encoding($false)
46
+ $saved = New-Object System.Collections.ArrayList
47
+
48
+ # Download one keyed file, wrap it with the request URL (placeholder, not the key) and save it.
49
+ # Returns the reply text, or $null when the request failed.
50
+ function Save-Keyed([string]$file, [string]$url, [bool]$bearer) {
51
+ try {
52
+ if ($bearer) {
53
+ $r = Invoke-WebRequest -UseBasicParsing -UserAgent 'data-analyst/1.0' -Uri $url `
54
+ -Headers @{ Authorization = "Bearer $key" }
55
+ } else {
56
+ $r = Invoke-WebRequest -UseBasicParsing -UserAgent 'data-analyst/1.0' -Uri ($url.Replace('__FRED_KEY__', $key))
57
+ }
58
+ } catch {
59
+ $msg = "$($_.Exception.Message) $($_.ErrorDetails.Message)"
60
+ if ($key) { $msg = $msg.Replace($key, '[KEY]') }
61
+ Write-Host "${file}: download failed ($msg)"
62
+ return $null
63
+ }
64
+ $text = $utf8.GetString($r.RawContentStream.ToArray())
65
+ $wrapped = '{"request": "' + $url + '", "response": ' + $text + '}'
66
+ $dest = Join-Path $here $file
67
+ [IO.File]::WriteAllText($dest, $wrapped, $utf8)
68
+ [void]$saved.Add($dest)
69
+ Write-Host "saved $file"
70
+ return $text
71
+ }
72
+
73
+ foreach ($it in $items) {
74
+ if ($it.Kind -eq 'plain') {
75
+ $dest = Join-Path $here $it.File
76
+ try {
77
+ Invoke-WebRequest -UseBasicParsing -UserAgent 'data-analyst/1.0' -Uri $it.Url -OutFile $dest
78
+ [void]$saved.Add($dest)
79
+ Write-Host "saved $($it.File)"
80
+ } catch { Write-Host "$($it.File): download failed ($($_.Exception.Message))" }
81
+ Start-Sleep -Milliseconds 600
82
+ continue
83
+ }
84
+ if (-not $key) { Write-Host "skipped (needs a key): $($it.File)"; continue }
85
+ switch ($it.Kind) {
86
+ 'key' { [void](Save-Keyed $it.File $it.Url $false) }
87
+ 'bearer' {
88
+ # FRED API v2 release/observations: follow next_cursor up to Pages pages
89
+ $n = 1; $u = $it.Url; $f = $it.File
90
+ while ($true) {
91
+ $body = Save-Keyed $f $u $true
92
+ if ($null -eq $body -or $body -notmatch '"has_more"\s*:\s*true') { break }
93
+ if ($body -notmatch '"next_cursor"\s*:\s*"([^"]*)"' -or $n -ge $it.Pages) { break }
94
+ $n++
95
+ $u = $it.Url + '&next_cursor=' + [Uri]::EscapeDataString($Matches[1])
96
+ $f = $it.File -replace '\.json$', ('__p{0:D2}.json' -f $n)
97
+ Start-Sleep -Milliseconds 600
98
+ }
99
+ }
100
+ 'group' {
101
+ # GeoFRED: the series group first, then the cross-section it names
102
+ $body = Save-Keyed $it.File $it.Url $false
103
+ if ($null -ne $body) {
104
+ $g = (ConvertFrom-Json $body).series_group
105
+ $u = $it.ThenUrl
106
+ foreach ($k in 'series_group', 'region_type', 'season', 'units', 'frequency', 'max_date') {
107
+ $v = "$($g.$k)"
108
+ if ($k -eq 'frequency') { $v = $v.Substring(0, [Math]::Min(1, $v.Length)).ToLower() }
109
+ $u = $u.Replace('__GROUP_' + $k.ToUpper() + '__', [Uri]::EscapeDataString($v))
110
+ }
111
+ Start-Sleep -Milliseconds 600
112
+ [void](Save-Keyed $it.ThenFile $u $false)
113
+ }
114
+ }
115
+ }
116
+ Start-Sleep -Milliseconds 600
117
+ }
118
+ $key = $null
119
+ Write-Host ''
120
+ Write-Host 'Done. Upload these files to the chat:'
121
+ $saved | ForEach-Object { Write-Host " $_" }
122
+ if (-not $NoPause -and -not [Console]::IsInputRedirected) { Read-Host 'Press Enter to close' | Out-Null }