data-analyst-mcp 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +262 -262
- package/dist/report/method_audit.js +7 -1
- package/dist/report/method_audit.js.map +1 -1
- package/dist/report/scan_secrets.d.ts +4 -1
- package/dist/report/scan_secrets.js +36 -5
- package/dist/report/scan_secrets.js.map +1 -1
- package/dist/server.d.ts +1 -1
- package/dist/server.js +3 -2
- package/dist/server.js.map +1 -1
- package/guide/WORKFLOW.md +134 -134
- package/guide/assets/disclaimer.txt +1 -1
- package/guide/assets/download_data.ps1 +162 -162
- package/guide/assets/report_template.html +597 -597
- package/guide/references/routing.md +171 -171
- package/guide/references/style_guide.md +398 -398
- package/package.json +56 -56
- package/packs/acf/endpoints.json +114 -114
- package/packs/acf/policy.json +104 -104
- package/packs/actuaries/endpoints.json +82 -82
- package/packs/actuaries/policy.json +109 -109
- package/packs/bea/assets/download_bea.ps1 +182 -182
- package/packs/bea/docs/bea_api.md +461 -461
- package/packs/bea/docs/datasets.md +157 -157
- package/packs/bea/docs/report_patterns.md +112 -112
- package/packs/bea/policy.json +170 -170
- package/packs/bea/source.md +152 -152
- package/packs/bls/assets/download_bls.ps1 +152 -152
- package/packs/bls/docs/analysis_patterns.md +108 -108
- package/packs/bls/docs/bls_api.md +73 -73
- package/packs/bls/docs/series_ids.md +149 -149
- package/packs/bls/policy.json +238 -238
- package/packs/bts/endpoints.json +346 -346
- package/packs/bts/policy.json +94 -94
- package/packs/cdc/endpoints.json +197 -197
- package/packs/cdc/policy.json +83 -83
- package/packs/census/endpoints.json +99 -99
- package/packs/cfpb/endpoints.json +326 -326
- package/packs/cfpb/policy.json +119 -119
- package/packs/clinical-trials/endpoints.json +166 -166
- package/packs/clinical-trials/policy.json +84 -84
- package/packs/cms/endpoints.json +69 -69
- package/packs/cms/policy.json +96 -96
- package/packs/college-scorecard/endpoints.json +252 -252
- package/packs/college-scorecard/policy.json +106 -106
- package/packs/congress/endpoints.json +1701 -1701
- package/packs/congress/policy.json +117 -117
- package/packs/dhs-ohss/endpoints.json +66 -66
- package/packs/dhs-ohss/policy.json +107 -107
- package/packs/doj-news/endpoints.json +59 -59
- package/packs/doj-news/policy.json +96 -96
- package/packs/dol/endpoints.json +817 -817
- package/packs/dol/policy.json +135 -135
- package/packs/eia/endpoints.json +717 -717
- package/packs/eia/policy.json +112 -112
- package/packs/epa/endpoints.json +291 -291
- package/packs/epa/policy.json +99 -99
- package/packs/epa-aqs/endpoints.json +173 -173
- package/packs/epa-aqs/policy.json +96 -96
- package/packs/fbi/endpoints.json +1841 -1841
- package/packs/fbi/policy.json +118 -118
- package/packs/fda/endpoints.json +237 -237
- package/packs/fda/policy.json +121 -121
- package/packs/fdic/endpoints.json +416 -416
- package/packs/fdic/policy.json +83 -83
- package/packs/fec/endpoints.json +707 -707
- package/packs/fec/policy.json +93 -93
- package/packs/fed-board/endpoints.json +89 -89
- package/packs/fed-board/policy.json +136 -136
- package/packs/federal-register/endpoints.json +143 -143
- package/packs/federal-register/policy.json +86 -86
- package/packs/fema/endpoints.json +645 -645
- package/packs/fema/policy.json +103 -103
- package/packs/fhfa/endpoints.json +105 -105
- package/packs/fhfa/policy.json +109 -109
- package/packs/fred/assets/download_fred.ps1 +122 -122
- package/packs/fred/docs/analysis_patterns.md +124 -124
- package/packs/fred/docs/fred_api.md +94 -94
- package/packs/fred/docs/series.md +188 -188
- package/packs/fred/policy.json +253 -253
- package/packs/fsa/endpoints.json +150 -150
- package/packs/fsa/policy.json +99 -99
- package/packs/govinfo/endpoints.json +519 -519
- package/packs/gsa-calc/endpoints.json +93 -93
- package/packs/gsa-calc/policy.json +97 -97
- package/packs/hud/endpoints.json +146 -146
- package/packs/hud/policy.json +84 -84
- package/packs/ipums/docs/report_patterns.md +117 -117
- package/packs/irs-soi/endpoints.json +114 -114
- package/packs/irs-soi/policy.json +107 -107
- package/packs/medicaid/endpoints.json +412 -412
- package/packs/medicaid/policy.json +108 -108
- package/packs/naep/endpoints.json +217 -217
- package/packs/naep/policy.json +100 -100
- package/packs/nhtsa/endpoints.json +132 -132
- package/packs/nhtsa/policy.json +93 -93
- package/packs/nih/endpoints.json +452 -452
- package/packs/nih/policy.json +86 -86
- package/packs/noaa/endpoints.json +431 -431
- package/packs/noaa/policy.json +95 -95
- package/packs/nrel/endpoints.json +327 -327
- package/packs/nrel/policy.json +96 -96
- package/packs/nws/endpoints.json +271 -271
- package/packs/nws/policy.json +82 -82
- package/packs/nyfed/endpoints.json +80 -80
- package/packs/nyfed/policy.json +92 -92
- package/packs/omb/endpoints.json +460 -460
- package/packs/omb/policy.json +106 -106
- package/packs/open-payments/endpoints.json +297 -297
- package/packs/open-payments/policy.json +101 -101
- package/packs/opm/assets/download_opm.ps1 +125 -125
- package/packs/opm/docs/fields.md +184 -184
- package/packs/opm/docs/opm_api.md +71 -71
- package/packs/opm/docs/report_patterns.md +83 -83
- package/packs/opm/lib/data_dictionary.csv +73 -73
- package/packs/pbgc/endpoints.json +483 -483
- package/packs/pbgc/policy.json +94 -94
- package/packs/philfed/endpoints.json +85 -85
- package/packs/philfed/policy.json +94 -94
- package/packs/regulations/endpoints.json +287 -287
- package/packs/regulations/policy.json +86 -86
- package/packs/sec/endpoints.json +236 -236
- package/packs/sec/policy.json +86 -86
- package/packs/senate-lobbying/endpoints.json +553 -553
- package/packs/senate-lobbying/policy.json +99 -99
- package/packs/snapqc/endpoints.json +1426 -1426
- package/packs/snapqc/policy.json +92 -92
- package/packs/treasury/catalog.json +2267 -2267
- package/packs/treasury/docs/api_guide.md +245 -245
- package/packs/treasury/docs/datasets.md +56 -56
- package/packs/treasury/docs/report_patterns.md +126 -126
- package/packs/treasury/policy.json +159 -159
- package/packs/treasury/source.md +170 -165
- package/packs/treasury-yields/endpoints.json +226 -226
- package/packs/treasury-yields/policy.json +86 -86
- package/packs/usaspending/endpoints.json +1315 -1315
- package/packs/usaspending/policy.json +109 -109
- package/packs/usda-fna/policy.json +133 -133
- package/packs/usda-fooddata/endpoints.json +180 -180
- package/packs/usda-fooddata/policy.json +94 -94
- package/packs/usda-nass/endpoints.json +323 -323
- package/packs/usda-nass/policy.json +95 -95
- package/packs/usda-outlook/endpoints.json +86 -86
- package/packs/usda-outlook/policy.json +109 -109
- package/packs/usgs/endpoints.json +313 -313
- package/packs/usgs/policy.json +95 -95
- package/packs/usitc/endpoints.json +74 -74
- package/packs/usitc/policy.json +111 -111
- package/packs/uspto/endpoints.json +141 -141
- package/packs/uspto/policy.json +95 -95
- package/packs/va/endpoints.json +414 -414
- package/packs/va/policy.json +101 -101
- package/packs/world-bank/endpoints.json +122 -122
- package/packs/world-bank/policy.json +101 -101
package/packs/fhfa/policy.json
CHANGED
|
@@ -1,109 +1,109 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "fhfa",
|
|
3
|
-
"title": "FHFA House Price Index, conforming loan limits and Public Use Databases (fhfa.gov files)",
|
|
4
|
-
"agency": "Federal Housing Finance Agency",
|
|
5
|
-
"depth": "standard",
|
|
6
|
-
"domains": [
|
|
7
|
-
"housing",
|
|
8
|
-
"house prices",
|
|
9
|
-
"mortgage finance"
|
|
10
|
-
],
|
|
11
|
-
"keywords": [
|
|
12
|
-
"fhfa",
|
|
13
|
-
"house price index",
|
|
14
|
-
"hpi",
|
|
15
|
-
"home prices",
|
|
16
|
-
"house prices",
|
|
17
|
-
"home values",
|
|
18
|
-
"purchase-only index",
|
|
19
|
-
"all-transactions index",
|
|
20
|
-
"conforming loan limit",
|
|
21
|
-
"loan limit",
|
|
22
|
-
"jumbo",
|
|
23
|
-
"high-cost area",
|
|
24
|
-
"fannie mae",
|
|
25
|
-
"freddie mac",
|
|
26
|
-
"enterprises",
|
|
27
|
-
"gse",
|
|
28
|
-
"federal home loan bank",
|
|
29
|
-
"fhlbank",
|
|
30
|
-
"public use database",
|
|
31
|
-
"pudb",
|
|
32
|
-
"mortgage"
|
|
33
|
-
],
|
|
34
|
-
"questions": [
|
|
35
|
-
"latest FHFA house price index for the U.S., seasonally adjusted",
|
|
36
|
-
"house price growth by state over the last year",
|
|
37
|
-
"home price appreciation in a metro area since 2020",
|
|
38
|
-
"annual house price index by county",
|
|
39
|
-
"2026 conforming loan limit for a one-unit home in a county",
|
|
40
|
-
"which counties have high-cost loan limits"
|
|
41
|
-
],
|
|
42
|
-
"homepage": "https://www.fhfa.gov/data",
|
|
43
|
-
"terms_url": "https://www.fhfa.gov/about/fhfa-policies",
|
|
44
|
-
"auth": {
|
|
45
|
-
"env": null,
|
|
46
|
-
"required": "none",
|
|
47
|
-
"placement": "none",
|
|
48
|
-
"param": null,
|
|
49
|
-
"prefix": "",
|
|
50
|
-
"home": null,
|
|
51
|
-
"signup_url": null,
|
|
52
|
-
"echoes_key": false,
|
|
53
|
-
"key_only": [],
|
|
54
|
-
"extra": {},
|
|
55
|
-
"demo_key": null
|
|
56
|
-
},
|
|
57
|
-
"limits": {
|
|
58
|
-
"rps": 1,
|
|
59
|
-
"burst": 2,
|
|
60
|
-
"daily_quota": null,
|
|
61
|
-
"errors_per_min": null,
|
|
62
|
-
"bytes_per_min": null,
|
|
63
|
-
"max_rows": 400000
|
|
64
|
-
},
|
|
65
|
-
"routes": {
|
|
66
|
-
"api": {
|
|
67
|
-
"base": "https://www.fhfa.gov",
|
|
68
|
-
"probe": "https://www.fhfa.gov/hpi/download/quarterly_datasets/hpi_at_state.csv",
|
|
69
|
-
"hosts": [
|
|
70
|
-
"www.fhfa.gov"
|
|
71
|
-
]
|
|
72
|
-
},
|
|
73
|
-
"manual": {
|
|
74
|
-
"how": "Open each link in manual_download/INSTRUCTIONS.md in a browser; the file downloads (hpi_master.csv is 17 MB). Upload it under the name shown; the pack applies the request's filters to the uploaded file. Without the links: HPI files are listed on https://www.fhfa.gov/data/hpi/datasets, conforming loan limits on https://www.fhfa.gov/data/conforming-loan-limit, Public Use Databases on https://www.fhfa.gov/data/pudb."
|
|
75
|
-
}
|
|
76
|
-
},
|
|
77
|
-
"cache": {
|
|
78
|
-
"mode": "run_only",
|
|
79
|
-
"ttl_hours": null,
|
|
80
|
-
"why": "Files are overwritten in place at each release (monthly HPI, quarterly, annual); data/ is reused only within a run."
|
|
81
|
-
},
|
|
82
|
-
"redistribution": {
|
|
83
|
-
"mode": "allowed",
|
|
84
|
-
"why": "FHFA data are works of a U.S. government agency, in the public domain (17 U.S.C. 105); cite FHFA as the source."
|
|
85
|
-
},
|
|
86
|
-
"notices": [],
|
|
87
|
-
"retention": null,
|
|
88
|
-
"citation": "Federal Housing Finance Agency, {dataset}, www.fhfa.gov, retrieved {retrieved_long}.",
|
|
89
|
-
"audit_hooks": [],
|
|
90
|
-
"clarify": [
|
|
91
|
-
"Which index: purchase-only (the flagship, sales only) or all-transactions (adds refinance appraisals, longer history)? Default: purchase-only.",
|
|
92
|
-
"Seasonally adjusted or not? Default: seasonally adjusted for month-to-month or quarter-to-quarter changes, not adjusted for year-over-year changes.",
|
|
93
|
-
"Which loan limit: the baseline (most counties) or a specific county's limit (high-cost areas are higher), and for how many units? Default: one-unit."
|
|
94
|
-
],
|
|
95
|
-
"formats": [
|
|
96
|
-
"csv",
|
|
97
|
-
"xlsx"
|
|
98
|
-
],
|
|
99
|
-
"use_instead": [
|
|
100
|
-
{
|
|
101
|
-
"question": "One FHFA house price series for the U.S. or a state, with no index flavor choice",
|
|
102
|
-
"source": "fred",
|
|
103
|
-
"terms": [
|
|
104
|
-
"USSTHPI"
|
|
105
|
-
],
|
|
106
|
-
"why": "FRED carries the all-transactions and purchase-only HPI series under stable IDs."
|
|
107
|
-
}
|
|
108
|
-
]
|
|
109
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "fhfa",
|
|
3
|
+
"title": "FHFA House Price Index, conforming loan limits and Public Use Databases (fhfa.gov files)",
|
|
4
|
+
"agency": "Federal Housing Finance Agency",
|
|
5
|
+
"depth": "standard",
|
|
6
|
+
"domains": [
|
|
7
|
+
"housing",
|
|
8
|
+
"house prices",
|
|
9
|
+
"mortgage finance"
|
|
10
|
+
],
|
|
11
|
+
"keywords": [
|
|
12
|
+
"fhfa",
|
|
13
|
+
"house price index",
|
|
14
|
+
"hpi",
|
|
15
|
+
"home prices",
|
|
16
|
+
"house prices",
|
|
17
|
+
"home values",
|
|
18
|
+
"purchase-only index",
|
|
19
|
+
"all-transactions index",
|
|
20
|
+
"conforming loan limit",
|
|
21
|
+
"loan limit",
|
|
22
|
+
"jumbo",
|
|
23
|
+
"high-cost area",
|
|
24
|
+
"fannie mae",
|
|
25
|
+
"freddie mac",
|
|
26
|
+
"enterprises",
|
|
27
|
+
"gse",
|
|
28
|
+
"federal home loan bank",
|
|
29
|
+
"fhlbank",
|
|
30
|
+
"public use database",
|
|
31
|
+
"pudb",
|
|
32
|
+
"mortgage"
|
|
33
|
+
],
|
|
34
|
+
"questions": [
|
|
35
|
+
"latest FHFA house price index for the U.S., seasonally adjusted",
|
|
36
|
+
"house price growth by state over the last year",
|
|
37
|
+
"home price appreciation in a metro area since 2020",
|
|
38
|
+
"annual house price index by county",
|
|
39
|
+
"2026 conforming loan limit for a one-unit home in a county",
|
|
40
|
+
"which counties have high-cost loan limits"
|
|
41
|
+
],
|
|
42
|
+
"homepage": "https://www.fhfa.gov/data",
|
|
43
|
+
"terms_url": "https://www.fhfa.gov/about/fhfa-policies",
|
|
44
|
+
"auth": {
|
|
45
|
+
"env": null,
|
|
46
|
+
"required": "none",
|
|
47
|
+
"placement": "none",
|
|
48
|
+
"param": null,
|
|
49
|
+
"prefix": "",
|
|
50
|
+
"home": null,
|
|
51
|
+
"signup_url": null,
|
|
52
|
+
"echoes_key": false,
|
|
53
|
+
"key_only": [],
|
|
54
|
+
"extra": {},
|
|
55
|
+
"demo_key": null
|
|
56
|
+
},
|
|
57
|
+
"limits": {
|
|
58
|
+
"rps": 1,
|
|
59
|
+
"burst": 2,
|
|
60
|
+
"daily_quota": null,
|
|
61
|
+
"errors_per_min": null,
|
|
62
|
+
"bytes_per_min": null,
|
|
63
|
+
"max_rows": 400000
|
|
64
|
+
},
|
|
65
|
+
"routes": {
|
|
66
|
+
"api": {
|
|
67
|
+
"base": "https://www.fhfa.gov",
|
|
68
|
+
"probe": "https://www.fhfa.gov/hpi/download/quarterly_datasets/hpi_at_state.csv",
|
|
69
|
+
"hosts": [
|
|
70
|
+
"www.fhfa.gov"
|
|
71
|
+
]
|
|
72
|
+
},
|
|
73
|
+
"manual": {
|
|
74
|
+
"how": "Open each link in manual_download/INSTRUCTIONS.md in a browser; the file downloads (hpi_master.csv is 17 MB). Upload it under the name shown; the pack applies the request's filters to the uploaded file. Without the links: HPI files are listed on https://www.fhfa.gov/data/hpi/datasets, conforming loan limits on https://www.fhfa.gov/data/conforming-loan-limit, Public Use Databases on https://www.fhfa.gov/data/pudb."
|
|
75
|
+
}
|
|
76
|
+
},
|
|
77
|
+
"cache": {
|
|
78
|
+
"mode": "run_only",
|
|
79
|
+
"ttl_hours": null,
|
|
80
|
+
"why": "Files are overwritten in place at each release (monthly HPI, quarterly, annual); data/ is reused only within a run."
|
|
81
|
+
},
|
|
82
|
+
"redistribution": {
|
|
83
|
+
"mode": "allowed",
|
|
84
|
+
"why": "FHFA data are works of a U.S. government agency, in the public domain (17 U.S.C. 105); cite FHFA as the source."
|
|
85
|
+
},
|
|
86
|
+
"notices": [],
|
|
87
|
+
"retention": null,
|
|
88
|
+
"citation": "Federal Housing Finance Agency, {dataset}, www.fhfa.gov, retrieved {retrieved_long}.",
|
|
89
|
+
"audit_hooks": [],
|
|
90
|
+
"clarify": [
|
|
91
|
+
"Which index: purchase-only (the flagship, sales only) or all-transactions (adds refinance appraisals, longer history)? Default: purchase-only.",
|
|
92
|
+
"Seasonally adjusted or not? Default: seasonally adjusted for month-to-month or quarter-to-quarter changes, not adjusted for year-over-year changes.",
|
|
93
|
+
"Which loan limit: the baseline (most counties) or a specific county's limit (high-cost areas are higher), and for how many units? Default: one-unit."
|
|
94
|
+
],
|
|
95
|
+
"formats": [
|
|
96
|
+
"csv",
|
|
97
|
+
"xlsx"
|
|
98
|
+
],
|
|
99
|
+
"use_instead": [
|
|
100
|
+
{
|
|
101
|
+
"question": "One FHFA house price series for the U.S. or a state, with no index flavor choice",
|
|
102
|
+
"source": "fred",
|
|
103
|
+
"terms": [
|
|
104
|
+
"USSTHPI"
|
|
105
|
+
],
|
|
106
|
+
"why": "FRED carries the all-transactions and purchase-only HPI series under stable IDs."
|
|
107
|
+
}
|
|
108
|
+
]
|
|
109
|
+
}
|
|
@@ -1,122 +1,122 @@
|
|
|
1
|
-
<#
|
|
2
|
-
.SYNOPSIS
|
|
3
|
-
Download the FRED files for one data-analyst request into the folder that holds this script.
|
|
4
|
-
|
|
5
|
-
.DESCRIPTION
|
|
6
|
-
Files marked key, bearer or group come from the FRED API and need your own free FRED API key
|
|
7
|
-
(https://fredaccount.stlouisfed.org/apikeys). The script uses $env:FRED_API_KEY when it is set,
|
|
8
|
-
otherwise asks for it with hidden input; press Enter to skip those files. The key is never printed
|
|
9
|
-
and never written into the saved files.
|
|
10
|
-
|
|
11
|
-
.EXAMPLE
|
|
12
|
-
powershell -NoProfile -ExecutionPolicy Bypass -File "$HOME\Downloads\download_fred.ps1"
|
|
13
|
-
#>
|
|
14
|
-
[CmdletBinding()]
|
|
15
|
-
param([switch]$NoPause)
|
|
16
|
-
$ErrorActionPreference = 'Stop'
|
|
17
|
-
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
|
|
18
|
-
$here = Split-Path -Parent $MyInvocation.MyCommand.Path
|
|
19
|
-
|
|
20
|
-
# file <TAB> kind <TAB> pages <TAB> url <TAB> then_file <TAB> then_url ("-" = empty, pages 0 = unpaged)
|
|
21
|
-
# kind: plain (no key), key (key in the query), bearer (FRED API v2, key in a header, paged),
|
|
22
|
-
# group (GeoFRED series group; its fields fill the __GROUP_*__ placeholders of then_url)
|
|
23
|
-
$manifest = @'
|
|
24
|
-
@MANIFEST@
|
|
25
|
-
'@
|
|
26
|
-
$items = $manifest -split "`n" | ForEach-Object { $_.TrimEnd("`r") } | Where-Object { $_ } | ForEach-Object {
|
|
27
|
-
$f = $_ -split "`t"
|
|
28
|
-
[pscustomobject]@{ File = $f[0]; Kind = $f[1]; Pages = [int]$f[2]; Url = $f[3]; ThenFile = $f[4]; ThenUrl = $f[5] }
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
$key = $env:FRED_API_KEY
|
|
32
|
-
if (-not $key -and ($items | Where-Object { $_.Kind -ne 'plain' })) {
|
|
33
|
-
Write-Host 'Some files need a FRED API key (https://fredaccount.stlouisfed.org/apikeys). Press Enter to skip them.'
|
|
34
|
-
if ([Console]::IsInputRedirected) { $key = [Console]::In.ReadLine() }
|
|
35
|
-
else {
|
|
36
|
-
$secure = Read-Host -Prompt 'FRED API key (hidden)' -AsSecureString
|
|
37
|
-
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
|
38
|
-
try { $key = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
|
|
39
|
-
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
if ($null -eq $key) { $key = '' }
|
|
43
|
-
$key = $key -replace '\s', ''
|
|
44
|
-
|
|
45
|
-
$utf8 = New-Object System.Text.UTF8Encoding($false)
|
|
46
|
-
$saved = New-Object System.Collections.ArrayList
|
|
47
|
-
|
|
48
|
-
# Download one keyed file, wrap it with the request URL (placeholder, not the key) and save it.
|
|
49
|
-
# Returns the reply text, or $null when the request failed.
|
|
50
|
-
function Save-Keyed([string]$file, [string]$url, [bool]$bearer) {
|
|
51
|
-
try {
|
|
52
|
-
if ($bearer) {
|
|
53
|
-
$r = Invoke-WebRequest -UseBasicParsing -UserAgent 'data-analyst/1.0' -Uri $url `
|
|
54
|
-
-Headers @{ Authorization = "Bearer $key" }
|
|
55
|
-
} else {
|
|
56
|
-
$r = Invoke-WebRequest -UseBasicParsing -UserAgent 'data-analyst/1.0' -Uri ($url.Replace('__FRED_KEY__', $key))
|
|
57
|
-
}
|
|
58
|
-
} catch {
|
|
59
|
-
$msg = "$($_.Exception.Message) $($_.ErrorDetails.Message)"
|
|
60
|
-
if ($key) { $msg = $msg.Replace($key, '[KEY]') }
|
|
61
|
-
Write-Host "${file}: download failed ($msg)"
|
|
62
|
-
return $null
|
|
63
|
-
}
|
|
64
|
-
$text = $utf8.GetString($r.RawContentStream.ToArray())
|
|
65
|
-
$wrapped = '{"request": "' + $url + '", "response": ' + $text + '}'
|
|
66
|
-
$dest = Join-Path $here $file
|
|
67
|
-
[IO.File]::WriteAllText($dest, $wrapped, $utf8)
|
|
68
|
-
[void]$saved.Add($dest)
|
|
69
|
-
Write-Host "saved $file"
|
|
70
|
-
return $text
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
foreach ($it in $items) {
|
|
74
|
-
if ($it.Kind -eq 'plain') {
|
|
75
|
-
$dest = Join-Path $here $it.File
|
|
76
|
-
try {
|
|
77
|
-
Invoke-WebRequest -UseBasicParsing -UserAgent 'data-analyst/1.0' -Uri $it.Url -OutFile $dest
|
|
78
|
-
[void]$saved.Add($dest)
|
|
79
|
-
Write-Host "saved $($it.File)"
|
|
80
|
-
} catch { Write-Host "$($it.File): download failed ($($_.Exception.Message))" }
|
|
81
|
-
Start-Sleep -Milliseconds 600
|
|
82
|
-
continue
|
|
83
|
-
}
|
|
84
|
-
if (-not $key) { Write-Host "skipped (needs a key): $($it.File)"; continue }
|
|
85
|
-
switch ($it.Kind) {
|
|
86
|
-
'key' { [void](Save-Keyed $it.File $it.Url $false) }
|
|
87
|
-
'bearer' {
|
|
88
|
-
# FRED API v2 release/observations: follow next_cursor up to Pages pages
|
|
89
|
-
$n = 1; $u = $it.Url; $f = $it.File
|
|
90
|
-
while ($true) {
|
|
91
|
-
$body = Save-Keyed $f $u $true
|
|
92
|
-
if ($null -eq $body -or $body -notmatch '"has_more"\s*:\s*true') { break }
|
|
93
|
-
if ($body -notmatch '"next_cursor"\s*:\s*"([^"]*)"' -or $n -ge $it.Pages) { break }
|
|
94
|
-
$n++
|
|
95
|
-
$u = $it.Url + '&next_cursor=' + [Uri]::EscapeDataString($Matches[1])
|
|
96
|
-
$f = $it.File -replace '\.json$', ('__p{0:D2}.json' -f $n)
|
|
97
|
-
Start-Sleep -Milliseconds 600
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
'group' {
|
|
101
|
-
# GeoFRED: the series group first, then the cross-section it names
|
|
102
|
-
$body = Save-Keyed $it.File $it.Url $false
|
|
103
|
-
if ($null -ne $body) {
|
|
104
|
-
$g = (ConvertFrom-Json $body).series_group
|
|
105
|
-
$u = $it.ThenUrl
|
|
106
|
-
foreach ($k in 'series_group', 'region_type', 'season', 'units', 'frequency', 'max_date') {
|
|
107
|
-
$v = "$($g.$k)"
|
|
108
|
-
if ($k -eq 'frequency') { $v = $v.Substring(0, [Math]::Min(1, $v.Length)).ToLower() }
|
|
109
|
-
$u = $u.Replace('__GROUP_' + $k.ToUpper() + '__', [Uri]::EscapeDataString($v))
|
|
110
|
-
}
|
|
111
|
-
Start-Sleep -Milliseconds 600
|
|
112
|
-
[void](Save-Keyed $it.ThenFile $u $false)
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
Start-Sleep -Milliseconds 600
|
|
117
|
-
}
|
|
118
|
-
$key = $null
|
|
119
|
-
Write-Host ''
|
|
120
|
-
Write-Host 'Done. Upload these files to the chat:'
|
|
121
|
-
$saved | ForEach-Object { Write-Host " $_" }
|
|
122
|
-
if (-not $NoPause -and -not [Console]::IsInputRedirected) { Read-Host 'Press Enter to close' | Out-Null }
|
|
1
|
+
<#
|
|
2
|
+
.SYNOPSIS
|
|
3
|
+
Download the FRED files for one data-analyst request into the folder that holds this script.
|
|
4
|
+
|
|
5
|
+
.DESCRIPTION
|
|
6
|
+
Files marked key, bearer or group come from the FRED API and need your own free FRED API key
|
|
7
|
+
(https://fredaccount.stlouisfed.org/apikeys). The script uses $env:FRED_API_KEY when it is set,
|
|
8
|
+
otherwise asks for it with hidden input; press Enter to skip those files. The key is never printed
|
|
9
|
+
and never written into the saved files.
|
|
10
|
+
|
|
11
|
+
.EXAMPLE
|
|
12
|
+
powershell -NoProfile -ExecutionPolicy Bypass -File "$HOME\Downloads\download_fred.ps1"
|
|
13
|
+
#>
|
|
14
|
+
[CmdletBinding()]
|
|
15
|
+
param([switch]$NoPause)
|
|
16
|
+
$ErrorActionPreference = 'Stop'
|
|
17
|
+
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
|
|
18
|
+
$here = Split-Path -Parent $MyInvocation.MyCommand.Path
|
|
19
|
+
|
|
20
|
+
# file <TAB> kind <TAB> pages <TAB> url <TAB> then_file <TAB> then_url ("-" = empty, pages 0 = unpaged)
|
|
21
|
+
# kind: plain (no key), key (key in the query), bearer (FRED API v2, key in a header, paged),
|
|
22
|
+
# group (GeoFRED series group; its fields fill the __GROUP_*__ placeholders of then_url)
|
|
23
|
+
$manifest = @'
|
|
24
|
+
@MANIFEST@
|
|
25
|
+
'@
|
|
26
|
+
$items = $manifest -split "`n" | ForEach-Object { $_.TrimEnd("`r") } | Where-Object { $_ } | ForEach-Object {
|
|
27
|
+
$f = $_ -split "`t"
|
|
28
|
+
[pscustomobject]@{ File = $f[0]; Kind = $f[1]; Pages = [int]$f[2]; Url = $f[3]; ThenFile = $f[4]; ThenUrl = $f[5] }
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
$key = $env:FRED_API_KEY
|
|
32
|
+
if (-not $key -and ($items | Where-Object { $_.Kind -ne 'plain' })) {
|
|
33
|
+
Write-Host 'Some files need a FRED API key (https://fredaccount.stlouisfed.org/apikeys). Press Enter to skip them.'
|
|
34
|
+
if ([Console]::IsInputRedirected) { $key = [Console]::In.ReadLine() }
|
|
35
|
+
else {
|
|
36
|
+
$secure = Read-Host -Prompt 'FRED API key (hidden)' -AsSecureString
|
|
37
|
+
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
|
38
|
+
try { $key = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
|
|
39
|
+
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
if ($null -eq $key) { $key = '' }
|
|
43
|
+
$key = $key -replace '\s', ''
|
|
44
|
+
|
|
45
|
+
$utf8 = New-Object System.Text.UTF8Encoding($false)
|
|
46
|
+
$saved = New-Object System.Collections.ArrayList
|
|
47
|
+
|
|
48
|
+
# Download one keyed file, wrap it with the request URL (placeholder, not the key) and save it.
|
|
49
|
+
# Returns the reply text, or $null when the request failed.
|
|
50
|
+
function Save-Keyed([string]$file, [string]$url, [bool]$bearer) {
|
|
51
|
+
try {
|
|
52
|
+
if ($bearer) {
|
|
53
|
+
$r = Invoke-WebRequest -UseBasicParsing -UserAgent 'data-analyst/1.0' -Uri $url `
|
|
54
|
+
-Headers @{ Authorization = "Bearer $key" }
|
|
55
|
+
} else {
|
|
56
|
+
$r = Invoke-WebRequest -UseBasicParsing -UserAgent 'data-analyst/1.0' -Uri ($url.Replace('__FRED_KEY__', $key))
|
|
57
|
+
}
|
|
58
|
+
} catch {
|
|
59
|
+
$msg = "$($_.Exception.Message) $($_.ErrorDetails.Message)"
|
|
60
|
+
if ($key) { $msg = $msg.Replace($key, '[KEY]') }
|
|
61
|
+
Write-Host "${file}: download failed ($msg)"
|
|
62
|
+
return $null
|
|
63
|
+
}
|
|
64
|
+
$text = $utf8.GetString($r.RawContentStream.ToArray())
|
|
65
|
+
$wrapped = '{"request": "' + $url + '", "response": ' + $text + '}'
|
|
66
|
+
$dest = Join-Path $here $file
|
|
67
|
+
[IO.File]::WriteAllText($dest, $wrapped, $utf8)
|
|
68
|
+
[void]$saved.Add($dest)
|
|
69
|
+
Write-Host "saved $file"
|
|
70
|
+
return $text
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
foreach ($it in $items) {
|
|
74
|
+
if ($it.Kind -eq 'plain') {
|
|
75
|
+
$dest = Join-Path $here $it.File
|
|
76
|
+
try {
|
|
77
|
+
Invoke-WebRequest -UseBasicParsing -UserAgent 'data-analyst/1.0' -Uri $it.Url -OutFile $dest
|
|
78
|
+
[void]$saved.Add($dest)
|
|
79
|
+
Write-Host "saved $($it.File)"
|
|
80
|
+
} catch { Write-Host "$($it.File): download failed ($($_.Exception.Message))" }
|
|
81
|
+
Start-Sleep -Milliseconds 600
|
|
82
|
+
continue
|
|
83
|
+
}
|
|
84
|
+
if (-not $key) { Write-Host "skipped (needs a key): $($it.File)"; continue }
|
|
85
|
+
switch ($it.Kind) {
|
|
86
|
+
'key' { [void](Save-Keyed $it.File $it.Url $false) }
|
|
87
|
+
'bearer' {
|
|
88
|
+
# FRED API v2 release/observations: follow next_cursor up to Pages pages
|
|
89
|
+
$n = 1; $u = $it.Url; $f = $it.File
|
|
90
|
+
while ($true) {
|
|
91
|
+
$body = Save-Keyed $f $u $true
|
|
92
|
+
if ($null -eq $body -or $body -notmatch '"has_more"\s*:\s*true') { break }
|
|
93
|
+
if ($body -notmatch '"next_cursor"\s*:\s*"([^"]*)"' -or $n -ge $it.Pages) { break }
|
|
94
|
+
$n++
|
|
95
|
+
$u = $it.Url + '&next_cursor=' + [Uri]::EscapeDataString($Matches[1])
|
|
96
|
+
$f = $it.File -replace '\.json$', ('__p{0:D2}.json' -f $n)
|
|
97
|
+
Start-Sleep -Milliseconds 600
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
'group' {
|
|
101
|
+
# GeoFRED: the series group first, then the cross-section it names
|
|
102
|
+
$body = Save-Keyed $it.File $it.Url $false
|
|
103
|
+
if ($null -ne $body) {
|
|
104
|
+
$g = (ConvertFrom-Json $body).series_group
|
|
105
|
+
$u = $it.ThenUrl
|
|
106
|
+
foreach ($k in 'series_group', 'region_type', 'season', 'units', 'frequency', 'max_date') {
|
|
107
|
+
$v = "$($g.$k)"
|
|
108
|
+
if ($k -eq 'frequency') { $v = $v.Substring(0, [Math]::Min(1, $v.Length)).ToLower() }
|
|
109
|
+
$u = $u.Replace('__GROUP_' + $k.ToUpper() + '__', [Uri]::EscapeDataString($v))
|
|
110
|
+
}
|
|
111
|
+
Start-Sleep -Milliseconds 600
|
|
112
|
+
[void](Save-Keyed $it.ThenFile $u $false)
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
Start-Sleep -Milliseconds 600
|
|
117
|
+
}
|
|
118
|
+
$key = $null
|
|
119
|
+
Write-Host ''
|
|
120
|
+
Write-Host 'Done. Upload these files to the chat:'
|
|
121
|
+
$saved | ForEach-Object { Write-Host " $_" }
|
|
122
|
+
if (-not $NoPause -and -not [Console]::IsInputRedirected) { Read-Host 'Press Enter to close' | Out-Null }
|