data-analyst-mcp 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +262 -262
- package/dist/report/method_audit.js +7 -1
- package/dist/report/method_audit.js.map +1 -1
- package/dist/report/scan_secrets.d.ts +4 -1
- package/dist/report/scan_secrets.js +36 -5
- package/dist/report/scan_secrets.js.map +1 -1
- package/dist/server.d.ts +1 -1
- package/dist/server.js +3 -2
- package/dist/server.js.map +1 -1
- package/guide/WORKFLOW.md +134 -134
- package/guide/assets/disclaimer.txt +1 -1
- package/guide/assets/download_data.ps1 +162 -162
- package/guide/assets/report_template.html +597 -597
- package/guide/references/routing.md +171 -171
- package/guide/references/style_guide.md +398 -398
- package/package.json +56 -56
- package/packs/acf/endpoints.json +114 -114
- package/packs/acf/policy.json +104 -104
- package/packs/actuaries/endpoints.json +82 -82
- package/packs/actuaries/policy.json +109 -109
- package/packs/bea/assets/download_bea.ps1 +182 -182
- package/packs/bea/docs/bea_api.md +461 -461
- package/packs/bea/docs/datasets.md +157 -157
- package/packs/bea/docs/report_patterns.md +112 -112
- package/packs/bea/policy.json +170 -170
- package/packs/bea/source.md +152 -152
- package/packs/bls/assets/download_bls.ps1 +152 -152
- package/packs/bls/docs/analysis_patterns.md +108 -108
- package/packs/bls/docs/bls_api.md +73 -73
- package/packs/bls/docs/series_ids.md +149 -149
- package/packs/bls/policy.json +238 -238
- package/packs/bts/endpoints.json +346 -346
- package/packs/bts/policy.json +94 -94
- package/packs/cdc/endpoints.json +197 -197
- package/packs/cdc/policy.json +83 -83
- package/packs/census/endpoints.json +99 -99
- package/packs/cfpb/endpoints.json +326 -326
- package/packs/cfpb/policy.json +119 -119
- package/packs/clinical-trials/endpoints.json +166 -166
- package/packs/clinical-trials/policy.json +84 -84
- package/packs/cms/endpoints.json +69 -69
- package/packs/cms/policy.json +96 -96
- package/packs/college-scorecard/endpoints.json +252 -252
- package/packs/college-scorecard/policy.json +106 -106
- package/packs/congress/endpoints.json +1701 -1701
- package/packs/congress/policy.json +117 -117
- package/packs/dhs-ohss/endpoints.json +66 -66
- package/packs/dhs-ohss/policy.json +107 -107
- package/packs/doj-news/endpoints.json +59 -59
- package/packs/doj-news/policy.json +96 -96
- package/packs/dol/endpoints.json +817 -817
- package/packs/dol/policy.json +135 -135
- package/packs/eia/endpoints.json +717 -717
- package/packs/eia/policy.json +112 -112
- package/packs/epa/endpoints.json +291 -291
- package/packs/epa/policy.json +99 -99
- package/packs/epa-aqs/endpoints.json +173 -173
- package/packs/epa-aqs/policy.json +96 -96
- package/packs/fbi/endpoints.json +1841 -1841
- package/packs/fbi/policy.json +118 -118
- package/packs/fda/endpoints.json +237 -237
- package/packs/fda/policy.json +121 -121
- package/packs/fdic/endpoints.json +416 -416
- package/packs/fdic/policy.json +83 -83
- package/packs/fec/endpoints.json +707 -707
- package/packs/fec/policy.json +93 -93
- package/packs/fed-board/endpoints.json +89 -89
- package/packs/fed-board/policy.json +136 -136
- package/packs/federal-register/endpoints.json +143 -143
- package/packs/federal-register/policy.json +86 -86
- package/packs/fema/endpoints.json +645 -645
- package/packs/fema/policy.json +103 -103
- package/packs/fhfa/endpoints.json +105 -105
- package/packs/fhfa/policy.json +109 -109
- package/packs/fred/assets/download_fred.ps1 +122 -122
- package/packs/fred/docs/analysis_patterns.md +124 -124
- package/packs/fred/docs/fred_api.md +94 -94
- package/packs/fred/docs/series.md +188 -188
- package/packs/fred/policy.json +253 -253
- package/packs/fsa/endpoints.json +150 -150
- package/packs/fsa/policy.json +99 -99
- package/packs/govinfo/endpoints.json +519 -519
- package/packs/gsa-calc/endpoints.json +93 -93
- package/packs/gsa-calc/policy.json +97 -97
- package/packs/hud/endpoints.json +146 -146
- package/packs/hud/policy.json +84 -84
- package/packs/ipums/docs/report_patterns.md +117 -117
- package/packs/irs-soi/endpoints.json +114 -114
- package/packs/irs-soi/policy.json +107 -107
- package/packs/medicaid/endpoints.json +412 -412
- package/packs/medicaid/policy.json +108 -108
- package/packs/naep/endpoints.json +217 -217
- package/packs/naep/policy.json +100 -100
- package/packs/nhtsa/endpoints.json +132 -132
- package/packs/nhtsa/policy.json +93 -93
- package/packs/nih/endpoints.json +452 -452
- package/packs/nih/policy.json +86 -86
- package/packs/noaa/endpoints.json +431 -431
- package/packs/noaa/policy.json +95 -95
- package/packs/nrel/endpoints.json +327 -327
- package/packs/nrel/policy.json +96 -96
- package/packs/nws/endpoints.json +271 -271
- package/packs/nws/policy.json +82 -82
- package/packs/nyfed/endpoints.json +80 -80
- package/packs/nyfed/policy.json +92 -92
- package/packs/omb/endpoints.json +460 -460
- package/packs/omb/policy.json +106 -106
- package/packs/open-payments/endpoints.json +297 -297
- package/packs/open-payments/policy.json +101 -101
- package/packs/opm/assets/download_opm.ps1 +125 -125
- package/packs/opm/docs/fields.md +184 -184
- package/packs/opm/docs/opm_api.md +71 -71
- package/packs/opm/docs/report_patterns.md +83 -83
- package/packs/opm/lib/data_dictionary.csv +73 -73
- package/packs/pbgc/endpoints.json +483 -483
- package/packs/pbgc/policy.json +94 -94
- package/packs/philfed/endpoints.json +85 -85
- package/packs/philfed/policy.json +94 -94
- package/packs/regulations/endpoints.json +287 -287
- package/packs/regulations/policy.json +86 -86
- package/packs/sec/endpoints.json +236 -236
- package/packs/sec/policy.json +86 -86
- package/packs/senate-lobbying/endpoints.json +553 -553
- package/packs/senate-lobbying/policy.json +99 -99
- package/packs/snapqc/endpoints.json +1426 -1426
- package/packs/snapqc/policy.json +92 -92
- package/packs/treasury/catalog.json +2267 -2267
- package/packs/treasury/docs/api_guide.md +245 -245
- package/packs/treasury/docs/datasets.md +56 -56
- package/packs/treasury/docs/report_patterns.md +126 -126
- package/packs/treasury/policy.json +159 -159
- package/packs/treasury/source.md +170 -165
- package/packs/treasury-yields/endpoints.json +226 -226
- package/packs/treasury-yields/policy.json +86 -86
- package/packs/usaspending/endpoints.json +1315 -1315
- package/packs/usaspending/policy.json +109 -109
- package/packs/usda-fna/policy.json +133 -133
- package/packs/usda-fooddata/endpoints.json +180 -180
- package/packs/usda-fooddata/policy.json +94 -94
- package/packs/usda-nass/endpoints.json +323 -323
- package/packs/usda-nass/policy.json +95 -95
- package/packs/usda-outlook/endpoints.json +86 -86
- package/packs/usda-outlook/policy.json +109 -109
- package/packs/usgs/endpoints.json +313 -313
- package/packs/usgs/policy.json +95 -95
- package/packs/usitc/endpoints.json +74 -74
- package/packs/usitc/policy.json +111 -111
- package/packs/uspto/endpoints.json +141 -141
- package/packs/uspto/policy.json +95 -95
- package/packs/va/endpoints.json +414 -414
- package/packs/va/policy.json +101 -101
- package/packs/world-bank/endpoints.json +122 -122
- package/packs/world-bank/policy.json +101 -101
package/packs/bea/source.md
CHANGED
|
@@ -1,152 +1,152 @@
|
|
|
1
|
-
# Bureau of Economic Analysis (BEA) Data API and bulk files
|
|
2
|
-
Depth: deep. Key: required (BEA_API_KEY) for the API; the bulk route needs none. U.S. national accounts
|
|
3
|
-
(GDP, income, PCE and its prices, saving, profits, government), state and county GDP and income, regional
|
|
4
|
-
price parities, GDP by industry, trade and international accounts, multinationals, fixed assets, input-output.
|
|
5
|
-
|
|
6
|
-
## When to use
|
|
7
|
-
- GDP and its growth, contributions, prices and deflators; personal income, disposable income, saving rate;
|
|
8
|
-
PCE and core PCE price indexes (the Federal Reserve's inflation measure); corporate profits; NIPA
|
|
9
|
-
government receipts and spending (calendar years); GDP and personal income by state and county; per capita
|
|
10
|
-
income; RPPs; value added and gross output by industry; goods and services trade balances by country;
|
|
11
|
-
current account; investment position; direct investment; capital stocks.
|
|
12
|
-
- Not here: CPI, jobs, unemployment, wages by occupation (bls); budget data by fiscal year (treasury);
|
|
13
|
-
market interest rates (treasury-yields; fred); forecasts (BEA publishes history only); monthly GDP.
|
|
14
|
-
- The deliverable decides the work: data only (download, short preview, stop), an answer (compute with
|
|
15
|
-
`compute`, answer with the numbers, measure and vintage), or a report (`guide`'s report steps).
|
|
16
|
-
|
|
17
|
-
## Endpoints and datasets
|
|
18
|
-
One API endpoint (`https://apps.bea.gov/api/data`, GET, key in `UserID`), 13 datasets. Bulk files serve
|
|
19
|
-
the most used ones without a key and count against no limit.
|
|
20
|
-
|
|
21
|
-
| `dataset` | What | Spec keys | Bulk (no key)? |
|
|
22
|
-
|---|---|---|---|
|
|
23
|
-
| `NIPA` | national accounts tables `T10101` ... (titles: `request_schema` search) | `table`, `frequency` A/Q/M, `years`, `line_code` | yes (NIPA flat files) |
|
|
24
|
-
| `NIUnderlyingDetail` | detail tables `U20305` ... (lower quality) | same | no (flat files hold only shared lines) |
|
|
25
|
-
| `FixedAssets` | `FAAt101` ... annual | `table`, `years` | no |
|
|
26
|
-
| `Regional` | state `SA*`/`SQ*`, county `CA*`, metro `MA*` tables | `table`, `line_code`, `geo_fips`, `years` | yes for STATE, COUNTY, postal code (its counties), FIPS lists, MARPP |
|
|
27
|
-
| `GDPbyIndustry` | workbook codes `TVA101`-`TVA113`, `TGO1xx`, `TII1xx` | `table`, `frequency` A/Q, `years` | yes (public workbooks) |
|
|
28
|
-
| `GDPbyIndustry` (API) | `TableID` 1, 5, 6, 25 ...; `Industry` filters | `params` | no |
|
|
29
|
-
| `UnderlyingGDPbyIndustry`, `ITA`, `IIP`, `IntlServTrade`, `IntlServSTA`, `MNE`, `InputOutput` | see `request_schema` with `names: ["<name>"]` | `params` (raw API names) | no |
|
|
30
|
-
|
|
31
|
-
Verified lines (September 2026 vintage): T10101 line 1 real GDP percent change (annual rate); T10106 line 1
|
|
32
|
-
real GDP (millions of chained 2017 dollars, SAAR); T10105 line 1 nominal GDP; T10111 line 1 year-over-year
|
|
33
|
-
real GDP change; T10102 contributions; T20100 lines 1/27/37/35/40 personal income / DPI / real DPI / saving
|
|
34
|
-
rate / population (annual and quarterly only: monthly values are in T20600, whose line numbers differ, so
|
|
35
|
-
check them with the search `lines T20600`; a table is monthly only when its title says so, and the bulk route refuses
|
|
36
|
-
frequency M for any other table); T20804 lines 1/25 PCE / core PCE price index (monthly; published changes: T20807 month over month, not
|
|
37
|
-
annualized, T20811 lines 32/37 over 12 months); T70100 lines 1/3 GDP and income
|
|
38
|
-
per capita (BEA divides by unrounded population: real GDP over T70100's population, published in thousands,
|
|
39
|
-
can miss the published per-capita level by just over 0.5 dollars; allow for that rounding in the benchmark and
|
|
40
|
-
say so). Federal civilian wages per FTE (T60600D) jump in 2024-2025: Deferred Resignation staff on paid
|
|
41
|
-
administrative leave stayed on the payroll while BEA cut the quantity of federal labor (FAQ 1479,
|
|
42
|
-
bea.gov/help/faq/1479): never read those years as federal pay raises; use OPM pay or the GS raise. Regional: SAGDP2/SAGDP9 line 1 all-industry total (current / chained 2017 dollars); SAINC1 and
|
|
43
|
-
CAINC1 lines 1/2/3 personal income / population / per capita income; SAINC4/CAINC4 lines 10/20/30; SARPP
|
|
44
|
-
and MARPP line 1 all-items RPP. API names may carry an `N` (`SAGDP9N`); the bulk files drop it: both work.
|
|
45
|
-
BEA marks nothing as preliminary in the data, but the latest quarter or year of every table is an early
|
|
46
|
-
estimate that BEA will revise: register facts from it with `preliminary: true` in `facts` and say so. A year that has
|
|
47
|
-
been through an annual update (published at the end of September) is revised, not early: say "subject to
|
|
48
|
-
revision at the next annual update" instead.
|
|
49
|
-
More tables, the question-to-table map and concepts: `docs/datasets.md`. API parameters, response quirks,
|
|
50
|
-
error codes and limits: `docs/bea_api.md`. Report conventions: `docs/report_patterns.md`.
|
|
51
|
-
|
|
52
|
-
Discovery with `request_schema` (no key unless noted): `search: "real gross domestic product"` (NIPA titles
|
|
53
|
-
and workbook tables), `search: "Regional income"` (the first word names the dataset; API datasets need a key),
|
|
54
|
-
`search: "lines T10106"` (lines and series codes), `names: ["ITA"]` (required parameters, rules, spec keys,
|
|
55
|
-
and with a key BEA's current parameters); with a key (cached a day): `search: "values Regional TableName"`,
|
|
56
|
-
`search: "values Regional LineCode TableName=SAINC1"`. `source_brief` lists the 13 datasets.
|
|
57
|
-
|
|
58
|
-
## Spec examples
|
|
59
|
-
```json
|
|
60
|
-
{"description": "Real GDP growth, national and by state, 2019 to 2025",
|
|
61
|
-
"requests": [
|
|
62
|
-
{"id": "gdp_pct", "dataset": "NIPA", "table": "T10101", "line_code": 1, "frequency": "Q", "years": "2019-2025"},
|
|
63
|
-
{"id": "gdp_real", "dataset": "NIPA", "table": "T10106", "line_code": 1, "frequency": "Q", "years": "2019-2025"},
|
|
64
|
-
{"id": "state_real", "dataset": "Regional", "table": "SAGDP9", "line_code": 1, "geo_fips": "STATE", "years": "2019-2025"},
|
|
65
|
-
{"id": "va_ind", "dataset": "GDPbyIndustry", "table": "TVA105", "frequency": "A", "years": "2024-2025"}]}
|
|
66
|
-
```
|
|
67
|
-
```json
|
|
68
|
-
{"description": "Per capita personal income, two counties, last five years",
|
|
69
|
-
"requests": [{"id": "cty_pi", "dataset": "Regional", "table": "CAINC1", "line_code": [1, 3],
|
|
70
|
-
"geo_fips": ["06037", "17031"], "years": "LAST5"}]}
|
|
71
|
-
```
|
|
72
|
-
```json
|
|
73
|
-
{"description": "U.S. goods trade balance with China (needs a key)",
|
|
74
|
-
"requests": [{"id": "ita", "dataset": "ITA",
|
|
75
|
-
"params": {"Indicator": "BalGds", "AreaOrCountry": "China", "Frequency": "A", "Year": "2023,2024"}}]}
|
|
76
|
-
```
|
|
77
|
-
`years`: `"2019-2025"`, a list, `"ALL"` or `"LAST5"`; `line_code`: a number, list or `"ALL"` (NIPA: BEA
|
|
78
|
-
sends the whole table and the rows are filtered); `geo_fips`: `STATE`, `COUNTY`, a postal code (its
|
|
79
|
-
counties), FIPS codes (`"06"` = `06000`). Keys a dataset does not use are reported in the result's notes. When a growth
|
|
80
|
-
rate is computed from levels, also request BEA's published percent change so it can be checked.
|
|
81
|
-
|
|
82
|
-
## Data format and traps
|
|
83
|
-
`fetch` writes `data/bea/`: `bea_data.csv` (tidy: `request_id, dataset, table, line, series_code,
|
|
84
|
-
description, geo_fips, geo_name, geo_level, industry, time_period, date, year, subperiod, frequency, value,
|
|
85
|
-
flag, unit, metric, unit_mult, note_ref, dims`), `notes.csv` (BEA's footnotes: units, reference year, what
|
|
86
|
-
the U.S. row means, flag codes), `request.json` (calls without the key, bulk files, route, vintage,
|
|
87
|
-
checksums) and `README.md`. Read the README and `notes.csv` before analysing (the NIPA bulk files carry no
|
|
88
|
-
footnotes: on the `bulk` route `notes.csv` has only the pack's own note; read a table's footnotes on its
|
|
89
|
-
apps.bea.gov page, or fetch it through the API with a key). `line` holds BEA's line numbers, which can carry
|
|
90
|
-
letters: match lines exactly (as text, or with `series_code`), never by integer order.
|
|
91
|
-
- **Units:** `value` is as BEA prints it; base units = value x 10^`unit_mult` (`compute` `derive`:
|
|
92
|
-
`value * pow(10, unit_mult)`).
|
|
93
|
-
Check `unit` and `metric` (current dollars, chained dollars, index, percent change) for every series.
|
|
94
|
-
- **Special values** `(D)` suppressed, `(NA)` not available, `(NM)` not meaningful, `(L)` under $50,000 or 10
|
|
95
|
-
jobs, `(S)`, `(X)`, `(*)` (MNE: rounds to zero): `value` is empty and the code is in `flag`. Never a zero;
|
|
96
|
-
list the rows with a `flag` and drop or footnote the area.
|
|
97
|
-
- **No sums of chained dollars**, of components or of states; shares from current dollars; drivers of growth
|
|
98
|
-
from BEA's contributions tables (T10102, SAGDP11). SAAR levels are averaged, not added, for annual values.
|
|
99
|
-
- **Growth measures differ** by a point or more: quarterly at a compound annual rate (BEA's headline:
|
|
100
|
-
(level / previous level)^4 - 1), year over year (T10111), Q4 over Q4, annual average; "since <year>":
|
|
101
|
-
cumulative change and the average annual rate. Name the measure everywhere; show the alternatives as a
|
|
102
|
-
sensitivity.
|
|
103
|
-
- **Check against BEA's published numbers** (docs/datasets.md section 5): growth
|
|
104
|
-
from T10106 levels vs T10101; the U.S. row of SAGDP9 vs T10106 line 1. A mismatch beyond rounding is an
|
|
105
|
-
error; a difference from an older release is a revision (compare dates with the vintage in README.md).
|
|
106
|
-
- **Vintage:** BEA serves only its latest estimates (advance, second, third; annual updates each September;
|
|
107
|
-
comprehensive updates every five years). Record the release date; old vintages are at apps.bea.gov/histdata.
|
|
108
|
-
- **Geography:** every row has `geo_level` (nation, region, state, county, msa); a COUNTY bulk download also
|
|
109
|
-
holds state and U.S. rows: filter before summing. `STATE` = 50 states + D.C. + U.S. (say whether D.C. is
|
|
110
|
-
ranked). Since February 2026 BEA no longer produces GDP or personal income for metro areas: sum component
|
|
111
|
-
counties' current-dollar values and say so. Rank on unrounded values; name near-ties.
|
|
112
|
-
- **Calendar years** everywhere (NIPA federal spending too, unlike fiscal-year budget data). NIPA line
|
|
113
|
-
numbers change at comprehensive updates: check the description or series code, not only the line.
|
|
114
|
-
GDP-by-industry line numbers differ between tables (TVA106 has an extra line): match by description.
|
|
115
|
-
- **No standard errors** exist for BEA estimates: never "statistically different".
|
|
116
|
-
- Analysis: build growth rates, shares and comparisons with `compute` on `data/bea/bea_data.csv`, and check each
|
|
117
|
-
headline against the published table.
|
|
118
|
-
|
|
119
|
-
## Access
|
|
120
|
-
- **Key** `BEA_API_KEY` (free, https://apps.bea.gov/API/signup/; BEA emails it and it works only after the
|
|
121
|
-
activation link is clicked). Found, in order, in the environment, `BEA_API_KEY_FILE`, keys.json, Claude Code
|
|
122
|
-
settings.json and `~/.bea/api_key`; the user stores it with `npx -y data-analyst-mcp setup bea` in their own
|
|
123
|
-
terminal. Never ask for the key in chat. BEA puts it in the URL and echoes it in every
|
|
124
|
-
reply: the server scrubs both (the key and the USERID echo); never build your own API URLs.
|
|
125
|
-
- **Limits** per key: 100 requests, 100 MB and 30 errors a minute; going over blocks the key for a minute to
|
|
126
|
-
an hour (HTTP 429, never retried). The client spaces calls 0.75 s, waits before crossing 100 MB a minute,
|
|
127
|
-
stops after 30 errors a minute or 10 in a run, validates table and line names against cached metadata
|
|
128
|
-
first, and checks dataset parameter rules offline (`invalid_request`), so errors are not spent.
|
|
129
|
-
- **Routes** (`route` in the result): `reuse_existing` (`data/bea/` already holds the request; `force: true`
|
|
130
|
-
downloads again), `api` (key works), `bulk` (no usable key and every request is in the bulk files: NIPA
|
|
131
|
-
flat files, state and county zips, GDP-by-industry workbooks; same latest estimates; the spec's
|
|
132
|
-
`"prefer": "bulk"` forces it, `"prefer": "api"` never falls back). A key BEA rejects or has locked out falls
|
|
133
|
-
back to the bulk files when they hold every request. Errors: `key_missing` (no key and a request needs the
|
|
134
|
-
API: the user runs `npx -y data-analyst-mcp setup bea`, or `manual_bundle`), `key_rejected`,
|
|
135
|
-
`rate_limited`, `agency_rejected` (fix the spec; never loop: errors count against the key) and
|
|
136
|
-
`agency_unavailable` (BEA unreachable; never invent or recall data). Bulk files are cached in
|
|
137
|
-
`<cache_root>/bea/bulk` (`DATA_ANALYST_CACHE` sets the root) and revalidated by Last-Modified; NIPA files are
|
|
138
|
-
12-37 MB, county zips up to 50 MB.
|
|
139
|
-
- **Manual download:** `manual_bundle` writes `manual_download/`: `INSTRUCTIONS.md` with the links, and
|
|
140
|
-
`download_data.sh` (macOS, Linux) and `download_data.ps1` (Windows), one-command scripts that download the
|
|
141
|
-
public files and ask for the key with hidden input only when an API request needs it. Send both scripts and
|
|
142
|
-
INSTRUCTIONS.md, and give the one command for the user's system (`user_commands`). BEA repeats the key inside
|
|
143
|
-
API replies: `ingest` removes it. Then `ingest` with the uploaded files or their folder (on a hosted server,
|
|
144
|
-
`files: ["uploads"]`). Say in the methods that the files were downloaded by hand.
|
|
145
|
-
|
|
146
|
-
## Citation and terms
|
|
147
|
-
"U.S. Bureau of Economic Analysis, <tables>, <route> (accessed <date>)." The exact line is in
|
|
148
|
-
`data/bea/README.md`; include the release (vintage) when it matters. Source line: "Analyst calculations using
|
|
149
|
-
data from the Bureau of Economic Analysis." BEA data are public domain: redistribution allowed (data files
|
|
150
|
-
may ship in a replication package). Products that use the API must show: "This product uses the Bureau of
|
|
151
|
-
Economic Analysis (BEA) Data API but is not endorsed or certified by BEA." (the API terms' wording) Do not
|
|
152
|
-
imply BEA endorsement.
|
|
1
|
+
# Bureau of Economic Analysis (BEA) Data API and bulk files
|
|
2
|
+
Depth: deep. Key: required (BEA_API_KEY) for the API; the bulk route needs none. U.S. national accounts
|
|
3
|
+
(GDP, income, PCE and its prices, saving, profits, government), state and county GDP and income, regional
|
|
4
|
+
price parities, GDP by industry, trade and international accounts, multinationals, fixed assets, input-output.
|
|
5
|
+
|
|
6
|
+
## When to use
|
|
7
|
+
- GDP and its growth, contributions, prices and deflators; personal income, disposable income, saving rate;
|
|
8
|
+
PCE and core PCE price indexes (the Federal Reserve's inflation measure); corporate profits; NIPA
|
|
9
|
+
government receipts and spending (calendar years); GDP and personal income by state and county; per capita
|
|
10
|
+
income; RPPs; value added and gross output by industry; goods and services trade balances by country;
|
|
11
|
+
current account; investment position; direct investment; capital stocks.
|
|
12
|
+
- Not here: CPI, jobs, unemployment, wages by occupation (bls); budget data by fiscal year (treasury);
|
|
13
|
+
market interest rates (treasury-yields; fred); forecasts (BEA publishes history only); monthly GDP.
|
|
14
|
+
- The deliverable decides the work: data only (download, short preview, stop), an answer (compute with
|
|
15
|
+
`compute`, answer with the numbers, measure and vintage), or a report (`guide`'s report steps).
|
|
16
|
+
|
|
17
|
+
## Endpoints and datasets
|
|
18
|
+
One API endpoint (`https://apps.bea.gov/api/data`, GET, key in `UserID`), 13 datasets. Bulk files serve
|
|
19
|
+
the most used ones without a key and count against no limit.
|
|
20
|
+
|
|
21
|
+
| `dataset` | What | Spec keys | Bulk (no key)? |
|
|
22
|
+
|---|---|---|---|
|
|
23
|
+
| `NIPA` | national accounts tables `T10101` ... (titles: `request_schema` search) | `table`, `frequency` A/Q/M, `years`, `line_code` | yes (NIPA flat files) |
|
|
24
|
+
| `NIUnderlyingDetail` | detail tables `U20305` ... (lower quality) | same | no (flat files hold only shared lines) |
|
|
25
|
+
| `FixedAssets` | `FAAt101` ... annual | `table`, `years` | no |
|
|
26
|
+
| `Regional` | state `SA*`/`SQ*`, county `CA*`, metro `MA*` tables | `table`, `line_code`, `geo_fips`, `years` | yes for STATE, COUNTY, postal code (its counties), FIPS lists, MARPP |
|
|
27
|
+
| `GDPbyIndustry` | workbook codes `TVA101`-`TVA113`, `TGO1xx`, `TII1xx` | `table`, `frequency` A/Q, `years` | yes (public workbooks) |
|
|
28
|
+
| `GDPbyIndustry` (API) | `TableID` 1, 5, 6, 25 ...; `Industry` filters | `params` | no |
|
|
29
|
+
| `UnderlyingGDPbyIndustry`, `ITA`, `IIP`, `IntlServTrade`, `IntlServSTA`, `MNE`, `InputOutput` | see `request_schema` with `names: ["<name>"]` | `params` (raw API names) | no |
|
|
30
|
+
|
|
31
|
+
Verified lines (September 2026 vintage): T10101 line 1 real GDP percent change (annual rate); T10106 line 1
|
|
32
|
+
real GDP (millions of chained 2017 dollars, SAAR); T10105 line 1 nominal GDP; T10111 line 1 year-over-year
|
|
33
|
+
real GDP change; T10102 contributions; T20100 lines 1/27/37/35/40 personal income / DPI / real DPI / saving
|
|
34
|
+
rate / population (annual and quarterly only: monthly values are in T20600, whose line numbers differ, so
|
|
35
|
+
check them with the search `lines T20600`; a table is monthly only when its title says so, and the bulk route refuses
|
|
36
|
+
frequency M for any other table); T20804 lines 1/25 PCE / core PCE price index (monthly; published changes: T20807 month over month, not
|
|
37
|
+
annualized, T20811 lines 32/37 over 12 months); T70100 lines 1/3 GDP and income
|
|
38
|
+
per capita (BEA divides by unrounded population: real GDP over T70100's population, published in thousands,
|
|
39
|
+
can miss the published per-capita level by just over 0.5 dollars; allow for that rounding in the benchmark and
|
|
40
|
+
say so). Federal civilian wages per FTE (T60600D) jump in 2024-2025: Deferred Resignation staff on paid
|
|
41
|
+
administrative leave stayed on the payroll while BEA cut the quantity of federal labor (FAQ 1479,
|
|
42
|
+
bea.gov/help/faq/1479): never read those years as federal pay raises; use OPM pay or the GS raise. Regional: SAGDP2/SAGDP9 line 1 all-industry total (current / chained 2017 dollars); SAINC1 and
|
|
43
|
+
CAINC1 lines 1/2/3 personal income / population / per capita income; SAINC4/CAINC4 lines 10/20/30; SARPP
|
|
44
|
+
and MARPP line 1 all-items RPP. API names may carry an `N` (`SAGDP9N`); the bulk files drop it: both work.
|
|
45
|
+
BEA marks nothing as preliminary in the data, but the latest quarter or year of every table is an early
|
|
46
|
+
estimate that BEA will revise: register facts from it with `preliminary: true` in `facts` and say so. A year that has
|
|
47
|
+
been through an annual update (published at the end of September) is revised, not early: say "subject to
|
|
48
|
+
revision at the next annual update" instead.
|
|
49
|
+
More tables, the question-to-table map and concepts: `docs/datasets.md`. API parameters, response quirks,
|
|
50
|
+
error codes and limits: `docs/bea_api.md`. Report conventions: `docs/report_patterns.md`.
|
|
51
|
+
|
|
52
|
+
Discovery with `request_schema` (no key unless noted): `search: "real gross domestic product"` (NIPA titles
|
|
53
|
+
and workbook tables), `search: "Regional income"` (the first word names the dataset; API datasets need a key),
|
|
54
|
+
`search: "lines T10106"` (lines and series codes), `names: ["ITA"]` (required parameters, rules, spec keys,
|
|
55
|
+
and with a key BEA's current parameters); with a key (cached a day): `search: "values Regional TableName"`,
|
|
56
|
+
`search: "values Regional LineCode TableName=SAINC1"`. `source_brief` lists the 13 datasets.
|
|
57
|
+
|
|
58
|
+
## Spec examples
|
|
59
|
+
```json
|
|
60
|
+
{"description": "Real GDP growth, national and by state, 2019 to 2025",
|
|
61
|
+
"requests": [
|
|
62
|
+
{"id": "gdp_pct", "dataset": "NIPA", "table": "T10101", "line_code": 1, "frequency": "Q", "years": "2019-2025"},
|
|
63
|
+
{"id": "gdp_real", "dataset": "NIPA", "table": "T10106", "line_code": 1, "frequency": "Q", "years": "2019-2025"},
|
|
64
|
+
{"id": "state_real", "dataset": "Regional", "table": "SAGDP9", "line_code": 1, "geo_fips": "STATE", "years": "2019-2025"},
|
|
65
|
+
{"id": "va_ind", "dataset": "GDPbyIndustry", "table": "TVA105", "frequency": "A", "years": "2024-2025"}]}
|
|
66
|
+
```
|
|
67
|
+
```json
|
|
68
|
+
{"description": "Per capita personal income, two counties, last five years",
|
|
69
|
+
"requests": [{"id": "cty_pi", "dataset": "Regional", "table": "CAINC1", "line_code": [1, 3],
|
|
70
|
+
"geo_fips": ["06037", "17031"], "years": "LAST5"}]}
|
|
71
|
+
```
|
|
72
|
+
```json
|
|
73
|
+
{"description": "U.S. goods trade balance with China (needs a key)",
|
|
74
|
+
"requests": [{"id": "ita", "dataset": "ITA",
|
|
75
|
+
"params": {"Indicator": "BalGds", "AreaOrCountry": "China", "Frequency": "A", "Year": "2023,2024"}}]}
|
|
76
|
+
```
|
|
77
|
+
`years`: `"2019-2025"`, a list, `"ALL"` or `"LAST5"`; `line_code`: a number, list or `"ALL"` (NIPA: BEA
|
|
78
|
+
sends the whole table and the rows are filtered); `geo_fips`: `STATE`, `COUNTY`, a postal code (its
|
|
79
|
+
counties), FIPS codes (`"06"` = `06000`). Keys a dataset does not use are reported in the result's notes. When a growth
|
|
80
|
+
rate is computed from levels, also request BEA's published percent change so it can be checked.
|
|
81
|
+
|
|
82
|
+
## Data format and traps
|
|
83
|
+
`fetch` writes `data/bea/`: `bea_data.csv` (tidy: `request_id, dataset, table, line, series_code,
|
|
84
|
+
description, geo_fips, geo_name, geo_level, industry, time_period, date, year, subperiod, frequency, value,
|
|
85
|
+
flag, unit, metric, unit_mult, note_ref, dims`), `notes.csv` (BEA's footnotes: units, reference year, what
|
|
86
|
+
the U.S. row means, flag codes), `request.json` (calls without the key, bulk files, route, vintage,
|
|
87
|
+
checksums) and `README.md`. Read the README and `notes.csv` before analysing (the NIPA bulk files carry no
|
|
88
|
+
footnotes: on the `bulk` route `notes.csv` has only the pack's own note; read a table's footnotes on its
|
|
89
|
+
apps.bea.gov page, or fetch it through the API with a key). `line` holds BEA's line numbers, which can carry
|
|
90
|
+
letters: match lines exactly (as text, or with `series_code`), never by integer order.
|
|
91
|
+
- **Units:** `value` is as BEA prints it; base units = value x 10^`unit_mult` (`compute` `derive`:
|
|
92
|
+
`value * pow(10, unit_mult)`).
|
|
93
|
+
Check `unit` and `metric` (current dollars, chained dollars, index, percent change) for every series.
|
|
94
|
+
- **Special values** `(D)` suppressed, `(NA)` not available, `(NM)` not meaningful, `(L)` under $50,000 or 10
|
|
95
|
+
jobs, `(S)`, `(X)`, `(*)` (MNE: rounds to zero): `value` is empty and the code is in `flag`. Never a zero;
|
|
96
|
+
list the rows with a `flag` and drop or footnote the area.
|
|
97
|
+
- **No sums of chained dollars**, of components or of states; shares from current dollars; drivers of growth
|
|
98
|
+
from BEA's contributions tables (T10102, SAGDP11). SAAR levels are averaged, not added, for annual values.
|
|
99
|
+
- **Growth measures differ** by a point or more: quarterly at a compound annual rate (BEA's headline:
|
|
100
|
+
(level / previous level)^4 - 1), year over year (T10111), Q4 over Q4, annual average; "since <year>":
|
|
101
|
+
cumulative change and the average annual rate. Name the measure everywhere; show the alternatives as a
|
|
102
|
+
sensitivity.
|
|
103
|
+
- **Check against BEA's published numbers** (docs/datasets.md section 5): growth
|
|
104
|
+
from T10106 levels vs T10101; the U.S. row of SAGDP9 vs T10106 line 1. A mismatch beyond rounding is an
|
|
105
|
+
error; a difference from an older release is a revision (compare dates with the vintage in README.md).
|
|
106
|
+
- **Vintage:** BEA serves only its latest estimates (advance, second, third; annual updates each September;
|
|
107
|
+
comprehensive updates every five years). Record the release date; old vintages are at apps.bea.gov/histdata.
|
|
108
|
+
- **Geography:** every row has `geo_level` (nation, region, state, county, msa); a COUNTY bulk download also
|
|
109
|
+
holds state and U.S. rows: filter before summing. `STATE` = 50 states + D.C. + U.S. (say whether D.C. is
|
|
110
|
+
ranked). Since February 2026 BEA no longer produces GDP or personal income for metro areas: sum component
|
|
111
|
+
counties' current-dollar values and say so. Rank on unrounded values; name near-ties.
|
|
112
|
+
- **Calendar years** everywhere (NIPA federal spending too, unlike fiscal-year budget data). NIPA line
|
|
113
|
+
numbers change at comprehensive updates: check the description or series code, not only the line.
|
|
114
|
+
GDP-by-industry line numbers differ between tables (TVA106 has an extra line): match by description.
|
|
115
|
+
- **No standard errors** exist for BEA estimates: never "statistically different".
|
|
116
|
+
- Analysis: build growth rates, shares and comparisons with `compute` on `data/bea/bea_data.csv`, and check each
|
|
117
|
+
headline against the published table.
|
|
118
|
+
|
|
119
|
+
## Access
|
|
120
|
+
- **Key** `BEA_API_KEY` (free, https://apps.bea.gov/API/signup/; BEA emails it and it works only after the
|
|
121
|
+
activation link is clicked). Found, in order, in the environment, `BEA_API_KEY_FILE`, keys.json, Claude Code
|
|
122
|
+
settings.json and `~/.bea/api_key`; the user stores it with `npx -y data-analyst-mcp setup bea` in their own
|
|
123
|
+
terminal. Never ask for the key in chat. BEA puts it in the URL and echoes it in every
|
|
124
|
+
reply: the server scrubs both (the key and the USERID echo); never build your own API URLs.
|
|
125
|
+
- **Limits** per key: 100 requests, 100 MB and 30 errors a minute; going over blocks the key for a minute to
|
|
126
|
+
an hour (HTTP 429, never retried). The client spaces calls 0.75 s, waits before crossing 100 MB a minute,
|
|
127
|
+
stops after 30 errors a minute or 10 in a run, validates table and line names against cached metadata
|
|
128
|
+
first, and checks dataset parameter rules offline (`invalid_request`), so errors are not spent.
|
|
129
|
+
- **Routes** (`route` in the result): `reuse_existing` (`data/bea/` already holds the request; `force: true`
|
|
130
|
+
downloads again), `api` (key works), `bulk` (no usable key and every request is in the bulk files: NIPA
|
|
131
|
+
flat files, state and county zips, GDP-by-industry workbooks; same latest estimates; the spec's
|
|
132
|
+
`"prefer": "bulk"` forces it, `"prefer": "api"` never falls back). A key BEA rejects or has locked out falls
|
|
133
|
+
back to the bulk files when they hold every request. Errors: `key_missing` (no key and a request needs the
|
|
134
|
+
API: the user runs `npx -y data-analyst-mcp setup bea`, or `manual_bundle`), `key_rejected`,
|
|
135
|
+
`rate_limited`, `agency_rejected` (fix the spec; never loop: errors count against the key) and
|
|
136
|
+
`agency_unavailable` (BEA unreachable; never invent or recall data). Bulk files are cached in
|
|
137
|
+
`<cache_root>/bea/bulk` (`DATA_ANALYST_CACHE` sets the root) and revalidated by Last-Modified; NIPA files are
|
|
138
|
+
12-37 MB, county zips up to 50 MB.
|
|
139
|
+
- **Manual download:** `manual_bundle` writes `manual_download/`: `INSTRUCTIONS.md` with the links, and
|
|
140
|
+
`download_data.sh` (macOS, Linux) and `download_data.ps1` (Windows), one-command scripts that download the
|
|
141
|
+
public files and ask for the key with hidden input only when an API request needs it. Send both scripts and
|
|
142
|
+
INSTRUCTIONS.md, and give the one command for the user's system (`user_commands`). BEA repeats the key inside
|
|
143
|
+
API replies: `ingest` removes it. Then `ingest` with the uploaded files or their folder (on a hosted server,
|
|
144
|
+
`files: ["uploads"]`). Say in the methods that the files were downloaded by hand.
|
|
145
|
+
|
|
146
|
+
## Citation and terms
|
|
147
|
+
"U.S. Bureau of Economic Analysis, <tables>, <route> (accessed <date>)." The exact line is in
|
|
148
|
+
`data/bea/README.md`; include the release (vintage) when it matters. Source line: "Analyst calculations using
|
|
149
|
+
data from the Bureau of Economic Analysis." BEA data are public domain: redistribution allowed (data files
|
|
150
|
+
may ship in a replication package). Products that use the API must show: "This product uses the Bureau of
|
|
151
|
+
Economic Analysis (BEA) Data API but is not endorsed or certified by BEA." (the API terms' wording) Do not
|
|
152
|
+
imply BEA endorsement.
|
|
@@ -1,152 +1,152 @@
|
|
|
1
|
-
<#
|
|
2
|
-
.SYNOPSIS
|
|
3
|
-
Download BLS time series with the BLS Public Data API: send the requests built into this script and
|
|
4
|
-
save the replies as response_*.json next to it. (Windows PowerShell counterpart of download_bls.sh.)
|
|
5
|
-
|
|
6
|
-
.DESCRIPTION
|
|
7
|
-
- Needs no other file: the requests are inside this script.
|
|
8
|
-
- When the requests need a BLS registration key (API v2), uses $env:BLS_API_KEY when it is set and
|
|
9
|
-
otherwise asks for the key (hidden input). The key is only sent to api.bls.gov: it is never shown,
|
|
10
|
-
saved, or put on a command line. With a key set, keyless requests also go through API v2.
|
|
11
|
-
- Saves response_01.json, response_02.json, ... next to this script, each with the request it answers
|
|
12
|
-
(without the key), and prints their full paths at the end.
|
|
13
|
-
- Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
|
|
14
|
-
|
|
15
|
-
Get a free key at https://data.bls.gov/registrationEngine/ (BLS emails it within minutes).
|
|
16
|
-
|
|
17
|
-
.PARAMETER NoPause
|
|
18
|
-
Do not wait for Enter before closing the window.
|
|
19
|
-
|
|
20
|
-
.EXAMPLE
|
|
21
|
-
powershell -NoProfile -ExecutionPolicy Bypass -File "$HOME\Downloads\download_bls.ps1"
|
|
22
|
-
#>
|
|
23
|
-
[CmdletBinding()]
|
|
24
|
-
param(
|
|
25
|
-
[switch]$NoPause
|
|
26
|
-
)
|
|
27
|
-
|
|
28
|
-
$ErrorActionPreference = 'Stop'
|
|
29
|
-
$ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
|
|
30
|
-
try {
|
|
31
|
-
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
|
32
|
-
} catch { }
|
|
33
|
-
|
|
34
|
-
$Here = $PSScriptRoot
|
|
35
|
-
if (-not $Here) { $Here = (Get-Location).Path }
|
|
36
|
-
$Utf8 = New-Object Text.UTF8Encoding($false)
|
|
37
|
-
$Key = $null
|
|
38
|
-
$Req = $null
|
|
39
|
-
|
|
40
|
-
# Keep the window open when the script was started with "Run with PowerShell".
|
|
41
|
-
function Finish([int]$Code) {
|
|
42
|
-
$script:Key = $null
|
|
43
|
-
if ($script:Req) { Remove-Item -LiteralPath $script:Req -Recurse -Force -ErrorAction SilentlyContinue }
|
|
44
|
-
if (-not $NoPause -and -not [Console]::IsInputRedirected) {
|
|
45
|
-
Write-Host ''
|
|
46
|
-
[void](Read-Host -Prompt 'Press Enter to close')
|
|
47
|
-
}
|
|
48
|
-
exit $Code
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
function Fail([string]$Message) {
|
|
52
|
-
Write-Host ''
|
|
53
|
-
Write-Host "ERROR: $(Hide-Key $Message)" -ForegroundColor Red
|
|
54
|
-
Finish 1
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
# BLS repeats a rejected key in its error message: never let it reach the screen or a file.
|
|
58
|
-
function Hide-Key([string]$Text) {
|
|
59
|
-
if ($script:Key -and $Text) { return $Text.Replace($script:Key, '[REDACTED]') }
|
|
60
|
-
return $Text
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
# ---------------------------------------------------------------------------- the requests
|
|
64
|
-
# manifest.json and query_NN.json are built into this script; they go to a temporary folder that is
|
|
65
|
-
# removed when the script ends.
|
|
66
|
-
$Embedded = [ordered]@{}
|
|
67
|
-
# @QUERIES@
|
|
68
|
-
$Req = Join-Path ([IO.Path]::GetTempPath()) ('download_bls.' + [IO.Path]::GetRandomFileName())
|
|
69
|
-
[void](New-Item -ItemType Directory -Path $Req)
|
|
70
|
-
foreach ($name in $Embedded.Keys) { [IO.File]::WriteAllText((Join-Path $Req $name), $Embedded[$name], $Utf8) }
|
|
71
|
-
$manifestFile = Join-Path $Req 'manifest.json'
|
|
72
|
-
if (-not (Test-Path -LiteralPath $manifestFile)) {
|
|
73
|
-
Fail "This script has no requests built in. Use the download_bls.ps1 from the analyst's download files."
|
|
74
|
-
}
|
|
75
|
-
$Manifest = [IO.File]::ReadAllText($manifestFile, $Utf8) | ConvertFrom-Json
|
|
76
|
-
$Queries = @(Get-ChildItem -LiteralPath $Req -Filter 'query_*.json' | Sort-Object Name)
|
|
77
|
-
if ($Queries.Count -eq 0) { Fail "This script has no requests built in. Use the download_bls.ps1 from the analyst's download files." }
|
|
78
|
-
$Bodies = @()
|
|
79
|
-
foreach ($q in $Queries) { $Bodies += ,([IO.File]::ReadAllText($q.FullName, $Utf8) | ConvertFrom-Json) }
|
|
80
|
-
|
|
81
|
-
$needKey = ($Manifest.api_version -eq 'v2')
|
|
82
|
-
foreach ($b in $Bodies) { if ($b.PSObject.Properties['registrationkey']) { $needKey = $true } }
|
|
83
|
-
|
|
84
|
-
# ---------------------------------------------------------------------------- API key
|
|
85
|
-
$Key = $env:BLS_API_KEY
|
|
86
|
-
if ($Key) {
|
|
87
|
-
Write-Host 'Using the BLS key in $env:BLS_API_KEY.'
|
|
88
|
-
} elseif ($needKey) {
|
|
89
|
-
Write-Host 'These requests need a free BLS registration key (https://data.bls.gov/registrationEngine/).'
|
|
90
|
-
if ([Console]::IsInputRedirected) {
|
|
91
|
-
$Key = [Console]::In.ReadLine()
|
|
92
|
-
} else {
|
|
93
|
-
$secure = Read-Host -Prompt 'Paste your BLS key (input is hidden), then press Enter' -AsSecureString
|
|
94
|
-
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
|
95
|
-
try { $Key = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
|
|
96
|
-
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
|
|
97
|
-
Remove-Variable secure
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
if ($null -eq $Key) { $Key = '' }
|
|
101
|
-
$Key = $Key -replace '\s', ''
|
|
102
|
-
if ($needKey -and -not $Key) { Fail 'No key entered. Get one at https://data.bls.gov/registrationEngine/' }
|
|
103
|
-
|
|
104
|
-
$Version = 'v1'
|
|
105
|
-
if ($Key -and $Manifest.urls.v2) { $Version = 'v2' }
|
|
106
|
-
$Url = [string]$Manifest.urls.$Version
|
|
107
|
-
Write-Host "Sending $($Queries.Count) request(s) to BLS API $Version ..."
|
|
108
|
-
|
|
109
|
-
# ---------------------------------------------------------------------------- send each request
|
|
110
|
-
$Saved = @()
|
|
111
|
-
for ($i = 0; $i -lt $Queries.Count; $i++) {
|
|
112
|
-
$q = $Queries[$i]
|
|
113
|
-
$body = $Bodies[$i]
|
|
114
|
-
$body.PSObject.Properties.Remove('registrationkey')
|
|
115
|
-
if ($Version -eq 'v2' -and $Manifest.v2_options) {
|
|
116
|
-
foreach ($p in $Manifest.v2_options.PSObject.Properties) {
|
|
117
|
-
$body | Add-Member -NotePropertyName $p.Name -NotePropertyValue $p.Value -Force
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
$requestJson = $body | ConvertTo-Json -Depth 10 -Compress # the request without the key
|
|
121
|
-
$body | Add-Member -NotePropertyName 'registrationkey' -NotePropertyValue $Key -Force
|
|
122
|
-
if ($Version -ne 'v2') { $body.PSObject.Properties.Remove('registrationkey') }
|
|
123
|
-
$sendJson = $body | ConvertTo-Json -Depth 10 -Compress # in memory only
|
|
124
|
-
$body.PSObject.Properties.Remove('registrationkey')
|
|
125
|
-
|
|
126
|
-
try {
|
|
127
|
-
$resp = Invoke-WebRequest -Uri $Url -Method Post -Body $Utf8.GetBytes($sendJson) `
|
|
128
|
-
-ContentType 'application/json' -UserAgent 'data-analyst/1.0' -UseBasicParsing -TimeoutSec 120
|
|
129
|
-
} catch {
|
|
130
|
-
Fail "$($q.Name): the request to api.bls.gov failed ($($_.Exception.Message)). Check the internet connection, VPN and proxy, then run this script again."
|
|
131
|
-
}
|
|
132
|
-
$sendJson = $null
|
|
133
|
-
$text = $Utf8.GetString($resp.RawContentStream.ToArray()).Trim()
|
|
134
|
-
try { $reply = $text | ConvertFrom-Json } catch { $reply = $null }
|
|
135
|
-
if ($null -eq $reply -or -not $text.EndsWith('}') -or $reply.PSObject.Properties.Count -eq 0) {
|
|
136
|
-
Fail "$($q.Name): BLS did not answer with JSON. The reply began: $($text.Substring(0, [Math]::Min(200, $text.Length)))"
|
|
137
|
-
}
|
|
138
|
-
# Keep the reply exactly as BLS sent it and add the request it answers, like download_bls.sh.
|
|
139
|
-
$text = $text.Substring(0, $text.LastIndexOf('}')).TrimEnd() + ",`n `"_request`": " + $requestJson + "`n}"
|
|
140
|
-
$dest = Join-Path $Here ($q.Name -replace '^query_', 'response_')
|
|
141
|
-
[IO.File]::WriteAllText($dest, (Hide-Key $text) + "`n", $Utf8)
|
|
142
|
-
$Saved += $dest
|
|
143
|
-
$msg = Hide-Key ((@($reply.message) | Where-Object { $_ }) -join '; ')
|
|
144
|
-
if ($msg.Length -gt 200) { $msg = $msg.Substring(0, 200) }
|
|
145
|
-
Write-Host "$(Split-Path -Leaf $dest) $($reply.status) $msg"
|
|
146
|
-
if ($i -lt $Queries.Count - 1) { Start-Sleep -Milliseconds 500 }
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
Write-Host ''
|
|
150
|
-
Write-Host 'Done. Upload these files to the chat (do not edit them):' -ForegroundColor Green
|
|
151
|
-
foreach ($p in $Saved) { Write-Host " $p" }
|
|
152
|
-
Finish 0
|
|
1
|
+
<#
|
|
2
|
+
.SYNOPSIS
|
|
3
|
+
Download BLS time series with the BLS Public Data API: send the requests built into this script and
|
|
4
|
+
save the replies as response_*.json next to it. (Windows PowerShell counterpart of download_bls.sh.)
|
|
5
|
+
|
|
6
|
+
.DESCRIPTION
|
|
7
|
+
- Needs no other file: the requests are inside this script.
|
|
8
|
+
- When the requests need a BLS registration key (API v2), uses $env:BLS_API_KEY when it is set and
|
|
9
|
+
otherwise asks for the key (hidden input). The key is only sent to api.bls.gov: it is never shown,
|
|
10
|
+
saved, or put on a command line. With a key set, keyless requests also go through API v2.
|
|
11
|
+
- Saves response_01.json, response_02.json, ... next to this script, each with the request it answers
|
|
12
|
+
(without the key), and prints their full paths at the end.
|
|
13
|
+
- Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
|
|
14
|
+
|
|
15
|
+
Get a free key at https://data.bls.gov/registrationEngine/ (BLS emails it within minutes).
|
|
16
|
+
|
|
17
|
+
.PARAMETER NoPause
|
|
18
|
+
Do not wait for Enter before closing the window.
|
|
19
|
+
|
|
20
|
+
.EXAMPLE
|
|
21
|
+
powershell -NoProfile -ExecutionPolicy Bypass -File "$HOME\Downloads\download_bls.ps1"
|
|
22
|
+
#>
|
|
23
|
+
[CmdletBinding()]
|
|
24
|
+
param(
|
|
25
|
+
[switch]$NoPause
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
$ErrorActionPreference = 'Stop'
|
|
29
|
+
$ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
|
|
30
|
+
try {
|
|
31
|
+
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
|
32
|
+
} catch { }
|
|
33
|
+
|
|
34
|
+
$Here = $PSScriptRoot
|
|
35
|
+
if (-not $Here) { $Here = (Get-Location).Path }
|
|
36
|
+
$Utf8 = New-Object Text.UTF8Encoding($false)
|
|
37
|
+
$Key = $null
|
|
38
|
+
$Req = $null
|
|
39
|
+
|
|
40
|
+
# Keep the window open when the script was started with "Run with PowerShell".
|
|
41
|
+
function Finish([int]$Code) {
|
|
42
|
+
$script:Key = $null
|
|
43
|
+
if ($script:Req) { Remove-Item -LiteralPath $script:Req -Recurse -Force -ErrorAction SilentlyContinue }
|
|
44
|
+
if (-not $NoPause -and -not [Console]::IsInputRedirected) {
|
|
45
|
+
Write-Host ''
|
|
46
|
+
[void](Read-Host -Prompt 'Press Enter to close')
|
|
47
|
+
}
|
|
48
|
+
exit $Code
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function Fail([string]$Message) {
|
|
52
|
+
Write-Host ''
|
|
53
|
+
Write-Host "ERROR: $(Hide-Key $Message)" -ForegroundColor Red
|
|
54
|
+
Finish 1
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
# BLS repeats a rejected key in its error message: never let it reach the screen or a file.
|
|
58
|
+
function Hide-Key([string]$Text) {
|
|
59
|
+
if ($script:Key -and $Text) { return $Text.Replace($script:Key, '[REDACTED]') }
|
|
60
|
+
return $Text
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
# ---------------------------------------------------------------------------- the requests
|
|
64
|
+
# manifest.json and query_NN.json are built into this script; they go to a temporary folder that is
|
|
65
|
+
# removed when the script ends.
|
|
66
|
+
$Embedded = [ordered]@{}
|
|
67
|
+
# @QUERIES@
|
|
68
|
+
$Req = Join-Path ([IO.Path]::GetTempPath()) ('download_bls.' + [IO.Path]::GetRandomFileName())
|
|
69
|
+
[void](New-Item -ItemType Directory -Path $Req)
|
|
70
|
+
foreach ($name in $Embedded.Keys) { [IO.File]::WriteAllText((Join-Path $Req $name), $Embedded[$name], $Utf8) }
|
|
71
|
+
$manifestFile = Join-Path $Req 'manifest.json'
|
|
72
|
+
if (-not (Test-Path -LiteralPath $manifestFile)) {
|
|
73
|
+
Fail "This script has no requests built in. Use the download_bls.ps1 from the analyst's download files."
|
|
74
|
+
}
|
|
75
|
+
$Manifest = [IO.File]::ReadAllText($manifestFile, $Utf8) | ConvertFrom-Json
|
|
76
|
+
$Queries = @(Get-ChildItem -LiteralPath $Req -Filter 'query_*.json' | Sort-Object Name)
|
|
77
|
+
if ($Queries.Count -eq 0) { Fail "This script has no requests built in. Use the download_bls.ps1 from the analyst's download files." }
|
|
78
|
+
$Bodies = @()
|
|
79
|
+
foreach ($q in $Queries) { $Bodies += ,([IO.File]::ReadAllText($q.FullName, $Utf8) | ConvertFrom-Json) }
|
|
80
|
+
|
|
81
|
+
$needKey = ($Manifest.api_version -eq 'v2')
|
|
82
|
+
foreach ($b in $Bodies) { if ($b.PSObject.Properties['registrationkey']) { $needKey = $true } }
|
|
83
|
+
|
|
84
|
+
# ---------------------------------------------------------------------------- API key
|
|
85
|
+
$Key = $env:BLS_API_KEY
|
|
86
|
+
if ($Key) {
|
|
87
|
+
Write-Host 'Using the BLS key in $env:BLS_API_KEY.'
|
|
88
|
+
} elseif ($needKey) {
|
|
89
|
+
Write-Host 'These requests need a free BLS registration key (https://data.bls.gov/registrationEngine/).'
|
|
90
|
+
if ([Console]::IsInputRedirected) {
|
|
91
|
+
$Key = [Console]::In.ReadLine()
|
|
92
|
+
} else {
|
|
93
|
+
$secure = Read-Host -Prompt 'Paste your BLS key (input is hidden), then press Enter' -AsSecureString
|
|
94
|
+
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
|
95
|
+
try { $Key = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
|
|
96
|
+
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
|
|
97
|
+
Remove-Variable secure
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
if ($null -eq $Key) { $Key = '' }
|
|
101
|
+
$Key = $Key -replace '\s', ''
|
|
102
|
+
if ($needKey -and -not $Key) { Fail 'No key entered. Get one at https://data.bls.gov/registrationEngine/' }
|
|
103
|
+
|
|
104
|
+
$Version = 'v1'
|
|
105
|
+
if ($Key -and $Manifest.urls.v2) { $Version = 'v2' }
|
|
106
|
+
$Url = [string]$Manifest.urls.$Version
|
|
107
|
+
Write-Host "Sending $($Queries.Count) request(s) to BLS API $Version ..."
|
|
108
|
+
|
|
109
|
+
# ---------------------------------------------------------------------------- send each request
|
|
110
|
+
$Saved = @()
|
|
111
|
+
for ($i = 0; $i -lt $Queries.Count; $i++) {
|
|
112
|
+
$q = $Queries[$i]
|
|
113
|
+
$body = $Bodies[$i]
|
|
114
|
+
$body.PSObject.Properties.Remove('registrationkey')
|
|
115
|
+
if ($Version -eq 'v2' -and $Manifest.v2_options) {
|
|
116
|
+
foreach ($p in $Manifest.v2_options.PSObject.Properties) {
|
|
117
|
+
$body | Add-Member -NotePropertyName $p.Name -NotePropertyValue $p.Value -Force
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
$requestJson = $body | ConvertTo-Json -Depth 10 -Compress # the request without the key
|
|
121
|
+
$body | Add-Member -NotePropertyName 'registrationkey' -NotePropertyValue $Key -Force
|
|
122
|
+
if ($Version -ne 'v2') { $body.PSObject.Properties.Remove('registrationkey') }
|
|
123
|
+
$sendJson = $body | ConvertTo-Json -Depth 10 -Compress # in memory only
|
|
124
|
+
$body.PSObject.Properties.Remove('registrationkey')
|
|
125
|
+
|
|
126
|
+
try {
|
|
127
|
+
$resp = Invoke-WebRequest -Uri $Url -Method Post -Body $Utf8.GetBytes($sendJson) `
|
|
128
|
+
-ContentType 'application/json' -UserAgent 'data-analyst/1.0' -UseBasicParsing -TimeoutSec 120
|
|
129
|
+
} catch {
|
|
130
|
+
Fail "$($q.Name): the request to api.bls.gov failed ($($_.Exception.Message)). Check the internet connection, VPN and proxy, then run this script again."
|
|
131
|
+
}
|
|
132
|
+
$sendJson = $null
|
|
133
|
+
$text = $Utf8.GetString($resp.RawContentStream.ToArray()).Trim()
|
|
134
|
+
try { $reply = $text | ConvertFrom-Json } catch { $reply = $null }
|
|
135
|
+
if ($null -eq $reply -or -not $text.EndsWith('}') -or $reply.PSObject.Properties.Count -eq 0) {
|
|
136
|
+
Fail "$($q.Name): BLS did not answer with JSON. The reply began: $($text.Substring(0, [Math]::Min(200, $text.Length)))"
|
|
137
|
+
}
|
|
138
|
+
# Keep the reply exactly as BLS sent it and add the request it answers, like download_bls.sh.
|
|
139
|
+
$text = $text.Substring(0, $text.LastIndexOf('}')).TrimEnd() + ",`n `"_request`": " + $requestJson + "`n}"
|
|
140
|
+
$dest = Join-Path $Here ($q.Name -replace '^query_', 'response_')
|
|
141
|
+
[IO.File]::WriteAllText($dest, (Hide-Key $text) + "`n", $Utf8)
|
|
142
|
+
$Saved += $dest
|
|
143
|
+
$msg = Hide-Key ((@($reply.message) | Where-Object { $_ }) -join '; ')
|
|
144
|
+
if ($msg.Length -gt 200) { $msg = $msg.Substring(0, 200) }
|
|
145
|
+
Write-Host "$(Split-Path -Leaf $dest) $($reply.status) $msg"
|
|
146
|
+
if ($i -lt $Queries.Count - 1) { Start-Sleep -Milliseconds 500 }
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
Write-Host ''
|
|
150
|
+
Write-Host 'Done. Upload these files to the chat (do not edit them):' -ForegroundColor Green
|
|
151
|
+
foreach ($p in $Saved) { Write-Host " $p" }
|
|
152
|
+
Finish 0
|