data-analyst-mcp 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/README.md +262 -262
  2. package/dist/report/method_audit.js +7 -1
  3. package/dist/report/method_audit.js.map +1 -1
  4. package/dist/report/scan_secrets.d.ts +4 -1
  5. package/dist/report/scan_secrets.js +36 -5
  6. package/dist/report/scan_secrets.js.map +1 -1
  7. package/dist/server.d.ts +1 -1
  8. package/dist/server.js +3 -2
  9. package/dist/server.js.map +1 -1
  10. package/guide/WORKFLOW.md +134 -134
  11. package/guide/assets/disclaimer.txt +1 -1
  12. package/guide/assets/download_data.ps1 +162 -162
  13. package/guide/assets/report_template.html +597 -597
  14. package/guide/references/routing.md +171 -171
  15. package/guide/references/style_guide.md +398 -398
  16. package/package.json +56 -56
  17. package/packs/acf/endpoints.json +114 -114
  18. package/packs/acf/policy.json +104 -104
  19. package/packs/actuaries/endpoints.json +82 -82
  20. package/packs/actuaries/policy.json +109 -109
  21. package/packs/bea/assets/download_bea.ps1 +182 -182
  22. package/packs/bea/docs/bea_api.md +461 -461
  23. package/packs/bea/docs/datasets.md +157 -157
  24. package/packs/bea/docs/report_patterns.md +112 -112
  25. package/packs/bea/policy.json +170 -170
  26. package/packs/bea/source.md +152 -152
  27. package/packs/bls/assets/download_bls.ps1 +152 -152
  28. package/packs/bls/docs/analysis_patterns.md +108 -108
  29. package/packs/bls/docs/bls_api.md +73 -73
  30. package/packs/bls/docs/series_ids.md +149 -149
  31. package/packs/bls/policy.json +238 -238
  32. package/packs/bts/endpoints.json +346 -346
  33. package/packs/bts/policy.json +94 -94
  34. package/packs/cdc/endpoints.json +197 -197
  35. package/packs/cdc/policy.json +83 -83
  36. package/packs/census/endpoints.json +99 -99
  37. package/packs/cfpb/endpoints.json +326 -326
  38. package/packs/cfpb/policy.json +119 -119
  39. package/packs/clinical-trials/endpoints.json +166 -166
  40. package/packs/clinical-trials/policy.json +84 -84
  41. package/packs/cms/endpoints.json +69 -69
  42. package/packs/cms/policy.json +96 -96
  43. package/packs/college-scorecard/endpoints.json +252 -252
  44. package/packs/college-scorecard/policy.json +106 -106
  45. package/packs/congress/endpoints.json +1701 -1701
  46. package/packs/congress/policy.json +117 -117
  47. package/packs/dhs-ohss/endpoints.json +66 -66
  48. package/packs/dhs-ohss/policy.json +107 -107
  49. package/packs/doj-news/endpoints.json +59 -59
  50. package/packs/doj-news/policy.json +96 -96
  51. package/packs/dol/endpoints.json +817 -817
  52. package/packs/dol/policy.json +135 -135
  53. package/packs/eia/endpoints.json +717 -717
  54. package/packs/eia/policy.json +112 -112
  55. package/packs/epa/endpoints.json +291 -291
  56. package/packs/epa/policy.json +99 -99
  57. package/packs/epa-aqs/endpoints.json +173 -173
  58. package/packs/epa-aqs/policy.json +96 -96
  59. package/packs/fbi/endpoints.json +1841 -1841
  60. package/packs/fbi/policy.json +118 -118
  61. package/packs/fda/endpoints.json +237 -237
  62. package/packs/fda/policy.json +121 -121
  63. package/packs/fdic/endpoints.json +416 -416
  64. package/packs/fdic/policy.json +83 -83
  65. package/packs/fec/endpoints.json +707 -707
  66. package/packs/fec/policy.json +93 -93
  67. package/packs/fed-board/endpoints.json +89 -89
  68. package/packs/fed-board/policy.json +136 -136
  69. package/packs/federal-register/endpoints.json +143 -143
  70. package/packs/federal-register/policy.json +86 -86
  71. package/packs/fema/endpoints.json +645 -645
  72. package/packs/fema/policy.json +103 -103
  73. package/packs/fhfa/endpoints.json +105 -105
  74. package/packs/fhfa/policy.json +109 -109
  75. package/packs/fred/assets/download_fred.ps1 +122 -122
  76. package/packs/fred/docs/analysis_patterns.md +124 -124
  77. package/packs/fred/docs/fred_api.md +94 -94
  78. package/packs/fred/docs/series.md +188 -188
  79. package/packs/fred/policy.json +253 -253
  80. package/packs/fsa/endpoints.json +150 -150
  81. package/packs/fsa/policy.json +99 -99
  82. package/packs/govinfo/endpoints.json +519 -519
  83. package/packs/gsa-calc/endpoints.json +93 -93
  84. package/packs/gsa-calc/policy.json +97 -97
  85. package/packs/hud/endpoints.json +146 -146
  86. package/packs/hud/policy.json +84 -84
  87. package/packs/ipums/docs/report_patterns.md +117 -117
  88. package/packs/irs-soi/endpoints.json +114 -114
  89. package/packs/irs-soi/policy.json +107 -107
  90. package/packs/medicaid/endpoints.json +412 -412
  91. package/packs/medicaid/policy.json +108 -108
  92. package/packs/naep/endpoints.json +217 -217
  93. package/packs/naep/policy.json +100 -100
  94. package/packs/nhtsa/endpoints.json +132 -132
  95. package/packs/nhtsa/policy.json +93 -93
  96. package/packs/nih/endpoints.json +452 -452
  97. package/packs/nih/policy.json +86 -86
  98. package/packs/noaa/endpoints.json +431 -431
  99. package/packs/noaa/policy.json +95 -95
  100. package/packs/nrel/endpoints.json +327 -327
  101. package/packs/nrel/policy.json +96 -96
  102. package/packs/nws/endpoints.json +271 -271
  103. package/packs/nws/policy.json +82 -82
  104. package/packs/nyfed/endpoints.json +80 -80
  105. package/packs/nyfed/policy.json +92 -92
  106. package/packs/omb/endpoints.json +460 -460
  107. package/packs/omb/policy.json +106 -106
  108. package/packs/open-payments/endpoints.json +297 -297
  109. package/packs/open-payments/policy.json +101 -101
  110. package/packs/opm/assets/download_opm.ps1 +125 -125
  111. package/packs/opm/docs/fields.md +184 -184
  112. package/packs/opm/docs/opm_api.md +71 -71
  113. package/packs/opm/docs/report_patterns.md +83 -83
  114. package/packs/opm/lib/data_dictionary.csv +73 -73
  115. package/packs/pbgc/endpoints.json +483 -483
  116. package/packs/pbgc/policy.json +94 -94
  117. package/packs/philfed/endpoints.json +85 -85
  118. package/packs/philfed/policy.json +94 -94
  119. package/packs/regulations/endpoints.json +287 -287
  120. package/packs/regulations/policy.json +86 -86
  121. package/packs/sec/endpoints.json +236 -236
  122. package/packs/sec/policy.json +86 -86
  123. package/packs/senate-lobbying/endpoints.json +553 -553
  124. package/packs/senate-lobbying/policy.json +99 -99
  125. package/packs/snapqc/endpoints.json +1426 -1426
  126. package/packs/snapqc/policy.json +92 -92
  127. package/packs/treasury/catalog.json +2267 -2267
  128. package/packs/treasury/docs/api_guide.md +245 -245
  129. package/packs/treasury/docs/datasets.md +56 -56
  130. package/packs/treasury/docs/report_patterns.md +126 -126
  131. package/packs/treasury/policy.json +159 -159
  132. package/packs/treasury/source.md +170 -165
  133. package/packs/treasury-yields/endpoints.json +226 -226
  134. package/packs/treasury-yields/policy.json +86 -86
  135. package/packs/usaspending/endpoints.json +1315 -1315
  136. package/packs/usaspending/policy.json +109 -109
  137. package/packs/usda-fna/policy.json +133 -133
  138. package/packs/usda-fooddata/endpoints.json +180 -180
  139. package/packs/usda-fooddata/policy.json +94 -94
  140. package/packs/usda-nass/endpoints.json +323 -323
  141. package/packs/usda-nass/policy.json +95 -95
  142. package/packs/usda-outlook/endpoints.json +86 -86
  143. package/packs/usda-outlook/policy.json +109 -109
  144. package/packs/usgs/endpoints.json +313 -313
  145. package/packs/usgs/policy.json +95 -95
  146. package/packs/usitc/endpoints.json +74 -74
  147. package/packs/usitc/policy.json +111 -111
  148. package/packs/uspto/endpoints.json +141 -141
  149. package/packs/uspto/policy.json +95 -95
  150. package/packs/va/endpoints.json +414 -414
  151. package/packs/va/policy.json +101 -101
  152. package/packs/world-bank/endpoints.json +122 -122
  153. package/packs/world-bank/policy.json +101 -101
@@ -1,152 +1,152 @@
1
- # Bureau of Economic Analysis (BEA) Data API and bulk files
2
- Depth: deep. Key: required (BEA_API_KEY) for the API; the bulk route needs none. U.S. national accounts
3
- (GDP, income, PCE and its prices, saving, profits, government), state and county GDP and income, regional
4
- price parities, GDP by industry, trade and international accounts, multinationals, fixed assets, input-output.
5
-
6
- ## When to use
7
- - GDP and its growth, contributions, prices and deflators; personal income, disposable income, saving rate;
8
- PCE and core PCE price indexes (the Federal Reserve's inflation measure); corporate profits; NIPA
9
- government receipts and spending (calendar years); GDP and personal income by state and county; per capita
10
- income; RPPs; value added and gross output by industry; goods and services trade balances by country;
11
- current account; investment position; direct investment; capital stocks.
12
- - Not here: CPI, jobs, unemployment, wages by occupation (bls); budget data by fiscal year (treasury);
13
- market interest rates (treasury-yields; fred); forecasts (BEA publishes history only); monthly GDP.
14
- - The deliverable decides the work: data only (download, short preview, stop), an answer (compute with
15
- `compute`, answer with the numbers, measure and vintage), or a report (`guide`'s report steps).
16
-
17
- ## Endpoints and datasets
18
- One API endpoint (`https://apps.bea.gov/api/data`, GET, key in `UserID`), 13 datasets. Bulk files serve
19
- the most used ones without a key and count against no limit.
20
-
21
- | `dataset` | What | Spec keys | Bulk (no key)? |
22
- |---|---|---|---|
23
- | `NIPA` | national accounts tables `T10101` ... (titles: `request_schema` search) | `table`, `frequency` A/Q/M, `years`, `line_code` | yes (NIPA flat files) |
24
- | `NIUnderlyingDetail` | detail tables `U20305` ... (lower quality) | same | no (flat files hold only shared lines) |
25
- | `FixedAssets` | `FAAt101` ... annual | `table`, `years` | no |
26
- | `Regional` | state `SA*`/`SQ*`, county `CA*`, metro `MA*` tables | `table`, `line_code`, `geo_fips`, `years` | yes for STATE, COUNTY, postal code (its counties), FIPS lists, MARPP |
27
- | `GDPbyIndustry` | workbook codes `TVA101`-`TVA113`, `TGO1xx`, `TII1xx` | `table`, `frequency` A/Q, `years` | yes (public workbooks) |
28
- | `GDPbyIndustry` (API) | `TableID` 1, 5, 6, 25 ...; `Industry` filters | `params` | no |
29
- | `UnderlyingGDPbyIndustry`, `ITA`, `IIP`, `IntlServTrade`, `IntlServSTA`, `MNE`, `InputOutput` | see `request_schema` with `names: ["<name>"]` | `params` (raw API names) | no |
30
-
31
- Verified lines (September 2026 vintage): T10101 line 1 real GDP percent change (annual rate); T10106 line 1
32
- real GDP (millions of chained 2017 dollars, SAAR); T10105 line 1 nominal GDP; T10111 line 1 year-over-year
33
- real GDP change; T10102 contributions; T20100 lines 1/27/37/35/40 personal income / DPI / real DPI / saving
34
- rate / population (annual and quarterly only: monthly values are in T20600, whose line numbers differ, so
35
- check them with the search `lines T20600`; a table is monthly only when its title says so, and the bulk route refuses
36
- frequency M for any other table); T20804 lines 1/25 PCE / core PCE price index (monthly; published changes: T20807 month over month, not
37
- annualized, T20811 lines 32/37 over 12 months); T70100 lines 1/3 GDP and income
38
- per capita (BEA divides by unrounded population: real GDP over T70100's population, published in thousands,
39
- can miss the published per-capita level by just over 0.5 dollars; allow for that rounding in the benchmark and
40
- say so). Federal civilian wages per FTE (T60600D) jump in 2024-2025: Deferred Resignation staff on paid
41
- administrative leave stayed on the payroll while BEA cut the quantity of federal labor (FAQ 1479,
42
- bea.gov/help/faq/1479): never read those years as federal pay raises; use OPM pay or the GS raise. Regional: SAGDP2/SAGDP9 line 1 all-industry total (current / chained 2017 dollars); SAINC1 and
43
- CAINC1 lines 1/2/3 personal income / population / per capita income; SAINC4/CAINC4 lines 10/20/30; SARPP
44
- and MARPP line 1 all-items RPP. API names may carry an `N` (`SAGDP9N`); the bulk files drop it: both work.
45
- BEA marks nothing as preliminary in the data, but the latest quarter or year of every table is an early
46
- estimate that BEA will revise: register facts from it with `preliminary: true` in `facts` and say so. A year that has
47
- been through an annual update (published at the end of September) is revised, not early: say "subject to
48
- revision at the next annual update" instead.
49
- More tables, the question-to-table map and concepts: `docs/datasets.md`. API parameters, response quirks,
50
- error codes and limits: `docs/bea_api.md`. Report conventions: `docs/report_patterns.md`.
51
-
52
- Discovery with `request_schema` (no key unless noted): `search: "real gross domestic product"` (NIPA titles
53
- and workbook tables), `search: "Regional income"` (the first word names the dataset; API datasets need a key),
54
- `search: "lines T10106"` (lines and series codes), `names: ["ITA"]` (required parameters, rules, spec keys,
55
- and with a key BEA's current parameters); with a key (cached a day): `search: "values Regional TableName"`,
56
- `search: "values Regional LineCode TableName=SAINC1"`. `source_brief` lists the 13 datasets.
57
-
58
- ## Spec examples
59
- ```json
60
- {"description": "Real GDP growth, national and by state, 2019 to 2025",
61
- "requests": [
62
- {"id": "gdp_pct", "dataset": "NIPA", "table": "T10101", "line_code": 1, "frequency": "Q", "years": "2019-2025"},
63
- {"id": "gdp_real", "dataset": "NIPA", "table": "T10106", "line_code": 1, "frequency": "Q", "years": "2019-2025"},
64
- {"id": "state_real", "dataset": "Regional", "table": "SAGDP9", "line_code": 1, "geo_fips": "STATE", "years": "2019-2025"},
65
- {"id": "va_ind", "dataset": "GDPbyIndustry", "table": "TVA105", "frequency": "A", "years": "2024-2025"}]}
66
- ```
67
- ```json
68
- {"description": "Per capita personal income, two counties, last five years",
69
- "requests": [{"id": "cty_pi", "dataset": "Regional", "table": "CAINC1", "line_code": [1, 3],
70
- "geo_fips": ["06037", "17031"], "years": "LAST5"}]}
71
- ```
72
- ```json
73
- {"description": "U.S. goods trade balance with China (needs a key)",
74
- "requests": [{"id": "ita", "dataset": "ITA",
75
- "params": {"Indicator": "BalGds", "AreaOrCountry": "China", "Frequency": "A", "Year": "2023,2024"}}]}
76
- ```
77
- `years`: `"2019-2025"`, a list, `"ALL"` or `"LAST5"`; `line_code`: a number, list or `"ALL"` (NIPA: BEA
78
- sends the whole table and the rows are filtered); `geo_fips`: `STATE`, `COUNTY`, a postal code (its
79
- counties), FIPS codes (`"06"` = `06000`). Keys a dataset does not use are reported in the result's notes. When a growth
80
- rate is computed from levels, also request BEA's published percent change so it can be checked.
81
-
82
- ## Data format and traps
83
- `fetch` writes `data/bea/`: `bea_data.csv` (tidy: `request_id, dataset, table, line, series_code,
84
- description, geo_fips, geo_name, geo_level, industry, time_period, date, year, subperiod, frequency, value,
85
- flag, unit, metric, unit_mult, note_ref, dims`), `notes.csv` (BEA's footnotes: units, reference year, what
86
- the U.S. row means, flag codes), `request.json` (calls without the key, bulk files, route, vintage,
87
- checksums) and `README.md`. Read the README and `notes.csv` before analysing (the NIPA bulk files carry no
88
- footnotes: on the `bulk` route `notes.csv` has only the pack's own note; read a table's footnotes on its
89
- apps.bea.gov page, or fetch it through the API with a key). `line` holds BEA's line numbers, which can carry
90
- letters: match lines exactly (as text, or with `series_code`), never by integer order.
91
- - **Units:** `value` is as BEA prints it; base units = value x 10^`unit_mult` (`compute` `derive`:
92
- `value * pow(10, unit_mult)`).
93
- Check `unit` and `metric` (current dollars, chained dollars, index, percent change) for every series.
94
- - **Special values** `(D)` suppressed, `(NA)` not available, `(NM)` not meaningful, `(L)` under $50,000 or 10
95
- jobs, `(S)`, `(X)`, `(*)` (MNE: rounds to zero): `value` is empty and the code is in `flag`. Never a zero;
96
- list the rows with a `flag` and drop or footnote the area.
97
- - **No sums of chained dollars**, of components or of states; shares from current dollars; drivers of growth
98
- from BEA's contributions tables (T10102, SAGDP11). SAAR levels are averaged, not added, for annual values.
99
- - **Growth measures differ** by a point or more: quarterly at a compound annual rate (BEA's headline:
100
- (level / previous level)^4 - 1), year over year (T10111), Q4 over Q4, annual average; "since <year>":
101
- cumulative change and the average annual rate. Name the measure everywhere; show the alternatives as a
102
- sensitivity.
103
- - **Check against BEA's published numbers** (docs/datasets.md section 5): growth
104
- from T10106 levels vs T10101; the U.S. row of SAGDP9 vs T10106 line 1. A mismatch beyond rounding is an
105
- error; a difference from an older release is a revision (compare dates with the vintage in README.md).
106
- - **Vintage:** BEA serves only its latest estimates (advance, second, third; annual updates each September;
107
- comprehensive updates every five years). Record the release date; old vintages are at apps.bea.gov/histdata.
108
- - **Geography:** every row has `geo_level` (nation, region, state, county, msa); a COUNTY bulk download also
109
- holds state and U.S. rows: filter before summing. `STATE` = 50 states + D.C. + U.S. (say whether D.C. is
110
- ranked). Since February 2026 BEA no longer produces GDP or personal income for metro areas: sum component
111
- counties' current-dollar values and say so. Rank on unrounded values; name near-ties.
112
- - **Calendar years** everywhere (NIPA federal spending too, unlike fiscal-year budget data). NIPA line
113
- numbers change at comprehensive updates: check the description or series code, not only the line.
114
- GDP-by-industry line numbers differ between tables (TVA106 has an extra line): match by description.
115
- - **No standard errors** exist for BEA estimates: never "statistically different".
116
- - Analysis: build growth rates, shares and comparisons with `compute` on `data/bea/bea_data.csv`, and check each
117
- headline against the published table.
118
-
119
- ## Access
120
- - **Key** `BEA_API_KEY` (free, https://apps.bea.gov/API/signup/; BEA emails it and it works only after the
121
- activation link is clicked). Found, in order, in the environment, `BEA_API_KEY_FILE`, keys.json, Claude Code
122
- settings.json and `~/.bea/api_key`; the user stores it with `npx -y data-analyst-mcp setup bea` in their own
123
- terminal. Never ask for the key in chat. BEA puts it in the URL and echoes it in every
124
- reply: the server scrubs both (the key and the USERID echo); never build your own API URLs.
125
- - **Limits** per key: 100 requests, 100 MB and 30 errors a minute; going over blocks the key for a minute to
126
- an hour (HTTP 429, never retried). The client spaces calls 0.75 s, waits before crossing 100 MB a minute,
127
- stops after 30 errors a minute or 10 in a run, validates table and line names against cached metadata
128
- first, and checks dataset parameter rules offline (`invalid_request`), so errors are not spent.
129
- - **Routes** (`route` in the result): `reuse_existing` (`data/bea/` already holds the request; `force: true`
130
- downloads again), `api` (key works), `bulk` (no usable key and every request is in the bulk files: NIPA
131
- flat files, state and county zips, GDP-by-industry workbooks; same latest estimates; the spec's
132
- `"prefer": "bulk"` forces it, `"prefer": "api"` never falls back). A key BEA rejects or has locked out falls
133
- back to the bulk files when they hold every request. Errors: `key_missing` (no key and a request needs the
134
- API: the user runs `npx -y data-analyst-mcp setup bea`, or `manual_bundle`), `key_rejected`,
135
- `rate_limited`, `agency_rejected` (fix the spec; never loop: errors count against the key) and
136
- `agency_unavailable` (BEA unreachable; never invent or recall data). Bulk files are cached in
137
- `<cache_root>/bea/bulk` (`DATA_ANALYST_CACHE` sets the root) and revalidated by Last-Modified; NIPA files are
138
- 12-37 MB, county zips up to 50 MB.
139
- - **Manual download:** `manual_bundle` writes `manual_download/`: `INSTRUCTIONS.md` with the links, and
140
- `download_data.sh` (macOS, Linux) and `download_data.ps1` (Windows), one-command scripts that download the
141
- public files and ask for the key with hidden input only when an API request needs it. Send both scripts and
142
- INSTRUCTIONS.md, and give the one command for the user's system (`user_commands`). BEA repeats the key inside
143
- API replies: `ingest` removes it. Then `ingest` with the uploaded files or their folder (on a hosted server,
144
- `files: ["uploads"]`). Say in the methods that the files were downloaded by hand.
145
-
146
- ## Citation and terms
147
- "U.S. Bureau of Economic Analysis, <tables>, <route> (accessed <date>)." The exact line is in
148
- `data/bea/README.md`; include the release (vintage) when it matters. Source line: "Analyst calculations using
149
- data from the Bureau of Economic Analysis." BEA data are public domain: redistribution allowed (data files
150
- may ship in a replication package). Products that use the API must show: "This product uses the Bureau of
151
- Economic Analysis (BEA) Data API but is not endorsed or certified by BEA." (the API terms' wording) Do not
152
- imply BEA endorsement.
1
+ # Bureau of Economic Analysis (BEA) Data API and bulk files
2
+ Depth: deep. Key: required (BEA_API_KEY) for the API; the bulk route needs none. U.S. national accounts
3
+ (GDP, income, PCE and its prices, saving, profits, government), state and county GDP and income, regional
4
+ price parities, GDP by industry, trade and international accounts, multinationals, fixed assets, input-output.
5
+
6
+ ## When to use
7
+ - GDP and its growth, contributions, prices and deflators; personal income, disposable income, saving rate;
8
+ PCE and core PCE price indexes (the Federal Reserve's inflation measure); corporate profits; NIPA
9
+ government receipts and spending (calendar years); GDP and personal income by state and county; per capita
10
+ income; RPPs; value added and gross output by industry; goods and services trade balances by country;
11
+ current account; investment position; direct investment; capital stocks.
12
+ - Not here: CPI, jobs, unemployment, wages by occupation (bls); budget data by fiscal year (treasury);
13
+ market interest rates (treasury-yields; fred); forecasts (BEA publishes history only); monthly GDP.
14
+ - The deliverable decides the work: data only (download, short preview, stop), an answer (compute with
15
+ `compute`, answer with the numbers, measure and vintage), or a report (`guide`'s report steps).
16
+
17
+ ## Endpoints and datasets
18
+ One API endpoint (`https://apps.bea.gov/api/data`, GET, key in `UserID`), 13 datasets. Bulk files serve
19
+ the most used ones without a key and count against no limit.
20
+
21
+ | `dataset` | What | Spec keys | Bulk (no key)? |
22
+ |---|---|---|---|
23
+ | `NIPA` | national accounts tables `T10101` ... (titles: `request_schema` search) | `table`, `frequency` A/Q/M, `years`, `line_code` | yes (NIPA flat files) |
24
+ | `NIUnderlyingDetail` | detail tables `U20305` ... (lower quality) | same | no (flat files hold only shared lines) |
25
+ | `FixedAssets` | `FAAt101` ... annual | `table`, `years` | no |
26
+ | `Regional` | state `SA*`/`SQ*`, county `CA*`, metro `MA*` tables | `table`, `line_code`, `geo_fips`, `years` | yes for STATE, COUNTY, postal code (its counties), FIPS lists, MARPP |
27
+ | `GDPbyIndustry` | workbook codes `TVA101`-`TVA113`, `TGO1xx`, `TII1xx` | `table`, `frequency` A/Q, `years` | yes (public workbooks) |
28
+ | `GDPbyIndustry` (API) | `TableID` 1, 5, 6, 25 ...; `Industry` filters | `params` | no |
29
+ | `UnderlyingGDPbyIndustry`, `ITA`, `IIP`, `IntlServTrade`, `IntlServSTA`, `MNE`, `InputOutput` | see `request_schema` with `names: ["<name>"]` | `params` (raw API names) | no |
30
+
31
+ Verified lines (September 2026 vintage): T10101 line 1 real GDP percent change (annual rate); T10106 line 1
32
+ real GDP (millions of chained 2017 dollars, SAAR); T10105 line 1 nominal GDP; T10111 line 1 year-over-year
33
+ real GDP change; T10102 contributions; T20100 lines 1/27/37/35/40 personal income / DPI / real DPI / saving
34
+ rate / population (annual and quarterly only: monthly values are in T20600, whose line numbers differ, so
35
+ check them with the search `lines T20600`; a table is monthly only when its title says so, and the bulk route refuses
36
+ frequency M for any other table); T20804 lines 1/25 PCE / core PCE price index (monthly; published changes: T20807 month over month, not
37
+ annualized, T20811 lines 32/37 over 12 months); T70100 lines 1/3 GDP and income
38
+ per capita (BEA divides by unrounded population: real GDP over T70100's population, published in thousands,
39
+ can miss the published per-capita level by just over 0.5 dollars; allow for that rounding in the benchmark and
40
+ say so). Federal civilian wages per FTE (T60600D) jump in 2024-2025: Deferred Resignation staff on paid
41
+ administrative leave stayed on the payroll while BEA cut the quantity of federal labor (FAQ 1479,
42
+ bea.gov/help/faq/1479): never read those years as federal pay raises; use OPM pay or the GS raise. Regional: SAGDP2/SAGDP9 line 1 all-industry total (current / chained 2017 dollars); SAINC1 and
43
+ CAINC1 lines 1/2/3 personal income / population / per capita income; SAINC4/CAINC4 lines 10/20/30; SARPP
44
+ and MARPP line 1 all-items RPP. API names may carry an `N` (`SAGDP9N`); the bulk files drop it: both work.
45
+ BEA marks nothing as preliminary in the data, but the latest quarter or year of every table is an early
46
+ estimate that BEA will revise: register facts from it with `preliminary: true` in `facts` and say so. A year that has
47
+ been through an annual update (published at the end of September) is revised, not early: say "subject to
48
+ revision at the next annual update" instead.
49
+ More tables, the question-to-table map and concepts: `docs/datasets.md`. API parameters, response quirks,
50
+ error codes and limits: `docs/bea_api.md`. Report conventions: `docs/report_patterns.md`.
51
+
52
+ Discovery with `request_schema` (no key unless noted): `search: "real gross domestic product"` (NIPA titles
53
+ and workbook tables), `search: "Regional income"` (the first word names the dataset; API datasets need a key),
54
+ `search: "lines T10106"` (lines and series codes), `names: ["ITA"]` (required parameters, rules, spec keys,
55
+ and with a key BEA's current parameters); with a key (cached a day): `search: "values Regional TableName"`,
56
+ `search: "values Regional LineCode TableName=SAINC1"`. `source_brief` lists the 13 datasets.
57
+
58
+ ## Spec examples
59
+ ```json
60
+ {"description": "Real GDP growth, national and by state, 2019 to 2025",
61
+ "requests": [
62
+ {"id": "gdp_pct", "dataset": "NIPA", "table": "T10101", "line_code": 1, "frequency": "Q", "years": "2019-2025"},
63
+ {"id": "gdp_real", "dataset": "NIPA", "table": "T10106", "line_code": 1, "frequency": "Q", "years": "2019-2025"},
64
+ {"id": "state_real", "dataset": "Regional", "table": "SAGDP9", "line_code": 1, "geo_fips": "STATE", "years": "2019-2025"},
65
+ {"id": "va_ind", "dataset": "GDPbyIndustry", "table": "TVA105", "frequency": "A", "years": "2024-2025"}]}
66
+ ```
67
+ ```json
68
+ {"description": "Per capita personal income, two counties, last five years",
69
+ "requests": [{"id": "cty_pi", "dataset": "Regional", "table": "CAINC1", "line_code": [1, 3],
70
+ "geo_fips": ["06037", "17031"], "years": "LAST5"}]}
71
+ ```
72
+ ```json
73
+ {"description": "U.S. goods trade balance with China (needs a key)",
74
+ "requests": [{"id": "ita", "dataset": "ITA",
75
+ "params": {"Indicator": "BalGds", "AreaOrCountry": "China", "Frequency": "A", "Year": "2023,2024"}}]}
76
+ ```
77
+ `years`: `"2019-2025"`, a list, `"ALL"` or `"LAST5"`; `line_code`: a number, list or `"ALL"` (NIPA: BEA
78
+ sends the whole table and the rows are filtered); `geo_fips`: `STATE`, `COUNTY`, a postal code (its
79
+ counties), FIPS codes (`"06"` = `06000`). Keys a dataset does not use are reported in the result's notes. When a growth
80
+ rate is computed from levels, also request BEA's published percent change so it can be checked.
81
+
82
+ ## Data format and traps
83
+ `fetch` writes `data/bea/`: `bea_data.csv` (tidy: `request_id, dataset, table, line, series_code,
84
+ description, geo_fips, geo_name, geo_level, industry, time_period, date, year, subperiod, frequency, value,
85
+ flag, unit, metric, unit_mult, note_ref, dims`), `notes.csv` (BEA's footnotes: units, reference year, what
86
+ the U.S. row means, flag codes), `request.json` (calls without the key, bulk files, route, vintage,
87
+ checksums) and `README.md`. Read the README and `notes.csv` before analysing (the NIPA bulk files carry no
88
+ footnotes: on the `bulk` route `notes.csv` has only the pack's own note; read a table's footnotes on its
89
+ apps.bea.gov page, or fetch it through the API with a key). `line` holds BEA's line numbers, which can carry
90
+ letters: match lines exactly (as text, or with `series_code`), never by integer order.
91
+ - **Units:** `value` is as BEA prints it; base units = value x 10^`unit_mult` (`compute` `derive`:
92
+ `value * pow(10, unit_mult)`).
93
+ Check `unit` and `metric` (current dollars, chained dollars, index, percent change) for every series.
94
+ - **Special values** `(D)` suppressed, `(NA)` not available, `(NM)` not meaningful, `(L)` under $50,000 or 10
95
+ jobs, `(S)`, `(X)`, `(*)` (MNE: rounds to zero): `value` is empty and the code is in `flag`. Never a zero;
96
+ list the rows with a `flag` and drop or footnote the area.
97
+ - **No sums of chained dollars**, of components or of states; shares from current dollars; drivers of growth
98
+ from BEA's contributions tables (T10102, SAGDP11). SAAR levels are averaged, not added, for annual values.
99
+ - **Growth measures differ** by a point or more: quarterly at a compound annual rate (BEA's headline:
100
+ (level / previous level)^4 - 1), year over year (T10111), Q4 over Q4, annual average; "since <year>":
101
+ cumulative change and the average annual rate. Name the measure everywhere; show the alternatives as a
102
+ sensitivity.
103
+ - **Check against BEA's published numbers** (docs/datasets.md section 5): growth
104
+ from T10106 levels vs T10101; the U.S. row of SAGDP9 vs T10106 line 1. A mismatch beyond rounding is an
105
+ error; a difference from an older release is a revision (compare dates with the vintage in README.md).
106
+ - **Vintage:** BEA serves only its latest estimates (advance, second, third; annual updates each September;
107
+ comprehensive updates every five years). Record the release date; old vintages are at apps.bea.gov/histdata.
108
+ - **Geography:** every row has `geo_level` (nation, region, state, county, msa); a COUNTY bulk download also
109
+ holds state and U.S. rows: filter before summing. `STATE` = 50 states + D.C. + U.S. (say whether D.C. is
110
+ ranked). Since February 2026 BEA no longer produces GDP or personal income for metro areas: sum component
111
+ counties' current-dollar values and say so. Rank on unrounded values; name near-ties.
112
+ - **Calendar years** everywhere (NIPA federal spending too, unlike fiscal-year budget data). NIPA line
113
+ numbers change at comprehensive updates: check the description or series code, not only the line.
114
+ GDP-by-industry line numbers differ between tables (TVA106 has an extra line): match by description.
115
+ - **No standard errors** exist for BEA estimates: never "statistically different".
116
+ - Analysis: build growth rates, shares and comparisons with `compute` on `data/bea/bea_data.csv`, and check each
117
+ headline against the published table.
118
+
119
+ ## Access
120
+ - **Key** `BEA_API_KEY` (free, https://apps.bea.gov/API/signup/; BEA emails it and it works only after the
121
+ activation link is clicked). Found, in order, in the environment, `BEA_API_KEY_FILE`, keys.json, Claude Code
122
+ settings.json and `~/.bea/api_key`; the user stores it with `npx -y data-analyst-mcp setup bea` in their own
123
+ terminal. Never ask for the key in chat. BEA puts it in the URL and echoes it in every
124
+ reply: the server scrubs both (the key and the USERID echo); never build your own API URLs.
125
+ - **Limits** per key: 100 requests, 100 MB and 30 errors a minute; going over blocks the key for a minute to
126
+ an hour (HTTP 429, never retried). The client spaces calls 0.75 s, waits before crossing 100 MB a minute,
127
+ stops after 30 errors a minute or 10 in a run, validates table and line names against cached metadata
128
+ first, and checks dataset parameter rules offline (`invalid_request`), so errors are not spent.
129
+ - **Routes** (`route` in the result): `reuse_existing` (`data/bea/` already holds the request; `force: true`
130
+ downloads again), `api` (key works), `bulk` (no usable key and every request is in the bulk files: NIPA
131
+ flat files, state and county zips, GDP-by-industry workbooks; same latest estimates; the spec's
132
+ `"prefer": "bulk"` forces it, `"prefer": "api"` never falls back). A key BEA rejects or has locked out falls
133
+ back to the bulk files when they hold every request. Errors: `key_missing` (no key and a request needs the
134
+ API: the user runs `npx -y data-analyst-mcp setup bea`, or `manual_bundle`), `key_rejected`,
135
+ `rate_limited`, `agency_rejected` (fix the spec; never loop: errors count against the key) and
136
+ `agency_unavailable` (BEA unreachable; never invent or recall data). Bulk files are cached in
137
+ `<cache_root>/bea/bulk` (`DATA_ANALYST_CACHE` sets the root) and revalidated by Last-Modified; NIPA files are
138
+ 12-37 MB, county zips up to 50 MB.
139
+ - **Manual download:** `manual_bundle` writes `manual_download/`: `INSTRUCTIONS.md` with the links, and
140
+ `download_data.sh` (macOS, Linux) and `download_data.ps1` (Windows), one-command scripts that download the
141
+ public files and ask for the key with hidden input only when an API request needs it. Send both scripts and
142
+ INSTRUCTIONS.md, and give the one command for the user's system (`user_commands`). BEA repeats the key inside
143
+ API replies: `ingest` removes it. Then `ingest` with the uploaded files or their folder (on a hosted server,
144
+ `files: ["uploads"]`). Say in the methods that the files were downloaded by hand.
145
+
146
+ ## Citation and terms
147
+ "U.S. Bureau of Economic Analysis, <tables>, <route> (accessed <date>)." The exact line is in
148
+ `data/bea/README.md`; include the release (vintage) when it matters. Source line: "Analyst calculations using
149
+ data from the Bureau of Economic Analysis." BEA data are public domain: redistribution allowed (data files
150
+ may ship in a replication package). Products that use the API must show: "This product uses the Bureau of
151
+ Economic Analysis (BEA) Data API but is not endorsed or certified by BEA." (the API terms' wording) Do not
152
+ imply BEA endorsement.
@@ -1,152 +1,152 @@
1
- <#
2
- .SYNOPSIS
3
- Download BLS time series with the BLS Public Data API: send the requests built into this script and
4
- save the replies as response_*.json next to it. (Windows PowerShell counterpart of download_bls.sh.)
5
-
6
- .DESCRIPTION
7
- - Needs no other file: the requests are inside this script.
8
- - When the requests need a BLS registration key (API v2), uses $env:BLS_API_KEY when it is set and
9
- otherwise asks for the key (hidden input). The key is only sent to api.bls.gov: it is never shown,
10
- saved, or put on a command line. With a key set, keyless requests also go through API v2.
11
- - Saves response_01.json, response_02.json, ... next to this script, each with the request it answers
12
- (without the key), and prints their full paths at the end.
13
- - Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
14
-
15
- Get a free key at https://data.bls.gov/registrationEngine/ (BLS emails it within minutes).
16
-
17
- .PARAMETER NoPause
18
- Do not wait for Enter before closing the window.
19
-
20
- .EXAMPLE
21
- powershell -NoProfile -ExecutionPolicy Bypass -File "$HOME\Downloads\download_bls.ps1"
22
- #>
23
- [CmdletBinding()]
24
- param(
25
- [switch]$NoPause
26
- )
27
-
28
- $ErrorActionPreference = 'Stop'
29
- $ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
30
- try {
31
- [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
32
- } catch { }
33
-
34
- $Here = $PSScriptRoot
35
- if (-not $Here) { $Here = (Get-Location).Path }
36
- $Utf8 = New-Object Text.UTF8Encoding($false)
37
- $Key = $null
38
- $Req = $null
39
-
40
- # Keep the window open when the script was started with "Run with PowerShell".
41
- function Finish([int]$Code) {
42
- $script:Key = $null
43
- if ($script:Req) { Remove-Item -LiteralPath $script:Req -Recurse -Force -ErrorAction SilentlyContinue }
44
- if (-not $NoPause -and -not [Console]::IsInputRedirected) {
45
- Write-Host ''
46
- [void](Read-Host -Prompt 'Press Enter to close')
47
- }
48
- exit $Code
49
- }
50
-
51
- function Fail([string]$Message) {
52
- Write-Host ''
53
- Write-Host "ERROR: $(Hide-Key $Message)" -ForegroundColor Red
54
- Finish 1
55
- }
56
-
57
- # BLS repeats a rejected key in its error message: never let it reach the screen or a file.
58
- function Hide-Key([string]$Text) {
59
- if ($script:Key -and $Text) { return $Text.Replace($script:Key, '[REDACTED]') }
60
- return $Text
61
- }
62
-
63
- # ---------------------------------------------------------------------------- the requests
64
- # manifest.json and query_NN.json are built into this script; they go to a temporary folder that is
65
- # removed when the script ends.
66
- $Embedded = [ordered]@{}
67
- # @QUERIES@
68
- $Req = Join-Path ([IO.Path]::GetTempPath()) ('download_bls.' + [IO.Path]::GetRandomFileName())
69
- [void](New-Item -ItemType Directory -Path $Req)
70
- foreach ($name in $Embedded.Keys) { [IO.File]::WriteAllText((Join-Path $Req $name), $Embedded[$name], $Utf8) }
71
- $manifestFile = Join-Path $Req 'manifest.json'
72
- if (-not (Test-Path -LiteralPath $manifestFile)) {
73
- Fail "This script has no requests built in. Use the download_bls.ps1 from the analyst's download files."
74
- }
75
- $Manifest = [IO.File]::ReadAllText($manifestFile, $Utf8) | ConvertFrom-Json
76
- $Queries = @(Get-ChildItem -LiteralPath $Req -Filter 'query_*.json' | Sort-Object Name)
77
- if ($Queries.Count -eq 0) { Fail "This script has no requests built in. Use the download_bls.ps1 from the analyst's download files." }
78
- $Bodies = @()
79
- foreach ($q in $Queries) { $Bodies += ,([IO.File]::ReadAllText($q.FullName, $Utf8) | ConvertFrom-Json) }
80
-
81
- $needKey = ($Manifest.api_version -eq 'v2')
82
- foreach ($b in $Bodies) { if ($b.PSObject.Properties['registrationkey']) { $needKey = $true } }
83
-
84
- # ---------------------------------------------------------------------------- API key
85
- $Key = $env:BLS_API_KEY
86
- if ($Key) {
87
- Write-Host 'Using the BLS key in $env:BLS_API_KEY.'
88
- } elseif ($needKey) {
89
- Write-Host 'These requests need a free BLS registration key (https://data.bls.gov/registrationEngine/).'
90
- if ([Console]::IsInputRedirected) {
91
- $Key = [Console]::In.ReadLine()
92
- } else {
93
- $secure = Read-Host -Prompt 'Paste your BLS key (input is hidden), then press Enter' -AsSecureString
94
- $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
95
- try { $Key = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
96
- finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
97
- Remove-Variable secure
98
- }
99
- }
100
- if ($null -eq $Key) { $Key = '' }
101
- $Key = $Key -replace '\s', ''
102
- if ($needKey -and -not $Key) { Fail 'No key entered. Get one at https://data.bls.gov/registrationEngine/' }
103
-
104
- $Version = 'v1'
105
- if ($Key -and $Manifest.urls.v2) { $Version = 'v2' }
106
- $Url = [string]$Manifest.urls.$Version
107
- Write-Host "Sending $($Queries.Count) request(s) to BLS API $Version ..."
108
-
109
- # ---------------------------------------------------------------------------- send each request
110
- $Saved = @()
111
- for ($i = 0; $i -lt $Queries.Count; $i++) {
112
- $q = $Queries[$i]
113
- $body = $Bodies[$i]
114
- $body.PSObject.Properties.Remove('registrationkey')
115
- if ($Version -eq 'v2' -and $Manifest.v2_options) {
116
- foreach ($p in $Manifest.v2_options.PSObject.Properties) {
117
- $body | Add-Member -NotePropertyName $p.Name -NotePropertyValue $p.Value -Force
118
- }
119
- }
120
- $requestJson = $body | ConvertTo-Json -Depth 10 -Compress # the request without the key
121
- $body | Add-Member -NotePropertyName 'registrationkey' -NotePropertyValue $Key -Force
122
- if ($Version -ne 'v2') { $body.PSObject.Properties.Remove('registrationkey') }
123
- $sendJson = $body | ConvertTo-Json -Depth 10 -Compress # in memory only
124
- $body.PSObject.Properties.Remove('registrationkey')
125
-
126
- try {
127
- $resp = Invoke-WebRequest -Uri $Url -Method Post -Body $Utf8.GetBytes($sendJson) `
128
- -ContentType 'application/json' -UserAgent 'data-analyst/1.0' -UseBasicParsing -TimeoutSec 120
129
- } catch {
130
- Fail "$($q.Name): the request to api.bls.gov failed ($($_.Exception.Message)). Check the internet connection, VPN and proxy, then run this script again."
131
- }
132
- $sendJson = $null
133
- $text = $Utf8.GetString($resp.RawContentStream.ToArray()).Trim()
134
- try { $reply = $text | ConvertFrom-Json } catch { $reply = $null }
135
- if ($null -eq $reply -or -not $text.EndsWith('}') -or $reply.PSObject.Properties.Count -eq 0) {
136
- Fail "$($q.Name): BLS did not answer with JSON. The reply began: $($text.Substring(0, [Math]::Min(200, $text.Length)))"
137
- }
138
- # Keep the reply exactly as BLS sent it and add the request it answers, like download_bls.sh.
139
- $text = $text.Substring(0, $text.LastIndexOf('}')).TrimEnd() + ",`n `"_request`": " + $requestJson + "`n}"
140
- $dest = Join-Path $Here ($q.Name -replace '^query_', 'response_')
141
- [IO.File]::WriteAllText($dest, (Hide-Key $text) + "`n", $Utf8)
142
- $Saved += $dest
143
- $msg = Hide-Key ((@($reply.message) | Where-Object { $_ }) -join '; ')
144
- if ($msg.Length -gt 200) { $msg = $msg.Substring(0, 200) }
145
- Write-Host "$(Split-Path -Leaf $dest) $($reply.status) $msg"
146
- if ($i -lt $Queries.Count - 1) { Start-Sleep -Milliseconds 500 }
147
- }
148
-
149
- Write-Host ''
150
- Write-Host 'Done. Upload these files to the chat (do not edit them):' -ForegroundColor Green
151
- foreach ($p in $Saved) { Write-Host " $p" }
152
- Finish 0
1
+ <#
2
+ .SYNOPSIS
3
+ Download BLS time series with the BLS Public Data API: send the requests built into this script and
4
+ save the replies as response_*.json next to it. (Windows PowerShell counterpart of download_bls.sh.)
5
+
6
+ .DESCRIPTION
7
+ - Needs no other file: the requests are inside this script.
8
+ - When the requests need a BLS registration key (API v2), uses $env:BLS_API_KEY when it is set and
9
+ otherwise asks for the key (hidden input). The key is only sent to api.bls.gov: it is never shown,
10
+ saved, or put on a command line. With a key set, keyless requests also go through API v2.
11
+ - Saves response_01.json, response_02.json, ... next to this script, each with the request it answers
12
+ (without the key), and prints their full paths at the end.
13
+ - Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
14
+
15
+ Get a free key at https://data.bls.gov/registrationEngine/ (BLS emails it within minutes).
16
+
17
+ .PARAMETER NoPause
18
+ Do not wait for Enter before closing the window.
19
+
20
+ .EXAMPLE
21
+ powershell -NoProfile -ExecutionPolicy Bypass -File "$HOME\Downloads\download_bls.ps1"
22
+ #>
23
+ [CmdletBinding()]
24
+ param(
25
+ [switch]$NoPause
26
+ )
27
+
28
+ $ErrorActionPreference = 'Stop'
29
+ $ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
30
+ try {
31
+ [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
32
+ } catch { }
33
+
34
+ $Here = $PSScriptRoot
35
+ if (-not $Here) { $Here = (Get-Location).Path }
36
+ $Utf8 = New-Object Text.UTF8Encoding($false)
37
+ $Key = $null
38
+ $Req = $null
39
+
40
+ # Keep the window open when the script was started with "Run with PowerShell".
41
+ function Finish([int]$Code) {
42
+ $script:Key = $null
43
+ if ($script:Req) { Remove-Item -LiteralPath $script:Req -Recurse -Force -ErrorAction SilentlyContinue }
44
+ if (-not $NoPause -and -not [Console]::IsInputRedirected) {
45
+ Write-Host ''
46
+ [void](Read-Host -Prompt 'Press Enter to close')
47
+ }
48
+ exit $Code
49
+ }
50
+
51
+ function Fail([string]$Message) {
52
+ Write-Host ''
53
+ Write-Host "ERROR: $(Hide-Key $Message)" -ForegroundColor Red
54
+ Finish 1
55
+ }
56
+
57
+ # BLS repeats a rejected key in its error message: never let it reach the screen or a file.
58
+ function Hide-Key([string]$Text) {
59
+ if ($script:Key -and $Text) { return $Text.Replace($script:Key, '[REDACTED]') }
60
+ return $Text
61
+ }
62
+
63
+ # ---------------------------------------------------------------------------- the requests
64
+ # manifest.json and query_NN.json are built into this script; they go to a temporary folder that is
65
+ # removed when the script ends.
66
+ $Embedded = [ordered]@{}
67
+ # @QUERIES@
68
+ $Req = Join-Path ([IO.Path]::GetTempPath()) ('download_bls.' + [IO.Path]::GetRandomFileName())
69
+ [void](New-Item -ItemType Directory -Path $Req)
70
+ foreach ($name in $Embedded.Keys) { [IO.File]::WriteAllText((Join-Path $Req $name), $Embedded[$name], $Utf8) }
71
+ $manifestFile = Join-Path $Req 'manifest.json'
72
+ if (-not (Test-Path -LiteralPath $manifestFile)) {
73
+ Fail "This script has no requests built in. Use the download_bls.ps1 from the analyst's download files."
74
+ }
75
+ $Manifest = [IO.File]::ReadAllText($manifestFile, $Utf8) | ConvertFrom-Json
76
+ $Queries = @(Get-ChildItem -LiteralPath $Req -Filter 'query_*.json' | Sort-Object Name)
77
+ if ($Queries.Count -eq 0) { Fail "This script has no requests built in. Use the download_bls.ps1 from the analyst's download files." }
78
+ $Bodies = @()
79
+ foreach ($q in $Queries) { $Bodies += ,([IO.File]::ReadAllText($q.FullName, $Utf8) | ConvertFrom-Json) }
80
+
81
+ $needKey = ($Manifest.api_version -eq 'v2')
82
+ foreach ($b in $Bodies) { if ($b.PSObject.Properties['registrationkey']) { $needKey = $true } }
83
+
84
+ # ---------------------------------------------------------------------------- API key
85
+ $Key = $env:BLS_API_KEY
86
+ if ($Key) {
87
+ Write-Host 'Using the BLS key in $env:BLS_API_KEY.'
88
+ } elseif ($needKey) {
89
+ Write-Host 'These requests need a free BLS registration key (https://data.bls.gov/registrationEngine/).'
90
+ if ([Console]::IsInputRedirected) {
91
+ $Key = [Console]::In.ReadLine()
92
+ } else {
93
+ $secure = Read-Host -Prompt 'Paste your BLS key (input is hidden), then press Enter' -AsSecureString
94
+ $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
95
+ try { $Key = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
96
+ finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
97
+ Remove-Variable secure
98
+ }
99
+ }
100
+ if ($null -eq $Key) { $Key = '' }
101
+ $Key = $Key -replace '\s', ''
102
+ if ($needKey -and -not $Key) { Fail 'No key entered. Get one at https://data.bls.gov/registrationEngine/' }
103
+
104
+ $Version = 'v1'
105
+ if ($Key -and $Manifest.urls.v2) { $Version = 'v2' }
106
+ $Url = [string]$Manifest.urls.$Version
107
+ Write-Host "Sending $($Queries.Count) request(s) to BLS API $Version ..."
108
+
109
+ # ---------------------------------------------------------------------------- send each request
110
+ $Saved = @()
111
+ for ($i = 0; $i -lt $Queries.Count; $i++) {
112
+ $q = $Queries[$i]
113
+ $body = $Bodies[$i]
114
+ $body.PSObject.Properties.Remove('registrationkey')
115
+ if ($Version -eq 'v2' -and $Manifest.v2_options) {
116
+ foreach ($p in $Manifest.v2_options.PSObject.Properties) {
117
+ $body | Add-Member -NotePropertyName $p.Name -NotePropertyValue $p.Value -Force
118
+ }
119
+ }
120
+ $requestJson = $body | ConvertTo-Json -Depth 10 -Compress # the request without the key
121
+ $body | Add-Member -NotePropertyName 'registrationkey' -NotePropertyValue $Key -Force
122
+ if ($Version -ne 'v2') { $body.PSObject.Properties.Remove('registrationkey') }
123
+ $sendJson = $body | ConvertTo-Json -Depth 10 -Compress # in memory only
124
+ $body.PSObject.Properties.Remove('registrationkey')
125
+
126
+ try {
127
+ $resp = Invoke-WebRequest -Uri $Url -Method Post -Body $Utf8.GetBytes($sendJson) `
128
+ -ContentType 'application/json' -UserAgent 'data-analyst/1.0' -UseBasicParsing -TimeoutSec 120
129
+ } catch {
130
+ Fail "$($q.Name): the request to api.bls.gov failed ($($_.Exception.Message)). Check the internet connection, VPN and proxy, then run this script again."
131
+ }
132
+ $sendJson = $null
133
+ $text = $Utf8.GetString($resp.RawContentStream.ToArray()).Trim()
134
+ try { $reply = $text | ConvertFrom-Json } catch { $reply = $null }
135
+ if ($null -eq $reply -or -not $text.EndsWith('}') -or $reply.PSObject.Properties.Count -eq 0) {
136
+ Fail "$($q.Name): BLS did not answer with JSON. The reply began: $($text.Substring(0, [Math]::Min(200, $text.Length)))"
137
+ }
138
+ # Keep the reply exactly as BLS sent it and add the request it answers, like download_bls.sh.
139
+ $text = $text.Substring(0, $text.LastIndexOf('}')).TrimEnd() + ",`n `"_request`": " + $requestJson + "`n}"
140
+ $dest = Join-Path $Here ($q.Name -replace '^query_', 'response_')
141
+ [IO.File]::WriteAllText($dest, (Hide-Key $text) + "`n", $Utf8)
142
+ $Saved += $dest
143
+ $msg = Hide-Key ((@($reply.message) | Where-Object { $_ }) -join '; ')
144
+ if ($msg.Length -gt 200) { $msg = $msg.Substring(0, 200) }
145
+ Write-Host "$(Split-Path -Leaf $dest) $($reply.status) $msg"
146
+ if ($i -lt $Queries.Count - 1) { Start-Sleep -Milliseconds 500 }
147
+ }
148
+
149
+ Write-Host ''
150
+ Write-Host 'Done. Upload these files to the chat (do not edit them):' -ForegroundColor Green
151
+ foreach ($p in $Saved) { Write-Host " $p" }
152
+ Finish 0