data-analyst-mcp 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/README.md +262 -262
  2. package/dist/report/method_audit.js +7 -1
  3. package/dist/report/method_audit.js.map +1 -1
  4. package/dist/report/scan_secrets.d.ts +4 -1
  5. package/dist/report/scan_secrets.js +36 -5
  6. package/dist/report/scan_secrets.js.map +1 -1
  7. package/dist/server.d.ts +1 -1
  8. package/dist/server.js +3 -2
  9. package/dist/server.js.map +1 -1
  10. package/guide/WORKFLOW.md +134 -134
  11. package/guide/assets/disclaimer.txt +1 -1
  12. package/guide/assets/download_data.ps1 +162 -162
  13. package/guide/assets/report_template.html +597 -597
  14. package/guide/references/routing.md +171 -171
  15. package/guide/references/style_guide.md +398 -398
  16. package/package.json +56 -56
  17. package/packs/acf/endpoints.json +114 -114
  18. package/packs/acf/policy.json +104 -104
  19. package/packs/actuaries/endpoints.json +82 -82
  20. package/packs/actuaries/policy.json +109 -109
  21. package/packs/bea/assets/download_bea.ps1 +182 -182
  22. package/packs/bea/docs/bea_api.md +461 -461
  23. package/packs/bea/docs/datasets.md +157 -157
  24. package/packs/bea/docs/report_patterns.md +112 -112
  25. package/packs/bea/policy.json +170 -170
  26. package/packs/bea/source.md +152 -152
  27. package/packs/bls/assets/download_bls.ps1 +152 -152
  28. package/packs/bls/docs/analysis_patterns.md +108 -108
  29. package/packs/bls/docs/bls_api.md +73 -73
  30. package/packs/bls/docs/series_ids.md +149 -149
  31. package/packs/bls/policy.json +238 -238
  32. package/packs/bts/endpoints.json +346 -346
  33. package/packs/bts/policy.json +94 -94
  34. package/packs/cdc/endpoints.json +197 -197
  35. package/packs/cdc/policy.json +83 -83
  36. package/packs/census/endpoints.json +99 -99
  37. package/packs/cfpb/endpoints.json +326 -326
  38. package/packs/cfpb/policy.json +119 -119
  39. package/packs/clinical-trials/endpoints.json +166 -166
  40. package/packs/clinical-trials/policy.json +84 -84
  41. package/packs/cms/endpoints.json +69 -69
  42. package/packs/cms/policy.json +96 -96
  43. package/packs/college-scorecard/endpoints.json +252 -252
  44. package/packs/college-scorecard/policy.json +106 -106
  45. package/packs/congress/endpoints.json +1701 -1701
  46. package/packs/congress/policy.json +117 -117
  47. package/packs/dhs-ohss/endpoints.json +66 -66
  48. package/packs/dhs-ohss/policy.json +107 -107
  49. package/packs/doj-news/endpoints.json +59 -59
  50. package/packs/doj-news/policy.json +96 -96
  51. package/packs/dol/endpoints.json +817 -817
  52. package/packs/dol/policy.json +135 -135
  53. package/packs/eia/endpoints.json +717 -717
  54. package/packs/eia/policy.json +112 -112
  55. package/packs/epa/endpoints.json +291 -291
  56. package/packs/epa/policy.json +99 -99
  57. package/packs/epa-aqs/endpoints.json +173 -173
  58. package/packs/epa-aqs/policy.json +96 -96
  59. package/packs/fbi/endpoints.json +1841 -1841
  60. package/packs/fbi/policy.json +118 -118
  61. package/packs/fda/endpoints.json +237 -237
  62. package/packs/fda/policy.json +121 -121
  63. package/packs/fdic/endpoints.json +416 -416
  64. package/packs/fdic/policy.json +83 -83
  65. package/packs/fec/endpoints.json +707 -707
  66. package/packs/fec/policy.json +93 -93
  67. package/packs/fed-board/endpoints.json +89 -89
  68. package/packs/fed-board/policy.json +136 -136
  69. package/packs/federal-register/endpoints.json +143 -143
  70. package/packs/federal-register/policy.json +86 -86
  71. package/packs/fema/endpoints.json +645 -645
  72. package/packs/fema/policy.json +103 -103
  73. package/packs/fhfa/endpoints.json +105 -105
  74. package/packs/fhfa/policy.json +109 -109
  75. package/packs/fred/assets/download_fred.ps1 +122 -122
  76. package/packs/fred/docs/analysis_patterns.md +124 -124
  77. package/packs/fred/docs/fred_api.md +94 -94
  78. package/packs/fred/docs/series.md +188 -188
  79. package/packs/fred/policy.json +253 -253
  80. package/packs/fsa/endpoints.json +150 -150
  81. package/packs/fsa/policy.json +99 -99
  82. package/packs/govinfo/endpoints.json +519 -519
  83. package/packs/gsa-calc/endpoints.json +93 -93
  84. package/packs/gsa-calc/policy.json +97 -97
  85. package/packs/hud/endpoints.json +146 -146
  86. package/packs/hud/policy.json +84 -84
  87. package/packs/ipums/docs/report_patterns.md +117 -117
  88. package/packs/irs-soi/endpoints.json +114 -114
  89. package/packs/irs-soi/policy.json +107 -107
  90. package/packs/medicaid/endpoints.json +412 -412
  91. package/packs/medicaid/policy.json +108 -108
  92. package/packs/naep/endpoints.json +217 -217
  93. package/packs/naep/policy.json +100 -100
  94. package/packs/nhtsa/endpoints.json +132 -132
  95. package/packs/nhtsa/policy.json +93 -93
  96. package/packs/nih/endpoints.json +452 -452
  97. package/packs/nih/policy.json +86 -86
  98. package/packs/noaa/endpoints.json +431 -431
  99. package/packs/noaa/policy.json +95 -95
  100. package/packs/nrel/endpoints.json +327 -327
  101. package/packs/nrel/policy.json +96 -96
  102. package/packs/nws/endpoints.json +271 -271
  103. package/packs/nws/policy.json +82 -82
  104. package/packs/nyfed/endpoints.json +80 -80
  105. package/packs/nyfed/policy.json +92 -92
  106. package/packs/omb/endpoints.json +460 -460
  107. package/packs/omb/policy.json +106 -106
  108. package/packs/open-payments/endpoints.json +297 -297
  109. package/packs/open-payments/policy.json +101 -101
  110. package/packs/opm/assets/download_opm.ps1 +125 -125
  111. package/packs/opm/docs/fields.md +184 -184
  112. package/packs/opm/docs/opm_api.md +71 -71
  113. package/packs/opm/docs/report_patterns.md +83 -83
  114. package/packs/opm/lib/data_dictionary.csv +73 -73
  115. package/packs/pbgc/endpoints.json +483 -483
  116. package/packs/pbgc/policy.json +94 -94
  117. package/packs/philfed/endpoints.json +85 -85
  118. package/packs/philfed/policy.json +94 -94
  119. package/packs/regulations/endpoints.json +287 -287
  120. package/packs/regulations/policy.json +86 -86
  121. package/packs/sec/endpoints.json +236 -236
  122. package/packs/sec/policy.json +86 -86
  123. package/packs/senate-lobbying/endpoints.json +553 -553
  124. package/packs/senate-lobbying/policy.json +99 -99
  125. package/packs/snapqc/endpoints.json +1426 -1426
  126. package/packs/snapqc/policy.json +92 -92
  127. package/packs/treasury/catalog.json +2267 -2267
  128. package/packs/treasury/docs/api_guide.md +245 -245
  129. package/packs/treasury/docs/datasets.md +56 -56
  130. package/packs/treasury/docs/report_patterns.md +126 -126
  131. package/packs/treasury/policy.json +159 -159
  132. package/packs/treasury/source.md +170 -165
  133. package/packs/treasury-yields/endpoints.json +226 -226
  134. package/packs/treasury-yields/policy.json +86 -86
  135. package/packs/usaspending/endpoints.json +1315 -1315
  136. package/packs/usaspending/policy.json +109 -109
  137. package/packs/usda-fna/policy.json +133 -133
  138. package/packs/usda-fooddata/endpoints.json +180 -180
  139. package/packs/usda-fooddata/policy.json +94 -94
  140. package/packs/usda-nass/endpoints.json +323 -323
  141. package/packs/usda-nass/policy.json +95 -95
  142. package/packs/usda-outlook/endpoints.json +86 -86
  143. package/packs/usda-outlook/policy.json +109 -109
  144. package/packs/usgs/endpoints.json +313 -313
  145. package/packs/usgs/policy.json +95 -95
  146. package/packs/usitc/endpoints.json +74 -74
  147. package/packs/usitc/policy.json +111 -111
  148. package/packs/uspto/endpoints.json +141 -141
  149. package/packs/uspto/policy.json +95 -95
  150. package/packs/va/endpoints.json +414 -414
  151. package/packs/va/policy.json +101 -101
  152. package/packs/world-bank/endpoints.json +122 -122
  153. package/packs/world-bank/policy.json +101 -101
package/README.md CHANGED
@@ -1,262 +1,262 @@
1
- # data-analyst-mcp
2
-
3
- An MCP server for official U.S. government data. It asks the user the questions that change a headline
4
- number before fetching anything, gets the data from the agency at call time, packages it (CSV, data
5
- dictionary, provenance) and cites every result. It covers 63 sources.
6
-
7
- > **Status: 0.3.** All 63 sources work, along with the analysis and audit tools, hosted mode (streamable HTTP, a
8
- > claude.ai connector) and per-user keys. The test suite runs offline on recorded fixtures.
9
- > One gap remains: usda-fna's three PDF endpoints need a PDF text reader, and return `not_supported`.
10
- > The package is not published to npm yet.
11
-
12
- ## Install
13
-
14
- Requires Node.js 22 or later. Most sources need no key, so steps 2 and 3 are optional.
15
-
16
- ### 1. Add the server to your client
17
-
18
- **Claude Code**
19
-
20
- ```bash
21
- claude mcp add data-analyst -- npx -y data-analyst-mcp
22
- ```
23
-
24
- Add `--scope user` to make it available in every project. To pass a key here, put `-e` before the `--`, for example
25
- `claude mcp add data-analyst -e DATA_GOV_API_KEY=your-key -- npx -y data-analyst-mcp`.
26
-
27
- **Claude Desktop**: `claude_desktop_config.json`, in `~/Library/Application Support/Claude/` (macOS) or
28
- `%APPDATA%\Claude\` (Windows). Settings → Developer → Edit Config opens it. Restart Claude Desktop after editing.
29
-
30
- ```json
31
- {
32
- "mcpServers": {
33
- "data-analyst": {
34
- "command": "npx",
35
- "args": ["-y", "data-analyst-mcp"],
36
- "env": { "DATA_GOV_API_KEY": "your-key" }
37
- }
38
- }
39
- }
40
- ```
41
-
42
- **Cursor**: the same `mcpServers` block in `~/.cursor/mcp.json` (every project) or `.cursor/mcp.json` (one project).
43
-
44
- **VS Code**: `.vscode/mcp.json` uses `servers` instead of `mcpServers`. An `inputs` entry makes VS Code ask for the key
45
- once and store it, so the key is not written into a file you might commit.
46
-
47
- ```json
48
- {
49
- "inputs": [
50
- { "type": "promptString", "id": "data-gov-key", "description": "api.data.gov API key", "password": true }
51
- ],
52
- "servers": {
53
- "data-analyst": {
54
- "type": "stdio",
55
- "command": "npx",
56
- "args": ["-y", "data-analyst-mcp"],
57
- "env": { "DATA_GOV_API_KEY": "${input:data-gov-key}" }
58
- }
59
- }
60
- }
61
- ```
62
-
63
- **Codex**: `~/.codex/config.toml`
64
-
65
- ```toml
66
- [mcp_servers.data-analyst]
67
- command = "npx"
68
- args = ["-y", "data-analyst-mcp"]
69
-
70
- [mcp_servers.data-analyst.env]
71
- DATA_GOV_API_KEY = "your-key"
72
- ```
73
-
74
- Leave out the `env` block if you have no keys, or if you store them with `setup` (step 2).
75
-
76
- **Until the package is published to npm**, build it and point the client at the build:
77
-
78
- ```bash
79
- git clone https://github.com/zsu/data-analyst-mcp.git
80
- ```
81
-
82
- ```bash
83
- cd data-analyst-mcp && npm install && npm run build
84
- ```
85
-
86
- Then use `"command": "node", "args": ["<path>/data-analyst-mcp/dist/bin/cli.js"]` in the configs above, and
87
- `node dist/bin/cli.js setup <source>` in place of `npx -y data-analyst-mcp setup <source>` below.
88
-
89
- ### 2. Add keys (optional)
90
-
91
- Each key is free. Without a key, a source either falls back to a keyless route with lower limits, or offers a
92
- manual download (`manual_bundle`): the user downloads the files in a browser, and the server packages them.
93
-
94
- | Variable | Sources | Without it | Free key |
95
- |---|---|---|---|
96
- | `AQS_API_KEY` + `AQS_EMAIL` | epa-aqs | manual download only | [aqs.epa.gov](https://aqs.epa.gov/data/api/signup?email=YOUR_EMAIL) |
97
- | `BEA_API_KEY` | bea | bulk files for the main tables; the API-only datasets need the key | [apps.bea.gov](https://apps.bea.gov/API/signup/) |
98
- | `BLS_API_KEY` | bls | 25 queries a day, 25 series and 10 years each; no annual averages or catalog titles | [data.bls.gov](https://data.bls.gov/registrationEngine/) |
99
- | `CENSUS_API_KEY` | census | manual download only | [api.census.gov](https://api.census.gov/data/key_signup.html) |
100
- | `DATA_GOV_API_KEY` | college-scorecard, congress, fbi, fda, fec, govinfo, nrel, regulations, usda-fooddata, usgs | `DEMO_KEY` or keyless, at a few requests an hour (fda: 1,000 a day; usgs earthquakes never need a key) | [api.data.gov](https://api.data.gov/signup/) |
101
- | `DOL_API_KEY` | dol | catalog and field lists only; data queries need the key | [dataportal.dol.gov](https://dataportal.dol.gov/registration) |
102
- | `EIA_API_KEY` | eia | `DEMO_KEY`, about 10 requests an hour | [eia.gov](https://www.eia.gov/opendata/register.php) |
103
- | `FRED_API_KEY` | fred | observations only; search, vintages, revision history and GeoFRED need the key | [fredaccount.stlouisfed.org](https://fredaccount.stlouisfed.org/apikeys) |
104
- | `HUD_USER_TOKEN` | hud | manual download only | [huduser.gov](https://www.huduser.gov/hudapi/public/register) |
105
- | `IPUMS_API_KEY` | ipums | build the extract on the IPUMS website (free account) | [account.ipums.org](https://account.ipums.org/api_keys) |
106
- | `LDA_API_KEY` | senate-lobbying | 15 requests a minute instead of 120 | [lda.gov](https://lda.gov/api/register/) |
107
- | `NOAA_API_KEY` | noaa | the keyless data and station-search endpoints; Climate Data Online v2 needs the key | [ncei.noaa.gov](https://www.ncei.noaa.gov/cdo-web/token) |
108
- | `USDA_NASS_API_KEY` | usda-nass | manual download only | [quickstats.nass.usda.gov](https://quickstats.nass.usda.gov/api) |
109
- | `USPTO_API_KEY` | uspto | manual download only | [data.uspto.gov](https://data.uspto.gov/apis/getting-started) |
110
-
111
- Two settings are not keys. SEC asks every client to identify itself, so set `SEC_USER_AGENT` to your name and email
112
- (`"Jane Doe jane@example.com"`). `NWS_USER_AGENT` is optional and does the same for the weather service.
113
-
114
- **Store a key with `setup` (recommended).** It shows the signup link, reads the key with hidden input, and writes
115
- it to `~/.config/data-analyst/keys.json`. The key stays out of client config files, and every client reads it. It
116
- works mid-session, with no restart.
117
-
118
- ```bash
119
- npx -y data-analyst-mcp setup congress
120
- ```
121
-
122
- `setup` takes any source that uses the key: `setup congress` stores `DATA_GOV_API_KEY`, and all 10 api.data.gov
123
- sources use it. `setup epa-aqs` also asks for `AQS_EMAIL`. Or put the variables in the client's `env` block, as in step 1.
124
- [API keys](#api-keys) lists every place the server looks.
125
-
126
- ### 3. Check it
127
-
128
- Ask your assistant: *"Is my api.data.gov key set up?"* The `check_key` tool reports whether the key was found and where,
129
- never its value. To list only the sources you can use, start the server with `--hide-unconfigured` (see
130
- [Options](#options)).
131
-
132
- ### 4. Try it
133
-
134
- - *"How has the unemployment rate compared with CPI inflation since 2020?"*
135
- - *"What is the median household income in each county of Ohio?"*
136
- - *"Which drugs had the most reported adverse events last year?"*
137
- - *"How much did each federal agency spend in fiscal year 2024?"*
138
- - *"How many unhealthy air-quality days did Phoenix have in 2023?"*
139
-
140
- The assistant asks the questions that change the answer first (which measure, which years, which population),
141
- then fetches, computes and cites.
142
-
143
- To host the server for claude.ai or a team, see [docs/hosting.md](docs/hosting.md).
144
-
145
- ## How it works
146
-
147
- | Tool | What it does |
148
- |---|---|
149
- | `guide` | the method: framing, asking only critical questions, the accuracy checklist, combining sources |
150
- | `find_sources` | ranks the sources for a question, or lists them all |
151
- | `source_brief` | one source's playbook, its **questions_to_ask**, key status, rules and request index |
152
- | `request_schema` | full parameters of the requests to use; searches a source's catalog |
153
- | `record_decisions` | how each critical question was settled (answered, Don't know, skipped, not asked and why) |
154
- | `fetch` | downloads into the session workspace; returns a summary, the files, the citation and notes |
155
- | `manual_bundle`, `ingest` | a one-command download for the user's own computer, then packaging what they downloaded |
156
- | `compute` | builds exhibits from the full tables with declarative steps (a whitelisted expression language, no eval); the steps are recorded for `rerun` |
157
- | `facts`, `write_table`, `cite` | register the numbers cited, write the benchmark and sensitivity tables, snapshot cited pages |
158
- | `audit_answer`, `rerun` | the release gate: every number in the answer is a registered fact, the method record is complete, and the analysis reproduces |
159
- | `build_report`, `style_check`, `audit_report` | an HTML report and replication ZIP, only when asked |
160
- | `handoff` | the per-source data hand-off |
161
- | `check_key` | whether a key is set up (never shows it) |
162
- | `read_file` | a small text file from the workspace |
163
-
164
- **Asking before fetching is enforced, not just suggested.** `fetch` returns `gate_blocked` until, for that
165
- source, the model has read `source_brief` and the schemas of the requests it uses, and has recorded how each
166
- critical question was settled. The model asks the questions in chat, in one round. A blocking question that
167
- the user skips is asked once more. The server also exposes an `analyze` prompt and `source://<id>`
168
- resources, for clients that let users pick prompts or attach resources.
169
-
170
- **Results** have one shape. Success: `{status: "ok" | "partial", summary, files, citation, notes}`. Failure:
171
- `{status: "error", code, message, fix, retry_after_s?}`. The codes are `invalid_request`, `not_found`,
172
- `not_supported`, `agency_rejected`, `agency_unavailable`, `key_missing`, `key_rejected`, `rate_limited`,
173
- `gate_blocked` and `server_error`.
174
-
175
- ## Options
176
-
177
- | Option (flag / environment) | What it does |
178
- |---|---|
179
- | `--sources fred,treasury` / `DATA_ANALYST_SOURCES` | serve only these sources |
180
- | `--domains economy,health` / `DATA_ANALYST_DOMAINS` | serve only the sources in these domains |
181
- | `--hide-unconfigured` / `DATA_ANALYST_HIDE_UNCONFIGURED=1` | leave out of listings the sources whose required key is not set up |
182
- | `--workspace <folder>` / `DATA_ANALYST_WORKSPACE` | (stdio) one fixed analysis folder, so a fetch is reused after a restart |
183
-
184
- ```json
185
- { "mcpServers": { "data-analyst": { "command": "npx", "args": ["-y", "data-analyst-mcp", "--domains", "economy,health"] } } }
186
- ```
187
-
188
- The tool list stays the same size whatever is enabled: sources are found with `find_sources`, not loaded as
189
- tools.
190
-
191
- Every result lists its upstream requests (`requests`: method, credential-free URL, status, cached, time), and
192
- `citation` names the publisher, page and retrieval date. Requests are rate-limited per host (a token bucket),
193
- retried with backoff on 429 and 5xx (honoring Retry-After), and kept within each agency's per-minute budgets.
194
-
195
- **Caching follows each source's terms.**
196
- - A source whose policy allows persistent copies (`cache.mode: persistent`) can be cached on disk in
197
- `DATA_ANALYST_CACHE` (default `~/.cache/data-analyst`). census and bea do; opm's terms allow it, but its port
198
- downloads again for now.
199
- - Most sources allow reuse only inside the analysis folder, and FRED none at all. For those, a fixed
200
- `--workspace` gives reuse across restarts without keeping copies anywhere else.
201
-
202
- ## As a library
203
-
204
- ```ts
205
- import { route, brief, fetchSource, compute, registerFacts } from "data-analyst-mcp";
206
-
207
- const r = await fetchSource("cdc", { description: "NY deaths 2017", requests: [{ name: "ny", endpoint: "query",
208
- params: { dataset: "leading_death", $where: "state = 'New York' AND year = '2017'" } }] }, { out: "./out/ny" });
209
- console.log(r.summary.tables[0].rows, r.citation.text);
210
- ```
211
-
212
- - **The functions:** the same sources, engine, analysis and audits as the server. The exports are `listSources`,
213
- `route`, `brief`, `requestSchema`, `fetchSource`, `ingest`, `manualBundle`, `compute`, `registerFacts`,
214
- `writeTable`, `rerun`, `recordDecisions`, `auditAnswer`, `buildReport`, `auditReport`, `createServer` and
215
- `createApp`, plus their types.
216
- - **No ask-first gate:** it is for assistants, so the library leaves it out.
217
- - **Errors:** a failure throws `ToolError`; its `failure` holds the code and the fix.
218
-
219
- ## API keys
220
-
221
- [Install, step 2](#2-add-keys-optional) lists the keys and how to get them. When a fetch needs a key that is not set up,
222
- `fetch` returns `key_missing` with a `setup` command for **you** to run in your own terminal. The model never asks
223
- for a key in chat. `setup` writes `~/.config/data-analyst/keys.json`.
224
-
225
- The local (stdio) server looks for a key in this order. `DATA_GOV_API_KEY` is the example; each source's variable is named in
226
- its `source_brief`.
227
-
228
- 1. The `DATA_GOV_API_KEY` environment variable, for example in the MCP client's `env` block.
229
- 2. The file named by `DATA_GOV_API_KEY_FILE` (for secret managers and mounted secrets).
230
- 3. `"env": {"DATA_GOV_API_KEY": "..."}` in `~/.config/data-analyst/keys.json` (or `$DATA_ANALYST_CONFIG_DIR/keys.json`).
231
- `setup` writes this file.
232
- 4. `"env": {"DATA_GOV_API_KEY": "..."}` in Claude Code's `~/.claude/settings.json` (or `$CLAUDE_CONFIG_DIR/settings.json`).
233
- 5. `~/.data-gov/api_key`, a file that holds only the key.
234
-
235
- The files are read again on each lookup, so a key stored during a session works without a restart.
236
-
237
- No key value ever appears in a tool result: every output is redacted, and a test checks this. The hosted server
238
- reads keys only from its environment and from each user's encrypted store (see [docs/hosting.md](docs/hosting.md)).
239
-
240
- ## Hosted mode (claude.ai connector)
241
-
242
- `data-analyst-mcp serve --http` serves MCP at `/mcp` over streamable HTTP. With `DATA_ANALYST_ACCOUNTS=1`, users
243
- sign in through OAuth and store their own API keys on a write-only page. The keys are encrypted at rest, and the
244
- model never sees them. See [docs/hosting.md](docs/hosting.md) and the `Dockerfile`.
245
-
246
- ## Workspace
247
-
248
- Each session writes to its own folder under `$DATA_ANALYST_OUT` (default: the system temp folder,
249
- `data-analyst-mcp/<session>`), with `data/<source>/` and `decisions.json` inside. Nothing is written inside the
250
- package.
251
-
252
- ## Development
253
-
254
- ```bash
255
- npm install
256
- npm test
257
- npm run build
258
- npm run smoke
259
- ```
260
-
261
- - `npm test` runs offline, on recorded fixtures.
262
- - `npm run smoke` drives the built server over stdio and makes one live fetch.
1
+ # data-analyst-mcp
2
+
3
+ An MCP server for official U.S. government data. It asks the user the questions that change a headline
4
+ number before fetching anything, gets the data from the agency at call time, packages it (CSV, data
5
+ dictionary, provenance) and cites every result. It covers 63 sources.
6
+
7
+ > **Status: 0.3.** All 63 sources work, along with the analysis and audit tools, hosted mode (streamable HTTP, a
8
+ > claude.ai connector) and per-user keys. The test suite runs offline on recorded fixtures.
9
+ > One gap remains: usda-fna's three PDF endpoints need a PDF text reader, and return `not_supported`.
10
+ > The package is not published to npm yet.
11
+
12
+ ## Install
13
+
14
+ Requires Node.js 22 or later. Most sources need no key, so steps 2 and 3 are optional.
15
+
16
+ ### 1. Add the server to your client
17
+
18
+ **Claude Code**
19
+
20
+ ```bash
21
+ claude mcp add data-analyst -- npx -y data-analyst-mcp
22
+ ```
23
+
24
+ Add `--scope user` to make it available in every project. To pass a key here, put `-e` before the `--`, for example
25
+ `claude mcp add data-analyst -e DATA_GOV_API_KEY=your-key -- npx -y data-analyst-mcp`.
26
+
27
+ **Claude Desktop**: `claude_desktop_config.json`, in `~/Library/Application Support/Claude/` (macOS) or
28
+ `%APPDATA%\Claude\` (Windows). Settings → Developer → Edit Config opens it. Restart Claude Desktop after editing.
29
+
30
+ ```json
31
+ {
32
+ "mcpServers": {
33
+ "data-analyst": {
34
+ "command": "npx",
35
+ "args": ["-y", "data-analyst-mcp"],
36
+ "env": { "DATA_GOV_API_KEY": "your-key" }
37
+ }
38
+ }
39
+ }
40
+ ```
41
+
42
+ **Cursor**: the same `mcpServers` block in `~/.cursor/mcp.json` (every project) or `.cursor/mcp.json` (one project).
43
+
44
+ **VS Code**: `.vscode/mcp.json` uses `servers` instead of `mcpServers`. An `inputs` entry makes VS Code ask for the key
45
+ once and store it, so the key is not written into a file you might commit.
46
+
47
+ ```json
48
+ {
49
+ "inputs": [
50
+ { "type": "promptString", "id": "data-gov-key", "description": "api.data.gov API key", "password": true }
51
+ ],
52
+ "servers": {
53
+ "data-analyst": {
54
+ "type": "stdio",
55
+ "command": "npx",
56
+ "args": ["-y", "data-analyst-mcp"],
57
+ "env": { "DATA_GOV_API_KEY": "${input:data-gov-key}" }
58
+ }
59
+ }
60
+ }
61
+ ```
62
+
63
+ **Codex**: `~/.codex/config.toml`
64
+
65
+ ```toml
66
+ [mcp_servers.data-analyst]
67
+ command = "npx"
68
+ args = ["-y", "data-analyst-mcp"]
69
+
70
+ [mcp_servers.data-analyst.env]
71
+ DATA_GOV_API_KEY = "your-key"
72
+ ```
73
+
74
+ Leave out the `env` block if you have no keys, or if you store them with `setup` (step 2).
75
+
76
+ **Until the package is published to npm**, build it and point the client at the build:
77
+
78
+ ```bash
79
+ git clone https://github.com/zsu/data-analyst-mcp.git
80
+ ```
81
+
82
+ ```bash
83
+ cd data-analyst-mcp && npm install && npm run build
84
+ ```
85
+
86
+ Then use `"command": "node", "args": ["<path>/data-analyst-mcp/dist/bin/cli.js"]` in the configs above, and
87
+ `node dist/bin/cli.js setup <source>` in place of `npx -y data-analyst-mcp setup <source>` below.
88
+
89
+ ### 2. Add keys (optional)
90
+
91
+ Each key is free. Without a key, a source either falls back to a keyless route with lower limits, or offers a
92
+ manual download (`manual_bundle`): the user downloads the files in a browser, and the server packages them.
93
+
94
+ | Variable | Sources | Without it | Free key |
95
+ |---|---|---|---|
96
+ | `AQS_API_KEY` + `AQS_EMAIL` | epa-aqs | manual download only | [aqs.epa.gov](https://aqs.epa.gov/data/api/signup?email=YOUR_EMAIL) |
97
+ | `BEA_API_KEY` | bea | bulk files for the main tables; the API-only datasets need the key | [apps.bea.gov](https://apps.bea.gov/API/signup/) |
98
+ | `BLS_API_KEY` | bls | 25 queries a day, 25 series and 10 years each; no annual averages or catalog titles | [data.bls.gov](https://data.bls.gov/registrationEngine/) |
99
+ | `CENSUS_API_KEY` | census | manual download only | [api.census.gov](https://api.census.gov/data/key_signup.html) |
100
+ | `DATA_GOV_API_KEY` | college-scorecard, congress, fbi, fda, fec, govinfo, nrel, regulations, usda-fooddata, usgs | `DEMO_KEY` or keyless, at a few requests an hour (fda: 1,000 a day; usgs earthquakes never need a key) | [api.data.gov](https://api.data.gov/signup/) |
101
+ | `DOL_API_KEY` | dol | catalog and field lists only; data queries need the key | [dataportal.dol.gov](https://dataportal.dol.gov/registration) |
102
+ | `EIA_API_KEY` | eia | `DEMO_KEY`, about 10 requests an hour | [eia.gov](https://www.eia.gov/opendata/register.php) |
103
+ | `FRED_API_KEY` | fred | observations only; search, vintages, revision history and GeoFRED need the key | [fredaccount.stlouisfed.org](https://fredaccount.stlouisfed.org/apikeys) |
104
+ | `HUD_USER_TOKEN` | hud | manual download only | [huduser.gov](https://www.huduser.gov/hudapi/public/register) |
105
+ | `IPUMS_API_KEY` | ipums | build the extract on the IPUMS website (free account) | [account.ipums.org](https://account.ipums.org/api_keys) |
106
+ | `LDA_API_KEY` | senate-lobbying | 15 requests a minute instead of 120 | [lda.gov](https://lda.gov/api/register/) |
107
+ | `NOAA_API_KEY` | noaa | the keyless data and station-search endpoints; Climate Data Online v2 needs the key | [ncei.noaa.gov](https://www.ncei.noaa.gov/cdo-web/token) |
108
+ | `USDA_NASS_API_KEY` | usda-nass | manual download only | [quickstats.nass.usda.gov](https://quickstats.nass.usda.gov/api) |
109
+ | `USPTO_API_KEY` | uspto | manual download only | [data.uspto.gov](https://data.uspto.gov/apis/getting-started) |
110
+
111
+ Two settings are not keys. SEC asks every client to identify itself, so set `SEC_USER_AGENT` to your name and email
112
+ (`"Jane Doe jane@example.com"`). `NWS_USER_AGENT` is optional and does the same for the weather service.
113
+
114
+ **Store a key with `setup` (recommended).** It shows the signup link, reads the key with hidden input, and writes
115
+ it to `~/.config/data-analyst/keys.json`. The key stays out of client config files, and every client reads it. It
116
+ works mid-session, with no restart.
117
+
118
+ ```bash
119
+ npx -y data-analyst-mcp setup congress
120
+ ```
121
+
122
+ `setup` takes any source that uses the key: `setup congress` stores `DATA_GOV_API_KEY`, and all 10 api.data.gov
123
+ sources use it. `setup epa-aqs` also asks for `AQS_EMAIL`. Or put the variables in the client's `env` block, as in step 1.
124
+ [API keys](#api-keys) lists every place the server looks.
125
+
126
+ ### 3. Check it
127
+
128
+ Ask your assistant: *"Is my api.data.gov key set up?"* The `check_key` tool reports whether the key was found and where,
129
+ never its value. To list only the sources you can use, start the server with `--hide-unconfigured` (see
130
+ [Options](#options)).
131
+
132
+ ### 4. Try it
133
+
134
+ - *"How has the unemployment rate compared with CPI inflation since 2020?"*
135
+ - *"What is the median household income in each county of Ohio?"*
136
+ - *"Which drugs had the most reported adverse events last year?"*
137
+ - *"How much did each federal agency spend in fiscal year 2024?"*
138
+ - *"How many unhealthy air-quality days did Phoenix have in 2023?"*
139
+
140
+ The assistant asks the questions that change the answer first (which measure, which years, which population),
141
+ then fetches, computes and cites.
142
+
143
+ To host the server for claude.ai or a team, see [docs/hosting.md](docs/hosting.md).
144
+
145
+ ## How it works
146
+
147
+ | Tool | What it does |
148
+ |---|---|
149
+ | `guide` | the method: framing, asking only critical questions, the accuracy checklist, combining sources |
150
+ | `find_sources` | ranks the sources for a question, or lists them all |
151
+ | `source_brief` | one source's playbook, its **questions_to_ask**, key status, rules and request index |
152
+ | `request_schema` | full parameters of the requests to use; searches a source's catalog |
153
+ | `record_decisions` | how each critical question was settled (answered, Don't know, skipped, not asked and why) |
154
+ | `fetch` | downloads into the session workspace; returns a summary, the files, the citation and notes |
155
+ | `manual_bundle`, `ingest` | a one-command download for the user's own computer, then packaging what they downloaded |
156
+ | `compute` | builds exhibits from the full tables with declarative steps (a whitelisted expression language, no eval); the steps are recorded for `rerun` |
157
+ | `facts`, `write_table`, `cite` | register the numbers cited, write the benchmark and sensitivity tables, snapshot cited pages |
158
+ | `audit_answer`, `rerun` | the release gate: every number in the answer is a registered fact, the method record is complete, and the analysis reproduces |
159
+ | `build_report`, `style_check`, `audit_report` | an HTML report and replication ZIP, only when asked |
160
+ | `handoff` | the per-source data hand-off |
161
+ | `check_key` | whether a key is set up (never shows it) |
162
+ | `read_file` | a small text file from the workspace |
163
+
164
+ **Asking before fetching is enforced, not just suggested.** `fetch` returns `gate_blocked` until, for that
165
+ source, the model has read `source_brief` and the schemas of the requests it uses, and has recorded how each
166
+ critical question was settled. The model asks the questions in chat, in one round. A blocking question that
167
+ the user skips is asked once more. The server also exposes an `analyze` prompt and `source://<id>`
168
+ resources, for clients that let users pick prompts or attach resources.
169
+
170
+ **Results** have one shape. Success: `{status: "ok" | "partial", summary, files, citation, notes}`. Failure:
171
+ `{status: "error", code, message, fix, retry_after_s?}`. The codes are `invalid_request`, `not_found`,
172
+ `not_supported`, `agency_rejected`, `agency_unavailable`, `key_missing`, `key_rejected`, `rate_limited`,
173
+ `gate_blocked` and `server_error`.
174
+
175
+ ## Options
176
+
177
+ | Option (flag / environment) | What it does |
178
+ |---|---|
179
+ | `--sources fred,treasury` / `DATA_ANALYST_SOURCES` | serve only these sources |
180
+ | `--domains economy,health` / `DATA_ANALYST_DOMAINS` | serve only the sources in these domains |
181
+ | `--hide-unconfigured` / `DATA_ANALYST_HIDE_UNCONFIGURED=1` | leave out of listings the sources whose required key is not set up |
182
+ | `--workspace <folder>` / `DATA_ANALYST_WORKSPACE` | (stdio) one fixed analysis folder, so a fetch is reused after a restart |
183
+
184
+ ```json
185
+ { "mcpServers": { "data-analyst": { "command": "npx", "args": ["-y", "data-analyst-mcp", "--domains", "economy,health"] } } }
186
+ ```
187
+
188
+ The tool list stays the same size whatever is enabled: sources are found with `find_sources`, not loaded as
189
+ tools.
190
+
191
+ Every result lists its upstream requests (`requests`: method, credential-free URL, status, cached, time), and
192
+ `citation` names the publisher, page and retrieval date. Requests are rate-limited per host (a token bucket),
193
+ retried with backoff on 429 and 5xx (honoring Retry-After), and kept within each agency's per-minute budgets.
194
+
195
+ **Caching follows each source's terms.**
196
+ - A source whose policy allows persistent copies (`cache.mode: persistent`) can be cached on disk in
197
+ `DATA_ANALYST_CACHE` (default `~/.cache/data-analyst`). census and bea do; opm's terms allow it, but its port
198
+ downloads again for now.
199
+ - Most sources allow reuse only inside the analysis folder, and FRED none at all. For those, a fixed
200
+ `--workspace` gives reuse across restarts without keeping copies anywhere else.
201
+
202
+ ## As a library
203
+
204
+ ```ts
205
+ import { route, brief, fetchSource, compute, registerFacts } from "data-analyst-mcp";
206
+
207
+ const r = await fetchSource("cdc", { description: "NY deaths 2017", requests: [{ name: "ny", endpoint: "query",
208
+ params: { dataset: "leading_death", $where: "state = 'New York' AND year = '2017'" } }] }, { out: "./out/ny" });
209
+ console.log(r.summary.tables[0].rows, r.citation.text);
210
+ ```
211
+
212
+ - **The functions:** the same sources, engine, analysis and audits as the server. The exports are `listSources`,
213
+ `route`, `brief`, `requestSchema`, `fetchSource`, `ingest`, `manualBundle`, `compute`, `registerFacts`,
214
+ `writeTable`, `rerun`, `recordDecisions`, `auditAnswer`, `buildReport`, `auditReport`, `createServer` and
215
+ `createApp`, plus their types.
216
+ - **No ask-first gate:** it is for assistants, so the library leaves it out.
217
+ - **Errors:** a failure throws `ToolError`; its `failure` holds the code and the fix.
218
+
219
+ ## API keys
220
+
221
+ [Install, step 2](#2-add-keys-optional) lists the keys and how to get them. When a fetch needs a key that is not set up,
222
+ `fetch` returns `key_missing` with a `setup` command for **you** to run in your own terminal. The model never asks
223
+ for a key in chat. `setup` writes `~/.config/data-analyst/keys.json`.
224
+
225
+ The local (stdio) server looks for a key in this order. `DATA_GOV_API_KEY` is the example; each source's variable is named in
226
+ its `source_brief`.
227
+
228
+ 1. The `DATA_GOV_API_KEY` environment variable, for example in the MCP client's `env` block.
229
+ 2. The file named by `DATA_GOV_API_KEY_FILE` (for secret managers and mounted secrets).
230
+ 3. `"env": {"DATA_GOV_API_KEY": "..."}` in `~/.config/data-analyst/keys.json` (or `$DATA_ANALYST_CONFIG_DIR/keys.json`).
231
+ `setup` writes this file.
232
+ 4. `"env": {"DATA_GOV_API_KEY": "..."}` in Claude Code's `~/.claude/settings.json` (or `$CLAUDE_CONFIG_DIR/settings.json`).
233
+ 5. `~/.data-gov/api_key`, a file that holds only the key.
234
+
235
+ The files are read again on each lookup, so a key stored during a session works without a restart.
236
+
237
+ No key value ever appears in a tool result: every output is redacted, and a test checks this. The hosted server
238
+ reads keys only from its environment and from each user's encrypted store (see [docs/hosting.md](docs/hosting.md)).
239
+
240
+ ## Hosted mode (claude.ai connector)
241
+
242
+ `data-analyst-mcp serve --http` serves MCP at `/mcp` over streamable HTTP. With `DATA_ANALYST_ACCOUNTS=1`, users
243
+ sign in through OAuth and store their own API keys on a write-only page. The keys are encrypted at rest, and the
244
+ model never sees them. See [docs/hosting.md](docs/hosting.md) and the `Dockerfile`.
245
+
246
+ ## Workspace
247
+
248
+ Each session writes to its own folder under `$DATA_ANALYST_OUT` (default: the system temp folder,
249
+ `data-analyst-mcp/<session>`), with `data/<source>/` and `decisions.json` inside. Nothing is written inside the
250
+ package.
251
+
252
+ ## Development
253
+
254
+ ```bash
255
+ npm install
256
+ npm test
257
+ npm run build
258
+ npm run smoke
259
+ ```
260
+
261
+ - `npm test` runs offline, on recorded fixtures.
262
+ - `npm run smoke` drives the built server over stdio and makes one live fetch.
@@ -436,10 +436,16 @@ export function checkMethod(R, base, facts) {
436
436
  "the request settles it";
437
437
  }
438
438
  else {
439
- const row = (sensRows || new Map()).get(a?.sensitivity || p);
439
+ const key = a?.sensitivity || p;
440
+ const row = (sensRows || new Map()).get(key);
440
441
  ok = !!strip(String(a?.why_not_empirical ?? "")) && row !== undefined && !!strip(String(row?.conclusion_changes ?? ""));
441
442
  why = "judgment is the last resort: estimate it from the data or adopt a published estimate; otherwise state " +
442
443
  "why_not_empirical and add its row to exhibits/sensitivity.csv (" + SENS_COLS.join(", ") + ")";
444
+ // "sensitivity" is the parameter of that row, not a description: say so when it names no row.
445
+ if (row === undefined && a?.sensitivity && sensRows?.size) {
446
+ why += `; "sensitivity" ${repr(key)} names no row: it is the row's parameter (${repr([...sensRows.keys()])}), ` +
447
+ "or leave it out when the row's parameter is the assumption's own";
448
+ }
443
449
  }
444
450
  res("assumptions", ok, ok ? `assumption ${repr(p)}: ${b} ok` : `assumption ${repr(p)} (${b}): ${why}`);
445
451
  }