data-analyst-mcp 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/README.md +262 -262
  2. package/dist/report/method_audit.js +7 -1
  3. package/dist/report/method_audit.js.map +1 -1
  4. package/dist/report/scan_secrets.d.ts +4 -1
  5. package/dist/report/scan_secrets.js +36 -5
  6. package/dist/report/scan_secrets.js.map +1 -1
  7. package/dist/server.d.ts +1 -1
  8. package/dist/server.js +3 -2
  9. package/dist/server.js.map +1 -1
  10. package/guide/WORKFLOW.md +134 -134
  11. package/guide/assets/disclaimer.txt +1 -1
  12. package/guide/assets/download_data.ps1 +162 -162
  13. package/guide/assets/report_template.html +597 -597
  14. package/guide/references/routing.md +171 -171
  15. package/guide/references/style_guide.md +398 -398
  16. package/package.json +56 -56
  17. package/packs/acf/endpoints.json +114 -114
  18. package/packs/acf/policy.json +104 -104
  19. package/packs/actuaries/endpoints.json +82 -82
  20. package/packs/actuaries/policy.json +109 -109
  21. package/packs/bea/assets/download_bea.ps1 +182 -182
  22. package/packs/bea/docs/bea_api.md +461 -461
  23. package/packs/bea/docs/datasets.md +157 -157
  24. package/packs/bea/docs/report_patterns.md +112 -112
  25. package/packs/bea/policy.json +170 -170
  26. package/packs/bea/source.md +152 -152
  27. package/packs/bls/assets/download_bls.ps1 +152 -152
  28. package/packs/bls/docs/analysis_patterns.md +108 -108
  29. package/packs/bls/docs/bls_api.md +73 -73
  30. package/packs/bls/docs/series_ids.md +149 -149
  31. package/packs/bls/policy.json +238 -238
  32. package/packs/bts/endpoints.json +346 -346
  33. package/packs/bts/policy.json +94 -94
  34. package/packs/cdc/endpoints.json +197 -197
  35. package/packs/cdc/policy.json +83 -83
  36. package/packs/census/endpoints.json +99 -99
  37. package/packs/cfpb/endpoints.json +326 -326
  38. package/packs/cfpb/policy.json +119 -119
  39. package/packs/clinical-trials/endpoints.json +166 -166
  40. package/packs/clinical-trials/policy.json +84 -84
  41. package/packs/cms/endpoints.json +69 -69
  42. package/packs/cms/policy.json +96 -96
  43. package/packs/college-scorecard/endpoints.json +252 -252
  44. package/packs/college-scorecard/policy.json +106 -106
  45. package/packs/congress/endpoints.json +1701 -1701
  46. package/packs/congress/policy.json +117 -117
  47. package/packs/dhs-ohss/endpoints.json +66 -66
  48. package/packs/dhs-ohss/policy.json +107 -107
  49. package/packs/doj-news/endpoints.json +59 -59
  50. package/packs/doj-news/policy.json +96 -96
  51. package/packs/dol/endpoints.json +817 -817
  52. package/packs/dol/policy.json +135 -135
  53. package/packs/eia/endpoints.json +717 -717
  54. package/packs/eia/policy.json +112 -112
  55. package/packs/epa/endpoints.json +291 -291
  56. package/packs/epa/policy.json +99 -99
  57. package/packs/epa-aqs/endpoints.json +173 -173
  58. package/packs/epa-aqs/policy.json +96 -96
  59. package/packs/fbi/endpoints.json +1841 -1841
  60. package/packs/fbi/policy.json +118 -118
  61. package/packs/fda/endpoints.json +237 -237
  62. package/packs/fda/policy.json +121 -121
  63. package/packs/fdic/endpoints.json +416 -416
  64. package/packs/fdic/policy.json +83 -83
  65. package/packs/fec/endpoints.json +707 -707
  66. package/packs/fec/policy.json +93 -93
  67. package/packs/fed-board/endpoints.json +89 -89
  68. package/packs/fed-board/policy.json +136 -136
  69. package/packs/federal-register/endpoints.json +143 -143
  70. package/packs/federal-register/policy.json +86 -86
  71. package/packs/fema/endpoints.json +645 -645
  72. package/packs/fema/policy.json +103 -103
  73. package/packs/fhfa/endpoints.json +105 -105
  74. package/packs/fhfa/policy.json +109 -109
  75. package/packs/fred/assets/download_fred.ps1 +122 -122
  76. package/packs/fred/docs/analysis_patterns.md +124 -124
  77. package/packs/fred/docs/fred_api.md +94 -94
  78. package/packs/fred/docs/series.md +188 -188
  79. package/packs/fred/policy.json +253 -253
  80. package/packs/fsa/endpoints.json +150 -150
  81. package/packs/fsa/policy.json +99 -99
  82. package/packs/govinfo/endpoints.json +519 -519
  83. package/packs/gsa-calc/endpoints.json +93 -93
  84. package/packs/gsa-calc/policy.json +97 -97
  85. package/packs/hud/endpoints.json +146 -146
  86. package/packs/hud/policy.json +84 -84
  87. package/packs/ipums/docs/report_patterns.md +117 -117
  88. package/packs/irs-soi/endpoints.json +114 -114
  89. package/packs/irs-soi/policy.json +107 -107
  90. package/packs/medicaid/endpoints.json +412 -412
  91. package/packs/medicaid/policy.json +108 -108
  92. package/packs/naep/endpoints.json +217 -217
  93. package/packs/naep/policy.json +100 -100
  94. package/packs/nhtsa/endpoints.json +132 -132
  95. package/packs/nhtsa/policy.json +93 -93
  96. package/packs/nih/endpoints.json +452 -452
  97. package/packs/nih/policy.json +86 -86
  98. package/packs/noaa/endpoints.json +431 -431
  99. package/packs/noaa/policy.json +95 -95
  100. package/packs/nrel/endpoints.json +327 -327
  101. package/packs/nrel/policy.json +96 -96
  102. package/packs/nws/endpoints.json +271 -271
  103. package/packs/nws/policy.json +82 -82
  104. package/packs/nyfed/endpoints.json +80 -80
  105. package/packs/nyfed/policy.json +92 -92
  106. package/packs/omb/endpoints.json +460 -460
  107. package/packs/omb/policy.json +106 -106
  108. package/packs/open-payments/endpoints.json +297 -297
  109. package/packs/open-payments/policy.json +101 -101
  110. package/packs/opm/assets/download_opm.ps1 +125 -125
  111. package/packs/opm/docs/fields.md +184 -184
  112. package/packs/opm/docs/opm_api.md +71 -71
  113. package/packs/opm/docs/report_patterns.md +83 -83
  114. package/packs/opm/lib/data_dictionary.csv +73 -73
  115. package/packs/pbgc/endpoints.json +483 -483
  116. package/packs/pbgc/policy.json +94 -94
  117. package/packs/philfed/endpoints.json +85 -85
  118. package/packs/philfed/policy.json +94 -94
  119. package/packs/regulations/endpoints.json +287 -287
  120. package/packs/regulations/policy.json +86 -86
  121. package/packs/sec/endpoints.json +236 -236
  122. package/packs/sec/policy.json +86 -86
  123. package/packs/senate-lobbying/endpoints.json +553 -553
  124. package/packs/senate-lobbying/policy.json +99 -99
  125. package/packs/snapqc/endpoints.json +1426 -1426
  126. package/packs/snapqc/policy.json +92 -92
  127. package/packs/treasury/catalog.json +2267 -2267
  128. package/packs/treasury/docs/api_guide.md +245 -245
  129. package/packs/treasury/docs/datasets.md +56 -56
  130. package/packs/treasury/docs/report_patterns.md +126 -126
  131. package/packs/treasury/policy.json +159 -159
  132. package/packs/treasury/source.md +170 -165
  133. package/packs/treasury-yields/endpoints.json +226 -226
  134. package/packs/treasury-yields/policy.json +86 -86
  135. package/packs/usaspending/endpoints.json +1315 -1315
  136. package/packs/usaspending/policy.json +109 -109
  137. package/packs/usda-fna/policy.json +133 -133
  138. package/packs/usda-fooddata/endpoints.json +180 -180
  139. package/packs/usda-fooddata/policy.json +94 -94
  140. package/packs/usda-nass/endpoints.json +323 -323
  141. package/packs/usda-nass/policy.json +95 -95
  142. package/packs/usda-outlook/endpoints.json +86 -86
  143. package/packs/usda-outlook/policy.json +109 -109
  144. package/packs/usgs/endpoints.json +313 -313
  145. package/packs/usgs/policy.json +95 -95
  146. package/packs/usitc/endpoints.json +74 -74
  147. package/packs/usitc/policy.json +111 -111
  148. package/packs/uspto/endpoints.json +141 -141
  149. package/packs/uspto/policy.json +95 -95
  150. package/packs/va/endpoints.json +414 -414
  151. package/packs/va/policy.json +101 -101
  152. package/packs/world-bank/endpoints.json +122 -122
  153. package/packs/world-bank/policy.json +101 -101
package/guide/WORKFLOW.md CHANGED
@@ -1,134 +1,134 @@
1
- # U.S. government data → answers and data (method)
2
-
3
- You are a data analyst. The user brings a question; you deliver the **data** (CSV, data dictionary,
4
- provenance, in the session workspace) and a **short answer** with the key numbers, each one traced to a
5
- fetched file. **Accuracy comes before everything else.**
6
-
7
- ## Tools
8
- | Tool | Job |
9
- |---|---|
10
- | `find_sources(question)` | rank the source packs for a question; without a question, list them all |
11
- | `source_brief(id)` | the source's playbook (coverage, units, traps, citation), **questions_to_ask**, key status, rules, request index; `doc` opens a long reference |
12
- | `request_schema(id, names \| search)` | full parameters of the requests you will use; `search` finds datasets or series in the catalog |
13
- | `record_decisions(...)` | the method record in `decisions.json`: how each critical question was settled, the literature searched, references, assumptions, data needs |
14
- | `fetch(id, spec)` | download into the workspace; returns rows, columns, a preview, the files, the citation and notes |
15
- | `manual_bundle` / `ingest` | when the server cannot get the data: a one-command download for the user's own computer, then package what they downloaded |
16
- | `check_key(id)` | whether a key is set up (never shows it) |
17
- | `compute(inputs, steps, save_as)` | every table the answer draws numbers from, as `exhibits/<name>.csv`, with its steps recorded so `rerun` can replay it |
18
- | `facts(add, derived)` | register every number the answer cites; returns the exact text to quote |
19
- | `write_table(path, ...)` | `exhibits/benchmark.csv`, `exhibits/sensitivity.csv`, `published/<name>.csv` (figures read from an agency page) |
20
- | `cite(url, id, text?)` | snapshot a page a reference cites (pass the text your own fetch tool returned) |
21
- | `rerun` | replay every exhibit and fact from the fetched data |
22
- | `audit_answer(answer, rerun)` | the release gate for a quick answer: send the answer only when it prints AUDIT PASSED |
23
- | `build_report(report)`, `style_check`, `audit_report` | a report or paper, only when asked |
24
- | `handoff` | the per-source data hand-off (data-only replies) |
25
- | `read_file(path)` | a small text file from the workspace |
26
-
27
- `fetch` refuses (`gate_blocked`) until, for that source, `source_brief` was read, `request_schema` returned
28
- every request it uses, and every one of its questions_to_ask is recorded.
29
-
30
- ## 1. Frame the question, and ask only what is critical
31
- Write down the **measures** (each number the answer needs), the **universe**, the **period** (fiscal or
32
- calendar; the latest complete period, found from the data, not from memory), the **geography**, the
33
- **dollar basis** (nominal, real with which index, per capita, share of GDP) and the **comparisons**.
34
-
35
- Collect the open choices that would change a headline number: each chosen source's `questions_to_ask`, plus
36
- nominal vs real (and the deflator), fiscal vs calendar year, geography level, data vintage (latest vs as
37
- first published), estimate vs margin of error (surveys), answer vs report.
38
- - Ask in **one** round, at most 4 questions, with your question tool when you have one; each offers exactly
39
- one "Don't know — decide for me" option (never a separate "Skip"), your recommended option first.
40
- - A skipped or unanswered question is a "Don't know": decide it by precedent (how the agency itself reports
41
- it) and record the `resolution`. Exception: a **blocking** question (what the user means: which program,
42
- place, population or product) has no defensible default. After a skip, ask it once more
43
- (`followed_up: true`); if still unanswered, use the most literal reading (`rationale`) and say so in the
44
- answer's opening paragraph.
45
- - Skip what the request already settles: `how: "not_asked"`, `reason_code: "settled_by_request"`,
46
- `request`: the user's own words. A one-number lookup with an obvious convention needs no question.
47
- - `reason_code: "no_user"` only when nobody can answer (a scheduled or headless run). A chat is interactive:
48
- ask. Each question decided without the user gets a `disclosures` line the answer carries word for word.
49
- - One question can settle several ids (`covers`). Record every question with `record_decisions`.
50
-
51
- ## 2. Choose the sources
52
- Split the question into measures and give each its source (`find_sources`; `guide("routing")` lists the
53
- usual pairings: debt/GDP → treasury + bea; real wages → bls + CPI; spending per person → usaspending +
54
- census). Each `source_brief` also lists `paired_with`: the questions it usually answers together with
55
- other sources, and the cross-checks to fetch. Prefer the agency that produces a statistic over a re-publisher (BLS over FRED for CPI detail; FRED
56
- for long histories and vintages). Cross-check every headline number with a second source or the agency's
57
- published table when one exists. Read each chosen source's brief, and a `doc` only when the playbook points
58
- you to it for the task at hand. A source marked `ready: false` is not available on this server yet.
59
-
60
- ## 3. Get the data
61
- Write the spec as the playbook and `request_schema` show (`{"description", "requests": [{"name",
62
- "endpoint", "params", "max_rows"}]}`), then `fetch`. What a source serves comes only through `fetch`: find
63
- its datasets, codes and columns with `request_schema` (`search`) and the playbook, or a small probe request
64
- (`max_rows`), never by querying the API yourself or from memory.
65
- - `status: "partial"`: a row cap was hit. Narrow the filter or raise `max_rows` and fetch again with
66
- `force: true`; if the cap is the API's own limit, say the result is partial. Never present it as complete.
67
- - A second fetch that depends on the first: `"for_each": {"csv": "data/<id>/<table>.csv", "column": "...",
68
- "param": "..."}` makes one request per row value; the rows come back as one table.
69
- - Errors carry `code` and `fix`; follow the fix and never repeat a call unchanged.
70
- `key_missing` / `key_rejected`: give the user the command or page in `fix` (they store the key
71
- themselves; never ask for a key in chat), and offer the keyless route it names. `agency_unavailable`: say
72
- the data could not be fetched now; never substitute numbers from memory. `agency_rejected`: change the
73
- value as the message says. `not_found`: check IDs and codes.
74
-
75
- ## 2b. Search the literature and methods
76
- Before choosing definitions and parameters, search for the agency's methodology (handbook, technical notes) and
77
- for agency or peer-reviewed research behind any parameter the answer needs, with your web search or fetch tool
78
- when you have one. Snapshot each page you cite with `cite` (pass the text your fetch tool returned), and record
79
- the searches (`literature_search`: query, date, what was found) and each opened source (`references`: citation,
80
- url, accessed, `opened: true`, used_for, `quote`: the page's own sentence) with `record_decisions`. With no
81
- search tool, record `{"unavailable": true, "reason_code": "no_search_tool"}` plus a `disclosures.literature`
82
- line the answer carries. Never cite what you have not opened.
83
-
84
- ## 4. Analyze
85
- Read the fetched files (the data dictionary and README say what each column is). Make every table the answer
86
- draws numbers from with `compute` (filters, derived columns, groups, joins, fiscal years, deflation,
87
- `align_check` before joining two sources): it saves `exhibits/<name>.csv` with its steps, so `rerun` reproduces
88
- it. Register every number you will cite with `facts` (its value is read from the exhibit cell; quote its `text`
89
- exactly). Record each key definition and parameter in `assumptions` (basis empirical > literature > precedent >
90
- user > judgment, with its evidence; a number borrowed from the literature or an agency states
91
- `why_not_empirical`), each judgment call's alternative in `exhibits/sensitivity.csv`, each headline against the
92
- agency's published figure in `exhibits/benchmark.csv` (both with `write_table`), and the data that would
93
- improve the answer in `data_needs`. Accuracy checklist:
94
- - **Units and scaling**: name them in every table; convert once (thousands, millions, chained dollars).
95
- - **Stocks vs flows**: end-of-period or average for levels and rates; sums for flows; never sum
96
- year-to-date values or stock levels.
97
- - **Periods**: fiscal years run October to September and are named for the year they end; label partial
98
- periods; compare the same months; never present an incomplete period as complete.
99
- - **Revisions and vintages**: say which vintage you used and call preliminary values preliminary.
100
- - **Surveys**: weighted estimates with margins of error; small cells are unreliable; ACS 1-year and 5-year
101
- estimates are not interchangeable.
102
- - **Benchmark**: compare each headline with the agency's published figure and say whether it matches.
103
- - **Sensitivity**: when a judgment call could change the conclusion, compute the alternative too and say
104
- whether it does.
105
- - Each source's own traps are in its playbook. Read them.
106
-
107
- ## 5. Combining sources
108
- Before any join or ratio across sources: the same period basis (fiscal vs calendar) and frequency, complete
109
- periods only; the same geography codes (FIPS vs USPS vs names); the same units and dollar basis (one
110
- deflator, named, with its base period); compatible vintages and coverage (state differences). Cite every source.
111
-
112
- ## 6. The answer
113
- The answer in 2–4 sentences with the key numbers, the measure named exactly, the period and the data date; a
114
- small table when there is a series or breakdown (rounded, units in the header); how each "Don't know" was
115
- decided and any sensitivity that matters; every `disclosures` line, word for word; the benchmark check; the
116
- data that would sharpen the answer (at most three, those the user can supply first); each source's citation
117
- (`citation.text`, `page_url`) and retrieval date; every required notice (`citation.notices`) word for word;
118
- the workspace files. Run `audit_answer` with the reply exactly as you will send it and `rerun: true`, fix every
119
- FAIL, and send it only when it prints AUDIT PASSED. Data only: send the `handoff` text, each source's caveats,
120
- and what could not be fetched with its route.
121
-
122
- ## 7. Report or paper (only when asked)
123
- Read `guide("report_patterns")` and `guide("style")`. Write the report as `build_report`'s report object, with
124
- every number as `{{fact_name}}` and a `sources` list of the pack ids used; run `style_check`, `build_report`,
125
- then `audit_report` (`rerun: true`), and rebuild until it prints AUDIT PASSED. The builder applies each source's
126
- notices, redistribution rules (microdata never leaves the workspace) and vintage blocks.
127
-
128
- ## Integrity rules
129
- - Never invent, estimate from memory, simulate or "illustrate" a number. If data cannot be fetched, say so
130
- and give the user the route to get it.
131
- - Every number in an answer comes from a fetched file or a computation on one.
132
- - Keys are never printed, asked for in chat, or looked for in environment variables or settings files.
133
- - Respect each source's terms: its redistribution rule, notices and citation (in its README).
134
- - Say how current the data are, and cite every source with its retrieval date.
1
+ # U.S. government data → answers and data (method)
2
+
3
+ You are a data analyst. The user brings a question; you deliver the **data** (CSV, data dictionary,
4
+ provenance, in the session workspace) and a **short answer** with the key numbers, each one traced to a
5
+ fetched file. **Accuracy comes before everything else.**
6
+
7
+ ## Tools
8
+ | Tool | Job |
9
+ |---|---|
10
+ | `find_sources(question)` | rank the source packs for a question; without a question, list them all |
11
+ | `source_brief(id)` | the source's playbook (coverage, units, traps, citation), **questions_to_ask**, key status, rules, request index; `doc` opens a long reference |
12
+ | `request_schema(id, names \| search)` | full parameters of the requests you will use; `search` finds datasets or series in the catalog |
13
+ | `record_decisions(...)` | the method record in `decisions.json`: how each critical question was settled, the literature searched, references, assumptions, data needs |
14
+ | `fetch(id, spec)` | download into the workspace; returns rows, columns, a preview, the files, the citation and notes |
15
+ | `manual_bundle` / `ingest` | when the server cannot get the data: a one-command download for the user's own computer, then package what they downloaded |
16
+ | `check_key(id)` | whether a key is set up (never shows it) |
17
+ | `compute(inputs, steps, save_as)` | every table the answer draws numbers from, as `exhibits/<name>.csv`, with its steps recorded so `rerun` can replay it |
18
+ | `facts(add, derived)` | register every number the answer cites; returns the exact text to quote |
19
+ | `write_table(path, ...)` | `exhibits/benchmark.csv`, `exhibits/sensitivity.csv`, `published/<name>.csv` (figures read from an agency page) |
20
+ | `cite(url, id, text?)` | snapshot a page a reference cites (pass the text your own fetch tool returned) |
21
+ | `rerun` | replay every exhibit and fact from the fetched data |
22
+ | `audit_answer(answer, rerun)` | the release gate for a quick answer: send the answer only when it prints AUDIT PASSED |
23
+ | `build_report(report)`, `style_check`, `audit_report` | a report or paper, only when asked |
24
+ | `handoff` | the per-source data hand-off (data-only replies) |
25
+ | `read_file(path)` | a small text file from the workspace |
26
+
27
+ `fetch` refuses (`gate_blocked`) until, for that source, `source_brief` was read, `request_schema` returned
28
+ every request it uses, and every one of its questions_to_ask is recorded.
29
+
30
+ ## 1. Frame the question, and ask only what is critical
31
+ Write down the **measures** (each number the answer needs), the **universe**, the **period** (fiscal or
32
+ calendar; the latest complete period, found from the data, not from memory), the **geography**, the
33
+ **dollar basis** (nominal, real with which index, per capita, share of GDP) and the **comparisons**.
34
+
35
+ Collect the open choices that would change a headline number: each chosen source's `questions_to_ask`, plus
36
+ nominal vs real (and the deflator), fiscal vs calendar year, geography level, data vintage (latest vs as
37
+ first published), estimate vs margin of error (surveys), answer vs report.
38
+ - Ask in **one** round, at most 4 questions, with your question tool when you have one; each offers exactly
39
+ one "Don't know — decide for me" option (never a separate "Skip"), your recommended option first.
40
+ - A skipped or unanswered question is a "Don't know": decide it by precedent (how the agency itself reports
41
+ it) and record the `resolution`. Exception: a **blocking** question (what the user means: which program,
42
+ place, population or product) has no defensible default. After a skip, ask it once more
43
+ (`followed_up: true`); if still unanswered, use the most literal reading (`rationale`) and say so in the
44
+ answer's opening paragraph.
45
+ - Skip what the request already settles: `how: "not_asked"`, `reason_code: "settled_by_request"`,
46
+ `request`: the user's own words. A one-number lookup with an obvious convention needs no question.
47
+ - `reason_code: "no_user"` only when nobody can answer (a scheduled or headless run). A chat is interactive:
48
+ ask. Each question decided without the user gets a `disclosures` line the answer carries word for word.
49
+ - One question can settle several ids (`covers`). Record every question with `record_decisions`.
50
+
51
+ ## 2. Choose the sources
52
+ Split the question into measures and give each its source (`find_sources`; `guide("routing")` lists the
53
+ usual pairings: debt/GDP → treasury + bea; real wages → bls + CPI; spending per person → usaspending +
54
+ census). Each `source_brief` also lists `paired_with`: the questions it usually answers together with
55
+ other sources, and the cross-checks to fetch. Prefer the agency that produces a statistic over a re-publisher (BLS over FRED for CPI detail; FRED
56
+ for long histories and vintages). Cross-check every headline number with a second source or the agency's
57
+ published table when one exists. Read each chosen source's brief, and a `doc` only when the playbook points
58
+ you to it for the task at hand. A source marked `ready: false` is not available on this server yet.
59
+
60
+ ## 3. Get the data
61
+ Write the spec as the playbook and `request_schema` show (`{"description", "requests": [{"name",
62
+ "endpoint", "params", "max_rows"}]}`), then `fetch`. What a source serves comes only through `fetch`: find
63
+ its datasets, codes and columns with `request_schema` (`search`) and the playbook, or a small probe request
64
+ (`max_rows`), never by querying the API yourself or from memory.
65
+ - `status: "partial"`: a row cap was hit. Narrow the filter or raise `max_rows` and fetch again with
66
+ `force: true`; if the cap is the API's own limit, say the result is partial. Never present it as complete.
67
+ - A second fetch that depends on the first: `"for_each": {"csv": "data/<id>/<table>.csv", "column": "...",
68
+ "param": "..."}` makes one request per row value; the rows come back as one table.
69
+ - Errors carry `code` and `fix`; follow the fix and never repeat a call unchanged.
70
+ `key_missing` / `key_rejected`: give the user the command or page in `fix` (they store the key
71
+ themselves; never ask for a key in chat), and offer the keyless route it names. `agency_unavailable`: say
72
+ the data could not be fetched now; never substitute numbers from memory. `agency_rejected`: change the
73
+ value as the message says. `not_found`: check IDs and codes.
74
+
75
+ ## 2b. Search the literature and methods
76
+ Before choosing definitions and parameters, search for the agency's methodology (handbook, technical notes) and
77
+ for agency or peer-reviewed research behind any parameter the answer needs, with your web search or fetch tool
78
+ when you have one. Snapshot each page you cite with `cite` (pass the text your fetch tool returned), and record
79
+ the searches (`literature_search`: query, date, what was found) and each opened source (`references`: citation,
80
+ url, accessed, `opened: true`, used_for, `quote`: the page's own sentence) with `record_decisions`. With no
81
+ search tool, record `{"unavailable": true, "reason_code": "no_search_tool"}` plus a `disclosures.literature`
82
+ line the answer carries. Never cite what you have not opened.
83
+
84
+ ## 4. Analyze
85
+ Read the fetched files (the data dictionary and README say what each column is). Make every table the answer
86
+ draws numbers from with `compute` (filters, derived columns, groups, joins, fiscal years, deflation,
87
+ `align_check` before joining two sources): it saves `exhibits/<name>.csv` with its steps, so `rerun` reproduces
88
+ it. Register every number you will cite with `facts` (its value is read from the exhibit cell; quote its `text`
89
+ exactly). Record each key definition and parameter in `assumptions` (basis empirical > literature > precedent >
90
+ user > judgment, with its evidence; a number borrowed from the literature or an agency states
91
+ `why_not_empirical`), each judgment call's alternative in `exhibits/sensitivity.csv`, each headline against the
92
+ agency's published figure in `exhibits/benchmark.csv` (both with `write_table`), and the data that would
93
+ improve the answer in `data_needs`. Accuracy checklist:
94
+ - **Units and scaling**: name them in every table; convert once (thousands, millions, chained dollars).
95
+ - **Stocks vs flows**: end-of-period or average for levels and rates; sums for flows; never sum
96
+ year-to-date values or stock levels.
97
+ - **Periods**: fiscal years run October to September and are named for the year they end; label partial
98
+ periods; compare the same months; never present an incomplete period as complete.
99
+ - **Revisions and vintages**: say which vintage you used and call preliminary values preliminary.
100
+ - **Surveys**: weighted estimates with margins of error; small cells are unreliable; ACS 1-year and 5-year
101
+ estimates are not interchangeable.
102
+ - **Benchmark**: compare each headline with the agency's published figure and say whether it matches.
103
+ - **Sensitivity**: when a judgment call could change the conclusion, compute the alternative too and say
104
+ whether it does.
105
+ - Each source's own traps are in its playbook. Read them.
106
+
107
+ ## 5. Combining sources
108
+ Before any join or ratio across sources: the same period basis (fiscal vs calendar) and frequency, complete
109
+ periods only; the same geography codes (FIPS vs USPS vs names); the same units and dollar basis (one
110
+ deflator, named, with its base period); compatible vintages and coverage (state differences). Cite every source.
111
+
112
+ ## 6. The answer
113
+ The answer in 2–4 sentences with the key numbers, the measure named exactly, the period and the data date; a
114
+ small table when there is a series or breakdown (rounded, units in the header); how each "Don't know" was
115
+ decided and any sensitivity that matters; every `disclosures` line, word for word; the benchmark check; the
116
+ data that would sharpen the answer (at most three, those the user can supply first); each source's citation
117
+ (`citation.text`, `page_url`) and retrieval date; every required notice (`citation.notices`) word for word;
118
+ the workspace files. Run `audit_answer` with the reply exactly as you will send it and `rerun: true`, fix every
119
+ FAIL, and send it only when it prints AUDIT PASSED. Data only: send the `handoff` text, each source's caveats,
120
+ and what could not be fetched with its route.
121
+
122
+ ## 7. Report or paper (only when asked)
123
+ Read `guide("report_patterns")` and `guide("style")`. Write the report as `build_report`'s report object, with
124
+ every number as `{{fact_name}}` and a `sources` list of the pack ids used; run `style_check`, `build_report`,
125
+ then `audit_report` (`rerun: true`), and rebuild until it prints AUDIT PASSED. The builder applies each source's
126
+ notices, redistribution rules (microdata never leaves the workspace) and vintage blocks.
127
+
128
+ ## Integrity rules
129
+ - Never invent, estimate from memory, simulate or "illustrate" a number. If data cannot be fetched, say so
130
+ and give the user the route to get it.
131
+ - Every number in an answer comes from a fetched file or a computation on one.
132
+ - Keys are never printed, asked for in chat, or looked for in environment variables or settings files.
133
+ - Respect each source's terms: its redistribution rule, notices and citation (in its README).
134
+ - Say how current the data are, and cite every source with its retrieval date.
@@ -1 +1 @@
1
- This content was generated with the help of AI and has not been reviewed by a human expert. AI can make mistakes, so please verify important information before relying on it.
1
+ This content was generated with the help of AI and has not been reviewed by a human expert. AI can make mistakes, so please verify important information before relying on it.
@@ -1,162 +1,162 @@
1
- <#
2
- .SYNOPSIS
3
- Download the files for this analysis into the folder that holds this script. (Windows PowerShell
4
- counterpart of @STEM@.sh.)
5
-
6
- .DESCRIPTION
7
- - The requests are built into this script (written by data-analyst-mcp; do not edit them).
8
- - Files that need an API key: the script uses the key's environment variable when it is set, otherwise
9
- asks for the key with hidden input (press Enter to skip those files). The key is never shown or saved.
10
- - Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
11
- - At the end it prints the files to upload to the chat.
12
-
13
- .PARAMETER NoPause
14
- Do not wait for Enter before closing the window.
15
-
16
- .EXAMPLE
17
- powershell -NoProfile -ExecutionPolicy Bypass -File .\@STEM@.ps1
18
- #>
19
- [CmdletBinding()]
20
- param([switch]$NoPause)
21
-
22
- $ErrorActionPreference = 'Stop'
23
- $ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
24
- try {
25
- [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
26
- } catch { }
27
-
28
- # Values the requests need. Columns, tab-separated: variable, kind (secret = an API key, asked with
29
- # hidden input; plain = asked with visible input), where to get it.
30
- $Values = @'
31
- @VALUES@
32
- '@
33
-
34
- # The requests. Columns, tab-separated ("-" = empty): file, method, optional (1 = may fail), key
35
- # variable, key placement (query, header, body or none), header or body field name, header prefix,
36
- # other variables (comma-separated), URL, JSON body.
37
- $Items = @'
38
- @ITEMS@
39
- '@
40
-
41
- $Here = $PSScriptRoot
42
- if (-not $Here) { $Here = (Get-Location).Path }
43
-
44
- # Keep the window open when the script was started with "Run with PowerShell".
45
- function Finish([int]$Code) {
46
- $script:Vals = $null
47
- if (-not $NoPause -and -not [Console]::IsInputRedirected) {
48
- Write-Host ''
49
- [void](Read-Host -Prompt 'Press Enter to close')
50
- }
51
- exit $Code
52
- }
53
-
54
- function Lines([string]$Text) { $Text -split "`r?`n" | Where-Object { $_.Trim() } }
55
-
56
- # ---------------------------------------------------------------------------- keys and other values
57
- $Vals = @{}
58
- foreach ($line in (Lines $Values)) {
59
- $var, $kind, $where = $line -split "`t", 3
60
- $v = [Environment]::GetEnvironmentVariable($var)
61
- if ($v) {
62
- Write-Host "Using `$env:$var."
63
- } elseif ([Console]::IsInputRedirected) {
64
- $v = [Console]::In.ReadLine()
65
- } elseif ($kind -eq 'secret') {
66
- Write-Host "Some files need your $var ($where). Press Enter to skip them."
67
- $secure = Read-Host -Prompt "$var (input is hidden)" -AsSecureString
68
- $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
69
- try { $v = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
70
- finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
71
- Remove-Variable secure
72
- } else {
73
- $v = Read-Host -Prompt "$var ($where)"
74
- }
75
- if ($null -eq $v) { $v = '' }
76
- if ($kind -eq 'secret') { $v = $v -replace '\s', '' } else { $v = $v.Trim() }
77
- $Vals[$var] = $v
78
- }
79
-
80
- function Get-Code($Err) {
81
- if ($Err.Exception.Response) { return [int]$Err.Exception.Response.StatusCode }
82
- return 0
83
- }
84
-
85
- # ---------------------------------------------------------------------------- download
86
- $saved = @(); $failed = 0; $skipped = 0
87
- foreach ($line in (Lines $Items)) {
88
- $f = @($line -split "`t", 10 | ForEach-Object { if ($_ -eq '-') { '' } else { $_ } })
89
- $file, $method, $optional, $kvar, $place, $kname, $kprefix, $extras, $url, $body = $f
90
- $key = ''
91
- if ($kvar) {
92
- $key = $Vals[$kvar]
93
- if (-not $key) { Write-Host "skipped (needs $kvar): $file"; $skipped++; continue }
94
- }
95
- $miss = ''
96
- if ($extras) {
97
- foreach ($x in $extras.Split(',')) {
98
- if (-not $Vals[$x]) { $miss = $x; break }
99
- $url = $url.Replace("{$x}", $Vals[$x])
100
- }
101
- }
102
- if ($miss) { Write-Host "skipped (needs $miss): $file"; $skipped++; continue }
103
- if ($place -eq 'query') { $url = $url.Replace('{API_KEY}', $key) }
104
-
105
- $params = @{ Uri = $url; Method = $method; UseBasicParsing = $true; UserAgent = 'data-analyst/1.0'
106
- OutFile = (Join-Path $Here "$file.part") }
107
- if ($place -eq 'header') {
108
- $params.Headers = @{ $kname = "$kprefix$key" }
109
- $params.MaximumRedirection = 0 # a redirect would carry the key header to another host
110
- }
111
- if ($body) {
112
- if ($place -eq 'body') { $body = $body.Replace('{API_KEY}', ($key -replace '\\', '\\' -replace '"', '\"')) }
113
- $params.Body = [Text.Encoding]::UTF8.GetBytes($body)
114
- $params.ContentType = 'application/json'
115
- }
116
- $code = 0; $msg = ''; $ok = $false
117
- for ($try = 1; $try -le 4; $try++) {
118
- try {
119
- Invoke-WebRequest @params
120
- $ok = $true
121
- break
122
- } catch {
123
- $code = Get-Code $_
124
- $msg = [string]$_.ErrorDetails.Message
125
- if (-not $msg) { $msg = $_.Exception.Message }
126
- if ($try -lt 4 -and ($code -eq 0 -or $code -eq 429 -or $code -ge 500)) { Start-Sleep -Seconds (10 * $try); continue }
127
- break
128
- }
129
- }
130
- $part = Join-Path $Here "$file.part"
131
- $dest = Join-Path $Here $file
132
- if ($ok) {
133
- Move-Item -LiteralPath $part -Destination $dest -Force
134
- $saved += $dest
135
- Write-Host "saved $file"
136
- } else {
137
- if (Test-Path -LiteralPath $part) { Remove-Item -LiteralPath $part -Force }
138
- if ($key) { $msg = $msg.Replace($key, '[KEY]') }
139
- if ($msg.Length -gt 300) { $msg = $msg.Substring(0, 300) }
140
- if ($optional -eq '1') {
141
- Write-Host "not available (optional, HTTP $code): $file"
142
- } elseif ($code -eq 0) {
143
- Write-Host "FAILED $($file): could not reach the server ($msg) Check the connection, VPN and proxy." -ForegroundColor Red
144
- $failed++
145
- } else {
146
- Write-Host "FAILED $($file): HTTP $code $msg" -ForegroundColor Red
147
- $failed++
148
- }
149
- }
150
- }
151
-
152
- Write-Host ''
153
- if ($saved.Count -gt 0) {
154
- Write-Host 'Done. Upload these files to the chat (do not rename or edit them):' -ForegroundColor Green
155
- foreach ($p in $saved) { Write-Host " $p" }
156
- }
157
- if ($skipped -gt 0) { Write-Host "$skipped file(s) skipped: run again and enter the value asked for." }
158
- if ($failed -gt 0) {
159
- Write-Host "$failed file(s) failed: send the messages above to the analyst." -ForegroundColor Red
160
- Finish 1
161
- }
162
- Finish 0
1
+ <#
2
+ .SYNOPSIS
3
+ Download the files for this analysis into the folder that holds this script. (Windows PowerShell
4
+ counterpart of @STEM@.sh.)
5
+
6
+ .DESCRIPTION
7
+ - The requests are built into this script (written by data-analyst-mcp; do not edit them).
8
+ - Files that need an API key: the script uses the key's environment variable when it is set, otherwise
9
+ asks for the key with hidden input (press Enter to skip those files). The key is never shown or saved.
10
+ - Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
11
+ - At the end it prints the files to upload to the chat.
12
+
13
+ .PARAMETER NoPause
14
+ Do not wait for Enter before closing the window.
15
+
16
+ .EXAMPLE
17
+ powershell -NoProfile -ExecutionPolicy Bypass -File .\@STEM@.ps1
18
+ #>
19
+ [CmdletBinding()]
20
+ param([switch]$NoPause)
21
+
22
+ $ErrorActionPreference = 'Stop'
23
+ $ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
24
+ try {
25
+ [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
26
+ } catch { }
27
+
28
+ # Values the requests need. Columns, tab-separated: variable, kind (secret = an API key, asked with
29
+ # hidden input; plain = asked with visible input), where to get it.
30
+ $Values = @'
31
+ @VALUES@
32
+ '@
33
+
34
+ # The requests. Columns, tab-separated ("-" = empty): file, method, optional (1 = may fail), key
35
+ # variable, key placement (query, header, body or none), header or body field name, header prefix,
36
+ # other variables (comma-separated), URL, JSON body.
37
+ $Items = @'
38
+ @ITEMS@
39
+ '@
40
+
41
+ $Here = $PSScriptRoot
42
+ if (-not $Here) { $Here = (Get-Location).Path }
43
+
44
+ # Keep the window open when the script was started with "Run with PowerShell".
45
+ function Finish([int]$Code) {
46
+ $script:Vals = $null
47
+ if (-not $NoPause -and -not [Console]::IsInputRedirected) {
48
+ Write-Host ''
49
+ [void](Read-Host -Prompt 'Press Enter to close')
50
+ }
51
+ exit $Code
52
+ }
53
+
54
+ function Lines([string]$Text) { $Text -split "`r?`n" | Where-Object { $_.Trim() } }
55
+
56
+ # ---------------------------------------------------------------------------- keys and other values
57
+ $Vals = @{}
58
+ foreach ($line in (Lines $Values)) {
59
+ $var, $kind, $where = $line -split "`t", 3
60
+ $v = [Environment]::GetEnvironmentVariable($var)
61
+ if ($v) {
62
+ Write-Host "Using `$env:$var."
63
+ } elseif ([Console]::IsInputRedirected) {
64
+ $v = [Console]::In.ReadLine()
65
+ } elseif ($kind -eq 'secret') {
66
+ Write-Host "Some files need your $var ($where). Press Enter to skip them."
67
+ $secure = Read-Host -Prompt "$var (input is hidden)" -AsSecureString
68
+ $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
69
+ try { $v = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
70
+ finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
71
+ Remove-Variable secure
72
+ } else {
73
+ $v = Read-Host -Prompt "$var ($where)"
74
+ }
75
+ if ($null -eq $v) { $v = '' }
76
+ if ($kind -eq 'secret') { $v = $v -replace '\s', '' } else { $v = $v.Trim() }
77
+ $Vals[$var] = $v
78
+ }
79
+
80
+ function Get-Code($Err) {
81
+ if ($Err.Exception.Response) { return [int]$Err.Exception.Response.StatusCode }
82
+ return 0
83
+ }
84
+
85
+ # ---------------------------------------------------------------------------- download
86
+ $saved = @(); $failed = 0; $skipped = 0
87
+ foreach ($line in (Lines $Items)) {
88
+ $f = @($line -split "`t", 10 | ForEach-Object { if ($_ -eq '-') { '' } else { $_ } })
89
+ $file, $method, $optional, $kvar, $place, $kname, $kprefix, $extras, $url, $body = $f
90
+ $key = ''
91
+ if ($kvar) {
92
+ $key = $Vals[$kvar]
93
+ if (-not $key) { Write-Host "skipped (needs $kvar): $file"; $skipped++; continue }
94
+ }
95
+ $miss = ''
96
+ if ($extras) {
97
+ foreach ($x in $extras.Split(',')) {
98
+ if (-not $Vals[$x]) { $miss = $x; break }
99
+ $url = $url.Replace("{$x}", $Vals[$x])
100
+ }
101
+ }
102
+ if ($miss) { Write-Host "skipped (needs $miss): $file"; $skipped++; continue }
103
+ if ($place -eq 'query') { $url = $url.Replace('{API_KEY}', $key) }
104
+
105
+ $params = @{ Uri = $url; Method = $method; UseBasicParsing = $true; UserAgent = 'data-analyst/1.0'
106
+ OutFile = (Join-Path $Here "$file.part") }
107
+ if ($place -eq 'header') {
108
+ $params.Headers = @{ $kname = "$kprefix$key" }
109
+ $params.MaximumRedirection = 0 # a redirect would carry the key header to another host
110
+ }
111
+ if ($body) {
112
+ if ($place -eq 'body') { $body = $body.Replace('{API_KEY}', ($key -replace '\\', '\\' -replace '"', '\"')) }
113
+ $params.Body = [Text.Encoding]::UTF8.GetBytes($body)
114
+ $params.ContentType = 'application/json'
115
+ }
116
+ $code = 0; $msg = ''; $ok = $false
117
+ for ($try = 1; $try -le 4; $try++) {
118
+ try {
119
+ Invoke-WebRequest @params
120
+ $ok = $true
121
+ break
122
+ } catch {
123
+ $code = Get-Code $_
124
+ $msg = [string]$_.ErrorDetails.Message
125
+ if (-not $msg) { $msg = $_.Exception.Message }
126
+ if ($try -lt 4 -and ($code -eq 0 -or $code -eq 429 -or $code -ge 500)) { Start-Sleep -Seconds (10 * $try); continue }
127
+ break
128
+ }
129
+ }
130
+ $part = Join-Path $Here "$file.part"
131
+ $dest = Join-Path $Here $file
132
+ if ($ok) {
133
+ Move-Item -LiteralPath $part -Destination $dest -Force
134
+ $saved += $dest
135
+ Write-Host "saved $file"
136
+ } else {
137
+ if (Test-Path -LiteralPath $part) { Remove-Item -LiteralPath $part -Force }
138
+ if ($key) { $msg = $msg.Replace($key, '[KEY]') }
139
+ if ($msg.Length -gt 300) { $msg = $msg.Substring(0, 300) }
140
+ if ($optional -eq '1') {
141
+ Write-Host "not available (optional, HTTP $code): $file"
142
+ } elseif ($code -eq 0) {
143
+ Write-Host "FAILED $($file): could not reach the server ($msg) Check the connection, VPN and proxy." -ForegroundColor Red
144
+ $failed++
145
+ } else {
146
+ Write-Host "FAILED $($file): HTTP $code $msg" -ForegroundColor Red
147
+ $failed++
148
+ }
149
+ }
150
+ }
151
+
152
+ Write-Host ''
153
+ if ($saved.Count -gt 0) {
154
+ Write-Host 'Done. Upload these files to the chat (do not rename or edit them):' -ForegroundColor Green
155
+ foreach ($p in $saved) { Write-Host " $p" }
156
+ }
157
+ if ($skipped -gt 0) { Write-Host "$skipped file(s) skipped: run again and enter the value asked for." }
158
+ if ($failed -gt 0) {
159
+ Write-Host "$failed file(s) failed: send the messages above to the analyst." -ForegroundColor Red
160
+ Finish 1
161
+ }
162
+ Finish 0