data-analyst-mcp 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/README.md +262 -262
  2. package/dist/report/method_audit.js +7 -1
  3. package/dist/report/method_audit.js.map +1 -1
  4. package/dist/report/scan_secrets.d.ts +4 -1
  5. package/dist/report/scan_secrets.js +36 -5
  6. package/dist/report/scan_secrets.js.map +1 -1
  7. package/dist/server.d.ts +1 -1
  8. package/dist/server.js +3 -2
  9. package/dist/server.js.map +1 -1
  10. package/guide/WORKFLOW.md +134 -134
  11. package/guide/assets/disclaimer.txt +1 -1
  12. package/guide/assets/download_data.ps1 +162 -162
  13. package/guide/assets/report_template.html +597 -597
  14. package/guide/references/routing.md +171 -171
  15. package/guide/references/style_guide.md +398 -398
  16. package/package.json +56 -56
  17. package/packs/acf/endpoints.json +114 -114
  18. package/packs/acf/policy.json +104 -104
  19. package/packs/actuaries/endpoints.json +82 -82
  20. package/packs/actuaries/policy.json +109 -109
  21. package/packs/bea/assets/download_bea.ps1 +182 -182
  22. package/packs/bea/docs/bea_api.md +461 -461
  23. package/packs/bea/docs/datasets.md +157 -157
  24. package/packs/bea/docs/report_patterns.md +112 -112
  25. package/packs/bea/policy.json +170 -170
  26. package/packs/bea/source.md +152 -152
  27. package/packs/bls/assets/download_bls.ps1 +152 -152
  28. package/packs/bls/docs/analysis_patterns.md +108 -108
  29. package/packs/bls/docs/bls_api.md +73 -73
  30. package/packs/bls/docs/series_ids.md +149 -149
  31. package/packs/bls/policy.json +238 -238
  32. package/packs/bts/endpoints.json +346 -346
  33. package/packs/bts/policy.json +94 -94
  34. package/packs/cdc/endpoints.json +197 -197
  35. package/packs/cdc/policy.json +83 -83
  36. package/packs/census/endpoints.json +99 -99
  37. package/packs/cfpb/endpoints.json +326 -326
  38. package/packs/cfpb/policy.json +119 -119
  39. package/packs/clinical-trials/endpoints.json +166 -166
  40. package/packs/clinical-trials/policy.json +84 -84
  41. package/packs/cms/endpoints.json +69 -69
  42. package/packs/cms/policy.json +96 -96
  43. package/packs/college-scorecard/endpoints.json +252 -252
  44. package/packs/college-scorecard/policy.json +106 -106
  45. package/packs/congress/endpoints.json +1701 -1701
  46. package/packs/congress/policy.json +117 -117
  47. package/packs/dhs-ohss/endpoints.json +66 -66
  48. package/packs/dhs-ohss/policy.json +107 -107
  49. package/packs/doj-news/endpoints.json +59 -59
  50. package/packs/doj-news/policy.json +96 -96
  51. package/packs/dol/endpoints.json +817 -817
  52. package/packs/dol/policy.json +135 -135
  53. package/packs/eia/endpoints.json +717 -717
  54. package/packs/eia/policy.json +112 -112
  55. package/packs/epa/endpoints.json +291 -291
  56. package/packs/epa/policy.json +99 -99
  57. package/packs/epa-aqs/endpoints.json +173 -173
  58. package/packs/epa-aqs/policy.json +96 -96
  59. package/packs/fbi/endpoints.json +1841 -1841
  60. package/packs/fbi/policy.json +118 -118
  61. package/packs/fda/endpoints.json +237 -237
  62. package/packs/fda/policy.json +121 -121
  63. package/packs/fdic/endpoints.json +416 -416
  64. package/packs/fdic/policy.json +83 -83
  65. package/packs/fec/endpoints.json +707 -707
  66. package/packs/fec/policy.json +93 -93
  67. package/packs/fed-board/endpoints.json +89 -89
  68. package/packs/fed-board/policy.json +136 -136
  69. package/packs/federal-register/endpoints.json +143 -143
  70. package/packs/federal-register/policy.json +86 -86
  71. package/packs/fema/endpoints.json +645 -645
  72. package/packs/fema/policy.json +103 -103
  73. package/packs/fhfa/endpoints.json +105 -105
  74. package/packs/fhfa/policy.json +109 -109
  75. package/packs/fred/assets/download_fred.ps1 +122 -122
  76. package/packs/fred/docs/analysis_patterns.md +124 -124
  77. package/packs/fred/docs/fred_api.md +94 -94
  78. package/packs/fred/docs/series.md +188 -188
  79. package/packs/fred/policy.json +253 -253
  80. package/packs/fsa/endpoints.json +150 -150
  81. package/packs/fsa/policy.json +99 -99
  82. package/packs/govinfo/endpoints.json +519 -519
  83. package/packs/gsa-calc/endpoints.json +93 -93
  84. package/packs/gsa-calc/policy.json +97 -97
  85. package/packs/hud/endpoints.json +146 -146
  86. package/packs/hud/policy.json +84 -84
  87. package/packs/ipums/docs/report_patterns.md +117 -117
  88. package/packs/irs-soi/endpoints.json +114 -114
  89. package/packs/irs-soi/policy.json +107 -107
  90. package/packs/medicaid/endpoints.json +412 -412
  91. package/packs/medicaid/policy.json +108 -108
  92. package/packs/naep/endpoints.json +217 -217
  93. package/packs/naep/policy.json +100 -100
  94. package/packs/nhtsa/endpoints.json +132 -132
  95. package/packs/nhtsa/policy.json +93 -93
  96. package/packs/nih/endpoints.json +452 -452
  97. package/packs/nih/policy.json +86 -86
  98. package/packs/noaa/endpoints.json +431 -431
  99. package/packs/noaa/policy.json +95 -95
  100. package/packs/nrel/endpoints.json +327 -327
  101. package/packs/nrel/policy.json +96 -96
  102. package/packs/nws/endpoints.json +271 -271
  103. package/packs/nws/policy.json +82 -82
  104. package/packs/nyfed/endpoints.json +80 -80
  105. package/packs/nyfed/policy.json +92 -92
  106. package/packs/omb/endpoints.json +460 -460
  107. package/packs/omb/policy.json +106 -106
  108. package/packs/open-payments/endpoints.json +297 -297
  109. package/packs/open-payments/policy.json +101 -101
  110. package/packs/opm/assets/download_opm.ps1 +125 -125
  111. package/packs/opm/docs/fields.md +184 -184
  112. package/packs/opm/docs/opm_api.md +71 -71
  113. package/packs/opm/docs/report_patterns.md +83 -83
  114. package/packs/opm/lib/data_dictionary.csv +73 -73
  115. package/packs/pbgc/endpoints.json +483 -483
  116. package/packs/pbgc/policy.json +94 -94
  117. package/packs/philfed/endpoints.json +85 -85
  118. package/packs/philfed/policy.json +94 -94
  119. package/packs/regulations/endpoints.json +287 -287
  120. package/packs/regulations/policy.json +86 -86
  121. package/packs/sec/endpoints.json +236 -236
  122. package/packs/sec/policy.json +86 -86
  123. package/packs/senate-lobbying/endpoints.json +553 -553
  124. package/packs/senate-lobbying/policy.json +99 -99
  125. package/packs/snapqc/endpoints.json +1426 -1426
  126. package/packs/snapqc/policy.json +92 -92
  127. package/packs/treasury/catalog.json +2267 -2267
  128. package/packs/treasury/docs/api_guide.md +245 -245
  129. package/packs/treasury/docs/datasets.md +56 -56
  130. package/packs/treasury/docs/report_patterns.md +126 -126
  131. package/packs/treasury/policy.json +159 -159
  132. package/packs/treasury/source.md +170 -165
  133. package/packs/treasury-yields/endpoints.json +226 -226
  134. package/packs/treasury-yields/policy.json +86 -86
  135. package/packs/usaspending/endpoints.json +1315 -1315
  136. package/packs/usaspending/policy.json +109 -109
  137. package/packs/usda-fna/policy.json +133 -133
  138. package/packs/usda-fooddata/endpoints.json +180 -180
  139. package/packs/usda-fooddata/policy.json +94 -94
  140. package/packs/usda-nass/endpoints.json +323 -323
  141. package/packs/usda-nass/policy.json +95 -95
  142. package/packs/usda-outlook/endpoints.json +86 -86
  143. package/packs/usda-outlook/policy.json +109 -109
  144. package/packs/usgs/endpoints.json +313 -313
  145. package/packs/usgs/policy.json +95 -95
  146. package/packs/usitc/endpoints.json +74 -74
  147. package/packs/usitc/policy.json +111 -111
  148. package/packs/uspto/endpoints.json +141 -141
  149. package/packs/uspto/policy.json +95 -95
  150. package/packs/va/endpoints.json +414 -414
  151. package/packs/va/policy.json +101 -101
  152. package/packs/world-bank/endpoints.json +122 -122
  153. package/packs/world-bank/policy.json +101 -101
@@ -1,117 +1,117 @@
1
- # Report patterns for IPUMS data products
2
-
3
- Use this file to shape the analysis for every product, and the layout when a report or paper is
4
- built. The questions and conventions apply to quick answers too; the publication structure applies only
5
- in report or paper mode. The aim is a product a careful reviewer would recognize: precise populations,
6
- weighted estimates, a lead summary with numbers, numbered exhibits that each carry notes and a source
7
- line, exhibit data shipped alongside, and limitations stated plainly.
8
-
9
- ## Typical questions and how they are framed
10
-
11
- These recur in published work by statistical and policy agencies (Census Bureau, BLS, GAO, CRS,
12
- state agencies, research institutes), so there is usually a precedent to follow and benchmark against.
13
-
14
- - **Employment of immigrants** (ACS, CPS ASEC): people ages 25 to 54, split by sex, by nativity and
15
- citizenship, by education and by birthplace. Measures: weighted population counts (millions),
16
- employment rates (employed ÷ population) and average annual earnings of workers. Headline style:
17
- "In 2018, 46 million people living in the United States, or 14 percent of the population, had been
18
- born in other countries." Public IPUMS data **cannot** identify legal status (published work that
19
- reports it imputes it), so report nativity and citizenship (naturalized vs. noncitizen) instead, and
20
- say so.
21
- - **Public-sector vs. private-sector pay** (CPS): full-time, full-year civilian workers, in 5 education
22
- groups (high school diploma or less; some college; bachelor's degree; master's degree; professional
23
- degree or doctorate). Measures: average hourly wages (annual wages ÷ annual hours), then
24
- regression-adjusted differences that control for education, experience, occupation, employer size,
25
- region, veteran status and demographics. With IPUMS alone: raw wage gaps, and optionally an OLS on
26
- log hourly wage with controls. Benefits are **not** in IPUMS, so say so.
27
- - **Veterans' labor market outcomes** (CPS, ACS): for example, men ages 22 to 44 who left active duty
28
- after September 2001, compared with men who never served. Measures: labor force participation rate,
29
- unemployment rate (unemployed ÷ labor force) and earnings, by age group and by disability rating.
30
- - Other recurring uses of the same data:
31
- - labor force projections by age and sex (CPS monthly);
32
- - minimum wage analyses that count affected workers by hourly wage (CPS outgoing rotation groups: EARNWEEK, HOURWAGE, EARNWT);
33
- - health insurance coverage by source (ACS HCOV*, CPS ASEC);
34
- - distribution of household income and poverty (CPS ASEC);
35
- - immigration surges (ACS and CPS: education and labor force participation of recent arrivals, using YRIMMIG / YRSUSA1);
36
- - state-level profiles (ACS: large samples with STATEFIP and PUMA detail).
37
- - **Health insurance and access by group** (NHIS): uninsured at the interview, private vs. public coverage,
38
- no usual place of care, care delayed or forgone because of cost, by age (0-17, 18-64, 65+), poverty ratio,
39
- race and Hispanic origin, nativity. Benchmark the headline against NCHS's annual Health Insurance Coverage
40
- report (2024: 8.2% of all ages and 11.6% of adults 18-64 uninsured). NHIS has no state identifier in the
41
- public files: state estimates come from the ACS.
42
- - **Health care spending** (MEPS): mean and median spending per person, the share with any spending,
43
- out-of-pocket spending and payer shares, by age, insurance type (`COVERTYPE`) and income group (`POVCAT`).
44
- Benchmark against AHRQ's MEPS-HC summary tables or Statistical Briefs (2022: $6,765 per person). Name the dollar
45
- year; per-person means hide extreme concentration, so pair them with medians or the top-percentile share.
46
-
47
- ## Conventions to follow
48
-
49
- **Population and definitions**
50
- - State the universe exactly: age range, civilian vs. all, noninstitutional (USA GQ 1, 2 or 5 = households), years and geography.
51
- - The prime-age default is 25–54. It avoids school-age and retirement-age confounding.
52
- - **Employment rate** = employed ÷ population. **Unemployment rate** = unemployed ÷ labor force. **Participation rate** = labor force ÷ population.
53
- - **Earnings:** say whose earnings are averaged. Usually it is workers with positive wages, sometimes full-time, full-year workers only. Also say whether self-employment income is included (INCWAGE excludes it; INCEARN in the ACS includes it).
54
-
55
- **Estimation**
56
- - Always weight. In the USA collection use PERWT (HHWT for household measures). In the CPS ASEC use ASECWT, in CPS basic monthly use WTFINL (COMPWT for labor-force levels that match BLS), and in the CPS outgoing rotation groups use EARNWT. In NHIS (2019 on) use SAMPWEIGHT on sample adults or sample children; in MEPS use PERWEIGHT (SAQWEIGHT for questionnaire items).
57
- - Report the unweighted cell size internally. Suppress or footnote cells with fewer than about 100 respondents; unstable estimates are not published.
58
- - **Dollars:** convert to a single year's dollars and name the year and index ("in 2023 dollars, using the CPI-U-RS"; the PCE price index is a common choice for earnings and income), with CPI99 or `adjustMonetaryValues`. CPS ASEC income is for the **previous calendar year** (ASEC 2023 means 2022 income). ACS income is for the past 12 months, and IPUMS expresses it in survey-year dollars.
59
- - **Top-coding:** INCWAGE is top-coded, which mostly matters for means at high education levels. Mention it.
60
- - **Sampling error:** for differences that drive conclusions, note whether they are large relative to sampling error. ACS replicate weights (REPWTP, 80) and CPS ASEC replicate weights (REPWTP, 160) give proper standard errors. CPS basic monthly files have none (REPWTP is ASEC-only) and no design variables: an SE there can only be approximate (clustered by household, `SERIAL` within month). This server does not compute standard errors yet: report which weight was used and that standard errors were not computed.
61
-
62
- **Publication structure (mirrored in `build_report`, `"mode": "report"`; only when a report is asked for)**
63
-
64
- The report layout, the **exhibit chartbook**, runs in this order:
65
- 1. **Cover:** "Month Year", the title (with the period for trend reports), and a hero chart that has its own title and unit label.
66
- 2. **At a Glance** (one page of about 300–420 words):
67
- - one framing paragraph saying what was examined, the population, the period and the dimensions;
68
- - 3–5 bullets, each opening with a **bold lead phrase.** that names a section, followed by 2–4 sentences with the key numbers.
69
- 3. **Contents:** the Introduction subsections, each section with an "Exhibits a–b" sub-line, the appendixes, and About This Document when there is one.
70
- 4. **List of Exhibits:** grouped under section headings, each entry "N. Title, years".
71
- 5. **Notes About This Report** (also titled "Notes" or "Notes and Definitions"). Short unnumbered paragraphs covering:
72
- - calendar years;
73
- - "Numbers in the text and exhibits may not add up to totals because of rounding.";
74
- - the dollar year and deflator;
75
- - the data source and universe;
76
- - key definitions;
77
- - recession shading, if any;
78
- - where the data are posted.
79
- 6. **Introduction** (or "Summary and Introduction"):
80
- - a lead paragraph with the headline numbers;
81
- - question-style subheadings, one per section ("How Do Demographic Factors Relate to …?");
82
- - closing subsections such as "Why Focus on People Ages 25 to 54?", "What Data Were Used?" and, when updating earlier work, "What Is New in This Report?".
83
- 7. **Thematic sections** (2–5, unnumbered):
84
- - Each opens on a full-width page with 1–3 summary paragraphs.
85
- - Then comes **one exhibit per page**: "Exhibit N." with a title-case title ending in ", 1998 to 2018", a unit label above the chart, and 2–4 narrative paragraphs beside it (levels, changes, comparisons, "(see Exhibit 5)").
86
- - Below the chart: the data source line ("Data source: `<organization>`, using data from …", or "Data source: Analyst calculations using data from …" when no organization is named), then the notes, then lettered notes ("a.").
87
- 8. **Appendix A: Data and Methods** (or "Appendix: Estimating …"): data, sample, universe, weights, imputations, reliability, comparison with other estimates. Sometimes followed by **Appendix B: Definitions**, an alphabetical glossary.
88
- 9. **About This Document** (optional), in this order:
89
- - who requested it, or what question prompted it;
90
- - who prepared it, with what data and tools;
91
- - the reviewers, if any;
92
- - the date.
93
- 10. **Footnotes** are numbered in each part (restarting in each section), and cite publications as: Organization, *Title* (Month Year), URL.
94
-
95
- Other variants, not built by `build_report`:
96
- - **Long-form**:
97
- - a Summary with Figure S-1 / Table S-1;
98
- - "Chapter N:" sections in two-column prose with embedded "Figure 1-1." / "Table 2-3.";
99
- - Boxes;
100
- - a List of Tables and Figures at the back.
101
- - **Hybrid**: full-width pages each with a heading, a paragraph and unnumbered charts; sources pooled in an appendix.
102
- - Use `"mode": "paper"` when a prose-led product is wanted.
103
-
104
- **Reading PDFs from sites that block scripts:** some publishers refuse scripts and the in-app browser (HTTP 403, bot-check slider). Fetch `https://web.archive.org/web/<year>/<full pdf url>` and extract the text with pypdf. If the plain year URL returns HTML, use a CDX timestamp with `id_` (for example `https://web.archive.org/web/20240430034127id_/<url>`). Cite the original URL.
105
-
106
- **Data files:** the codebook, the extract definition (`data/ipums/extract_spec.json`), `reproduce/ipums/`
107
- (written by `build_report`: the request as `spec.json` and `REPRODUCE.md`, which says how to request the same
108
- extract again), all attached to the page with Download buttons, and each exhibit's CSV ship with the product,
109
- so every number can be traced and the data requested again. The extract CSV (microdata) stays local: the
110
- IPUMS terms forbid redistributing it, so the report never links or embeds it, and `build_report` never puts
111
- it in a package (`"replication": {"include_microdata": true}` is ignored).
112
-
113
- **Chart forms**
114
- - Trends over time: line charts with at most 4–5 series. Facet (one exhibit per sex) rather than drawing 8 lines.
115
- - Group comparisons at a point in time: grouped columns, or horizontal bars when the category labels are long (education, birthplace).
116
- - Composition (wages + benefits, population by status): stacked columns.
117
- - No dual axes, no pie charts, no 3-D.
1
+ # Report patterns for IPUMS data products
2
+
3
+ Use this file to shape the analysis for every product, and the layout when a report or paper is
4
+ built. The questions and conventions apply to quick answers too; the publication structure applies only
5
+ in report or paper mode. The aim is a product a careful reviewer would recognize: precise populations,
6
+ weighted estimates, a lead summary with numbers, numbered exhibits that each carry notes and a source
7
+ line, exhibit data shipped alongside, and limitations stated plainly.
8
+
9
+ ## Typical questions and how they are framed
10
+
11
+ These recur in published work by statistical and policy agencies (Census Bureau, BLS, GAO, CRS,
12
+ state agencies, research institutes), so there is usually a precedent to follow and benchmark against.
13
+
14
+ - **Employment of immigrants** (ACS, CPS ASEC): people ages 25 to 54, split by sex, by nativity and
15
+ citizenship, by education and by birthplace. Measures: weighted population counts (millions),
16
+ employment rates (employed ÷ population) and average annual earnings of workers. Headline style:
17
+ "In 2018, 46 million people living in the United States, or 14 percent of the population, had been
18
+ born in other countries." Public IPUMS data **cannot** identify legal status (published work that
19
+ reports it imputes it), so report nativity and citizenship (naturalized vs. noncitizen) instead, and
20
+ say so.
21
+ - **Public-sector vs. private-sector pay** (CPS): full-time, full-year civilian workers, in 5 education
22
+ groups (high school diploma or less; some college; bachelor's degree; master's degree; professional
23
+ degree or doctorate). Measures: average hourly wages (annual wages ÷ annual hours), then
24
+ regression-adjusted differences that control for education, experience, occupation, employer size,
25
+ region, veteran status and demographics. With IPUMS alone: raw wage gaps, and optionally an OLS on
26
+ log hourly wage with controls. Benefits are **not** in IPUMS, so say so.
27
+ - **Veterans' labor market outcomes** (CPS, ACS): for example, men ages 22 to 44 who left active duty
28
+ after September 2001, compared with men who never served. Measures: labor force participation rate,
29
+ unemployment rate (unemployed ÷ labor force) and earnings, by age group and by disability rating.
30
+ - Other recurring uses of the same data:
31
+ - labor force projections by age and sex (CPS monthly);
32
+ - minimum wage analyses that count affected workers by hourly wage (CPS outgoing rotation groups: EARNWEEK, HOURWAGE, EARNWT);
33
+ - health insurance coverage by source (ACS HCOV*, CPS ASEC);
34
+ - distribution of household income and poverty (CPS ASEC);
35
+ - immigration surges (ACS and CPS: education and labor force participation of recent arrivals, using YRIMMIG / YRSUSA1);
36
+ - state-level profiles (ACS: large samples with STATEFIP and PUMA detail).
37
+ - **Health insurance and access by group** (NHIS): uninsured at the interview, private vs. public coverage,
38
+ no usual place of care, care delayed or forgone because of cost, by age (0-17, 18-64, 65+), poverty ratio,
39
+ race and Hispanic origin, nativity. Benchmark the headline against NCHS's annual Health Insurance Coverage
40
+ report (2024: 8.2% of all ages and 11.6% of adults 18-64 uninsured). NHIS has no state identifier in the
41
+ public files: state estimates come from the ACS.
42
+ - **Health care spending** (MEPS): mean and median spending per person, the share with any spending,
43
+ out-of-pocket spending and payer shares, by age, insurance type (`COVERTYPE`) and income group (`POVCAT`).
44
+ Benchmark against AHRQ's MEPS-HC summary tables or Statistical Briefs (2022: $6,765 per person). Name the dollar
45
+ year; per-person means hide extreme concentration, so pair them with medians or the top-percentile share.
46
+
47
+ ## Conventions to follow
48
+
49
+ **Population and definitions**
50
+ - State the universe exactly: age range, civilian vs. all, noninstitutional (USA GQ 1, 2 or 5 = households), years and geography.
51
+ - The prime-age default is 25–54. It avoids school-age and retirement-age confounding.
52
+ - **Employment rate** = employed ÷ population. **Unemployment rate** = unemployed ÷ labor force. **Participation rate** = labor force ÷ population.
53
+ - **Earnings:** say whose earnings are averaged. Usually it is workers with positive wages, sometimes full-time, full-year workers only. Also say whether self-employment income is included (INCWAGE excludes it; INCEARN in the ACS includes it).
54
+
55
+ **Estimation**
56
+ - Always weight. In the USA collection use PERWT (HHWT for household measures). In the CPS ASEC use ASECWT, in CPS basic monthly use WTFINL (COMPWT for labor-force levels that match BLS), and in the CPS outgoing rotation groups use EARNWT. In NHIS (2019 on) use SAMPWEIGHT on sample adults or sample children; in MEPS use PERWEIGHT (SAQWEIGHT for questionnaire items).
57
+ - Report the unweighted cell size internally. Suppress or footnote cells with fewer than about 100 respondents; unstable estimates are not published.
58
+ - **Dollars:** convert to a single year's dollars and name the year and index ("in 2023 dollars, using the CPI-U-RS"; the PCE price index is a common choice for earnings and income), with CPI99 or `adjustMonetaryValues`. CPS ASEC income is for the **previous calendar year** (ASEC 2023 means 2022 income). ACS income is for the past 12 months, and IPUMS expresses it in survey-year dollars.
59
+ - **Top-coding:** INCWAGE is top-coded, which mostly matters for means at high education levels. Mention it.
60
+ - **Sampling error:** for differences that drive conclusions, note whether they are large relative to sampling error. ACS replicate weights (REPWTP, 80) and CPS ASEC replicate weights (REPWTP, 160) give proper standard errors. CPS basic monthly files have none (REPWTP is ASEC-only) and no design variables: an SE there can only be approximate (clustered by household, `SERIAL` within month). This server does not compute standard errors yet: report which weight was used and that standard errors were not computed.
61
+
62
+ **Publication structure (mirrored in `build_report`, `"mode": "report"`; only when a report is asked for)**
63
+
64
+ The report layout, the **exhibit chartbook**, runs in this order:
65
+ 1. **Cover:** "Month Year", the title (with the period for trend reports), and a hero chart that has its own title and unit label.
66
+ 2. **At a Glance** (one page of about 300–420 words):
67
+ - one framing paragraph saying what was examined, the population, the period and the dimensions;
68
+ - 3–5 bullets, each opening with a **bold lead phrase.** that names a section, followed by 2–4 sentences with the key numbers.
69
+ 3. **Contents:** the Introduction subsections, each section with an "Exhibits a–b" sub-line, the appendixes, and About This Document when there is one.
70
+ 4. **List of Exhibits:** grouped under section headings, each entry "N. Title, years".
71
+ 5. **Notes About This Report** (also titled "Notes" or "Notes and Definitions"). Short unnumbered paragraphs covering:
72
+ - calendar years;
73
+ - "Numbers in the text and exhibits may not add up to totals because of rounding.";
74
+ - the dollar year and deflator;
75
+ - the data source and universe;
76
+ - key definitions;
77
+ - recession shading, if any;
78
+ - where the data are posted.
79
+ 6. **Introduction** (or "Summary and Introduction"):
80
+ - a lead paragraph with the headline numbers;
81
+ - question-style subheadings, one per section ("How Do Demographic Factors Relate to …?");
82
+ - closing subsections such as "Why Focus on People Ages 25 to 54?", "What Data Were Used?" and, when updating earlier work, "What Is New in This Report?".
83
+ 7. **Thematic sections** (2–5, unnumbered):
84
+ - Each opens on a full-width page with 1–3 summary paragraphs.
85
+ - Then comes **one exhibit per page**: "Exhibit N." with a title-case title ending in ", 1998 to 2018", a unit label above the chart, and 2–4 narrative paragraphs beside it (levels, changes, comparisons, "(see Exhibit 5)").
86
+ - Below the chart: the data source line ("Data source: `<organization>`, using data from …", or "Data source: Analyst calculations using data from …" when no organization is named), then the notes, then lettered notes ("a.").
87
+ 8. **Appendix A: Data and Methods** (or "Appendix: Estimating …"): data, sample, universe, weights, imputations, reliability, comparison with other estimates. Sometimes followed by **Appendix B: Definitions**, an alphabetical glossary.
88
+ 9. **About This Document** (optional), in this order:
89
+ - who requested it, or what question prompted it;
90
+ - who prepared it, with what data and tools;
91
+ - the reviewers, if any;
92
+ - the date.
93
+ 10. **Footnotes** are numbered in each part (restarting in each section), and cite publications as: Organization, *Title* (Month Year), URL.
94
+
95
+ Other variants, not built by `build_report`:
96
+ - **Long-form**:
97
+ - a Summary with Figure S-1 / Table S-1;
98
+ - "Chapter N:" sections in two-column prose with embedded "Figure 1-1." / "Table 2-3.";
99
+ - Boxes;
100
+ - a List of Tables and Figures at the back.
101
+ - **Hybrid**: full-width pages each with a heading, a paragraph and unnumbered charts; sources pooled in an appendix.
102
+ - Use `"mode": "paper"` when a prose-led product is wanted.
103
+
104
+ **Reading PDFs from sites that block scripts:** some publishers refuse scripts and the in-app browser (HTTP 403, bot-check slider). Fetch `https://web.archive.org/web/<year>/<full pdf url>` and extract the text with pypdf. If the plain year URL returns HTML, use a CDX timestamp with `id_` (for example `https://web.archive.org/web/20240430034127id_/<url>`). Cite the original URL.
105
+
106
+ **Data files:** the codebook, the extract definition (`data/ipums/extract_spec.json`), `reproduce/ipums/`
107
+ (written by `build_report`: the request as `spec.json` and `REPRODUCE.md`, which says how to request the same
108
+ extract again), all attached to the page with Download buttons, and each exhibit's CSV ship with the product,
109
+ so every number can be traced and the data requested again. The extract CSV (microdata) stays local: the
110
+ IPUMS terms forbid redistributing it, so the report never links or embeds it, and `build_report` never puts
111
+ it in a package (`"replication": {"include_microdata": true}` is ignored).
112
+
113
+ **Chart forms**
114
+ - Trends over time: line charts with at most 4–5 series. Facet (one exhibit per sex) rather than drawing 8 lines.
115
+ - Group comparisons at a point in time: grouped columns, or horizontal bars when the category labels are long (education, birthplace).
116
+ - Composition (wages + benefits, population by status): stacked columns.
117
+ - No dual axes, no pie charts, no 3-D.
@@ -1,114 +1,114 @@
1
- {
2
- "base_url": "https://www.irs.gov/pub/irs-soi",
3
- "timeout": 120,
4
- "endpoints": {
5
- "zip_agi": {
6
- "path": "/{yy}zpallagi.csv",
7
- "method": "GET",
8
- "description": "Individual income tax returns by ZIP code and AGI size class (6 classes): number of returns (N*), amounts in thousands of dollars (A*), about 160 items. One all-states file per tax year (216 MB for TY2022), streamed to a temporary file and filtered by state while reading. agi_classes=false reads {yy}zpallnoagi.csv (one row per ZIP, AGI_STUB 0). Tax years 2011-2023.",
9
- "first_year": 2011,
10
- "last_year_verified": 2023,
11
- "stream": true,
12
- "params": {
13
- "tax_year": {"in": "spec", "desc": "tax year, e.g. 2022 (or start_year/end_year for several; each is one file)"},
14
- "start_year": {"in": "spec", "desc": "first tax year of a range"},
15
- "end_year": {"in": "spec", "desc": "last tax year of a range"},
16
- "state": {"in": "spec", "desc": "state(s): USPS code, FIPS code or name, or a list (required unless zipcode is given)"},
17
- "zipcode": {"in": "spec", "desc": "5-digit ZIP code(s) to keep; 00000 = state total, 99999 = small and nonresidential ZIPs combined"},
18
- "agi_stub": {"in": "spec", "desc": "AGI size class(es) to keep, 1-6 (see source.md)"},
19
- "agi_classes": {"in": "spec", "desc": "false: read the file without AGI classes (one row per ZIP); default true"},
20
- "columns": {"in": "spec", "desc": "variables to keep, e.g. [\"N1\", \"A00100\"]; the identifier columns are always kept"}
21
- },
22
- "records": "$csv",
23
- "paging": {"type": "none"},
24
- "max_rows": 200000,
25
- "flatten": false
26
- },
27
- "county_agi": {
28
- "path": "/{yy}incyallagi.csv",
29
- "method": "GET",
30
- "description": "Individual income tax returns by county and AGI size class (8 classes), with each state's total as COUNTYFIPS 000: number of returns (N*), amounts in thousands of dollars (A*), about 160 items. One all-states file per tax year (36 MB). agi_classes=false reads {yy}incyallnoagi.csv (one row per county, AGI_STUB 0). Tax years 2011-2023.",
31
- "first_year": 2011,
32
- "last_year_verified": 2023,
33
- "params": {
34
- "tax_year": {"in": "spec", "desc": "tax year, e.g. 2022 (or start_year/end_year)"},
35
- "start_year": {"in": "spec", "desc": "first tax year of a range"},
36
- "end_year": {"in": "spec", "desc": "last tax year of a range"},
37
- "state": {"in": "spec", "desc": "state(s): USPS code, FIPS code or name, or a list (default: all states)"},
38
- "county": {"in": "spec", "desc": "county(ies): 3-digit county FIPS (with state), 5-digit state+county FIPS, or the county name as published (\"Montgomery County\"); 000 = the state total"},
39
- "agi_stub": {"in": "spec", "desc": "AGI size class(es) to keep, 1-8 (see source.md)"},
40
- "agi_classes": {"in": "spec", "desc": "false: read the file without AGI classes; default true"},
41
- "columns": {"in": "spec", "desc": "variables to keep; the identifier columns are always kept"}
42
- },
43
- "records": "$csv",
44
- "paging": {"type": "none"},
45
- "max_rows": 200000,
46
- "flatten": false
47
- },
48
- "state_agi": {
49
- "path": "/{yy}in55cmcsv.csv",
50
- "method": "GET",
51
- "description": "Historic Table 2: individual income tax returns by state and AGI size class (AGI_STUB 0 = all returns, 1-10 = classes), for 50 states, DC, Puerto Rico (PR), other areas (OA) and the United States (US); amounts in thousands of dollars. One file per tax year (0.8 MB): tax years 2012 and 2014-2023 (2013 is published only as .xlsx). File names vary by year (12/14-17in54cmcsv.csv, 18in55cmagi.csv, 19-23in55cmcsv.csv); the adapter picks the right one.",
52
- "first_year": 2012,
53
- "last_year_verified": 2023,
54
- "params": {
55
- "tax_year": {"in": "spec", "desc": "tax year, e.g. 2022 (or start_year/end_year)"},
56
- "start_year": {"in": "spec", "desc": "first tax year of a range"},
57
- "end_year": {"in": "spec", "desc": "last tax year of a range"},
58
- "state": {"in": "spec", "desc": "state(s): USPS code, FIPS code or name, or US / PR / OA; default all"},
59
- "agi_stub": {"in": "spec", "desc": "AGI size class(es) to keep, 0-10 (0 = all returns)"},
60
- "columns": {"in": "spec", "desc": "variables to keep; STATE and AGI_STUB are always kept"}
61
- },
62
- "records": "$csv",
63
- "paging": {"type": "none"},
64
- "max_rows": 200000,
65
- "flatten": false
66
- },
67
- "migration": {
68
- "path": "/{level}{direction}{y1y2}.csv",
69
- "method": "GET",
70
- "description": "Taxpayer migration from matched returns of two consecutive filing years: number of returns (N1, about households), individuals (N2) and AGI (thousands of dollars, year-2 return) moving between states or counties. level state|county, direction inflow (into the state/county, by origin) | outflow (out of it, by destination); y1 = first filing year (2022 = 2022-2023). Filing-year pairs 2011-2012 to 2022-2023.",
71
- "first_year": 2011,
72
- "last_year_verified": 2022,
73
- "params": {
74
- "level": {"in": "spec", "required": true, "enum": ["state", "county"], "desc": "state-to-state or county-to-county flows"},
75
- "direction": {"in": "spec", "required": true, "enum": ["inflow", "outflow"], "desc": "inflow: into the state/county by origin; outflow: out of it by destination"},
76
- "y1": {"in": "spec", "desc": "first filing year of the pair, e.g. 2022 for 2022-2023 (or start_y1/end_y1)"},
77
- "start_y1": {"in": "spec", "desc": "first y1 of a range of pairs"},
78
- "end_y1": {"in": "spec", "desc": "last y1 of a range of pairs"},
79
- "state": {"in": "spec", "desc": "the state(s) whose inflow or outflow to keep (destination for inflow, origin for outflow); default all"},
80
- "county": {"in": "spec", "desc": "county level: the county(ies) whose flows to keep: 3-digit county FIPS (with state) or 5-digit FIPS"}
81
- },
82
- "records": "$csv",
83
- "paging": {"type": "none"},
84
- "max_rows": 200000,
85
- "flatten": false
86
- }
87
- },
88
- "catalog": {
89
- "zip": {
90
- "id": "zpallagi",
91
- "name": "SOI ZIP Code Data (all states, by AGI class)",
92
- "endpoint": "zip_agi",
93
- "description": "income tax returns, adjusted gross income AGI, wages, credits by ZIP code and income size class; zip code income"
94
- },
95
- "county": {
96
- "id": "incyallagi",
97
- "name": "SOI County Data (all states, by AGI class)",
98
- "endpoint": "county_agi",
99
- "description": "income tax returns, adjusted gross income AGI, wages, earned income credit by county and income size class; county income"
100
- },
101
- "state": {
102
- "id": "in55cmcsv",
103
- "name": "SOI Historic Table 2 (state data by AGI class)",
104
- "endpoint": "state_agi",
105
- "description": "income tax returns and AGI by state and income size class including $1 million or more; national total"
106
- },
107
- "migration": {
108
- "id": "migration",
109
- "name": "SOI Migration Data (state-to-state and county-to-county)",
110
- "endpoint": "migration",
111
- "description": "taxpayer migration inflow outflow between states and counties; households, people and income moving; net migration"
112
- }
113
- }
114
- }
1
+ {
2
+ "base_url": "https://www.irs.gov/pub/irs-soi",
3
+ "timeout": 120,
4
+ "endpoints": {
5
+ "zip_agi": {
6
+ "path": "/{yy}zpallagi.csv",
7
+ "method": "GET",
8
+ "description": "Individual income tax returns by ZIP code and AGI size class (6 classes): number of returns (N*), amounts in thousands of dollars (A*), about 160 items. One all-states file per tax year (216 MB for TY2022), streamed to a temporary file and filtered by state while reading. agi_classes=false reads {yy}zpallnoagi.csv (one row per ZIP, AGI_STUB 0). Tax years 2011-2023.",
9
+ "first_year": 2011,
10
+ "last_year_verified": 2023,
11
+ "stream": true,
12
+ "params": {
13
+ "tax_year": {"in": "spec", "desc": "tax year, e.g. 2022 (or start_year/end_year for several; each is one file)"},
14
+ "start_year": {"in": "spec", "desc": "first tax year of a range"},
15
+ "end_year": {"in": "spec", "desc": "last tax year of a range"},
16
+ "state": {"in": "spec", "desc": "state(s): USPS code, FIPS code or name, or a list (required unless zipcode is given)"},
17
+ "zipcode": {"in": "spec", "desc": "5-digit ZIP code(s) to keep; 00000 = state total, 99999 = small and nonresidential ZIPs combined"},
18
+ "agi_stub": {"in": "spec", "desc": "AGI size class(es) to keep, 1-6 (see source.md)"},
19
+ "agi_classes": {"in": "spec", "desc": "false: read the file without AGI classes (one row per ZIP); default true"},
20
+ "columns": {"in": "spec", "desc": "variables to keep, e.g. [\"N1\", \"A00100\"]; the identifier columns are always kept"}
21
+ },
22
+ "records": "$csv",
23
+ "paging": {"type": "none"},
24
+ "max_rows": 200000,
25
+ "flatten": false
26
+ },
27
+ "county_agi": {
28
+ "path": "/{yy}incyallagi.csv",
29
+ "method": "GET",
30
+ "description": "Individual income tax returns by county and AGI size class (8 classes), with each state's total as COUNTYFIPS 000: number of returns (N*), amounts in thousands of dollars (A*), about 160 items. One all-states file per tax year (36 MB). agi_classes=false reads {yy}incyallnoagi.csv (one row per county, AGI_STUB 0). Tax years 2011-2023.",
31
+ "first_year": 2011,
32
+ "last_year_verified": 2023,
33
+ "params": {
34
+ "tax_year": {"in": "spec", "desc": "tax year, e.g. 2022 (or start_year/end_year)"},
35
+ "start_year": {"in": "spec", "desc": "first tax year of a range"},
36
+ "end_year": {"in": "spec", "desc": "last tax year of a range"},
37
+ "state": {"in": "spec", "desc": "state(s): USPS code, FIPS code or name, or a list (default: all states)"},
38
+ "county": {"in": "spec", "desc": "county(ies): 3-digit county FIPS (with state), 5-digit state+county FIPS, or the county name as published (\"Montgomery County\"); 000 = the state total"},
39
+ "agi_stub": {"in": "spec", "desc": "AGI size class(es) to keep, 1-8 (see source.md)"},
40
+ "agi_classes": {"in": "spec", "desc": "false: read the file without AGI classes; default true"},
41
+ "columns": {"in": "spec", "desc": "variables to keep; the identifier columns are always kept"}
42
+ },
43
+ "records": "$csv",
44
+ "paging": {"type": "none"},
45
+ "max_rows": 200000,
46
+ "flatten": false
47
+ },
48
+ "state_agi": {
49
+ "path": "/{yy}in55cmcsv.csv",
50
+ "method": "GET",
51
+ "description": "Historic Table 2: individual income tax returns by state and AGI size class (AGI_STUB 0 = all returns, 1-10 = classes), for 50 states, DC, Puerto Rico (PR), other areas (OA) and the United States (US); amounts in thousands of dollars. One file per tax year (0.8 MB): tax years 2012 and 2014-2023 (2013 is published only as .xlsx). File names vary by year (12/14-17in54cmcsv.csv, 18in55cmagi.csv, 19-23in55cmcsv.csv); the adapter picks the right one.",
52
+ "first_year": 2012,
53
+ "last_year_verified": 2023,
54
+ "params": {
55
+ "tax_year": {"in": "spec", "desc": "tax year, e.g. 2022 (or start_year/end_year)"},
56
+ "start_year": {"in": "spec", "desc": "first tax year of a range"},
57
+ "end_year": {"in": "spec", "desc": "last tax year of a range"},
58
+ "state": {"in": "spec", "desc": "state(s): USPS code, FIPS code or name, or US / PR / OA; default all"},
59
+ "agi_stub": {"in": "spec", "desc": "AGI size class(es) to keep, 0-10 (0 = all returns)"},
60
+ "columns": {"in": "spec", "desc": "variables to keep; STATE and AGI_STUB are always kept"}
61
+ },
62
+ "records": "$csv",
63
+ "paging": {"type": "none"},
64
+ "max_rows": 200000,
65
+ "flatten": false
66
+ },
67
+ "migration": {
68
+ "path": "/{level}{direction}{y1y2}.csv",
69
+ "method": "GET",
70
+ "description": "Taxpayer migration from matched returns of two consecutive filing years: number of returns (N1, about households), individuals (N2) and AGI (thousands of dollars, year-2 return) moving between states or counties. level state|county, direction inflow (into the state/county, by origin) | outflow (out of it, by destination); y1 = first filing year (2022 = 2022-2023). Filing-year pairs 2011-2012 to 2022-2023.",
71
+ "first_year": 2011,
72
+ "last_year_verified": 2022,
73
+ "params": {
74
+ "level": {"in": "spec", "required": true, "enum": ["state", "county"], "desc": "state-to-state or county-to-county flows"},
75
+ "direction": {"in": "spec", "required": true, "enum": ["inflow", "outflow"], "desc": "inflow: into the state/county by origin; outflow: out of it by destination"},
76
+ "y1": {"in": "spec", "desc": "first filing year of the pair, e.g. 2022 for 2022-2023 (or start_y1/end_y1)"},
77
+ "start_y1": {"in": "spec", "desc": "first y1 of a range of pairs"},
78
+ "end_y1": {"in": "spec", "desc": "last y1 of a range of pairs"},
79
+ "state": {"in": "spec", "desc": "the state(s) whose inflow or outflow to keep (destination for inflow, origin for outflow); default all"},
80
+ "county": {"in": "spec", "desc": "county level: the county(ies) whose flows to keep: 3-digit county FIPS (with state) or 5-digit FIPS"}
81
+ },
82
+ "records": "$csv",
83
+ "paging": {"type": "none"},
84
+ "max_rows": 200000,
85
+ "flatten": false
86
+ }
87
+ },
88
+ "catalog": {
89
+ "zip": {
90
+ "id": "zpallagi",
91
+ "name": "SOI ZIP Code Data (all states, by AGI class)",
92
+ "endpoint": "zip_agi",
93
+ "description": "income tax returns, adjusted gross income AGI, wages, credits by ZIP code and income size class; zip code income"
94
+ },
95
+ "county": {
96
+ "id": "incyallagi",
97
+ "name": "SOI County Data (all states, by AGI class)",
98
+ "endpoint": "county_agi",
99
+ "description": "income tax returns, adjusted gross income AGI, wages, earned income credit by county and income size class; county income"
100
+ },
101
+ "state": {
102
+ "id": "in55cmcsv",
103
+ "name": "SOI Historic Table 2 (state data by AGI class)",
104
+ "endpoint": "state_agi",
105
+ "description": "income tax returns and AGI by state and income size class including $1 million or more; national total"
106
+ },
107
+ "migration": {
108
+ "id": "migration",
109
+ "name": "SOI Migration Data (state-to-state and county-to-county)",
110
+ "endpoint": "migration",
111
+ "description": "taxpayer migration inflow outflow between states and counties; households, people and income moving; net migration"
112
+ }
113
+ }
114
+ }