data-analyst-mcp 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +286 -262
- package/dist/adapters/bjs.d.ts +13 -0
- package/dist/adapters/bjs.js +28 -0
- package/dist/adapters/bjs.js.map +1 -0
- package/dist/adapters/cbp.d.ts +5 -0
- package/dist/adapters/cbp.js +64 -0
- package/dist/adapters/cbp.js.map +1 -0
- package/dist/adapters/cdc-wonder.d.ts +5 -0
- package/dist/adapters/cdc-wonder.js +68 -0
- package/dist/adapters/cdc-wonder.js.map +1 -0
- package/dist/adapters/cpsc.d.ts +4 -0
- package/dist/adapters/cpsc.js +125 -0
- package/dist/adapters/cpsc.js.map +1 -0
- package/dist/adapters/fhwa.d.ts +9 -0
- package/dist/adapters/fhwa.js +16 -0
- package/dist/adapters/fhwa.js.map +1 -0
- package/dist/adapters/foreign-assistance.d.ts +9 -0
- package/dist/adapters/foreign-assistance.js +42 -0
- package/dist/adapters/foreign-assistance.js.map +1 -0
- package/dist/adapters/hud-ahar.d.ts +3 -0
- package/dist/adapters/hud-ahar.js +80 -0
- package/dist/adapters/hud-ahar.js.map +1 -0
- package/dist/adapters/nces.d.ts +10 -0
- package/dist/adapters/nces.js +32 -0
- package/dist/adapters/nces.js.map +1 -0
- package/dist/adapters/nsf.d.ts +11 -0
- package/dist/adapters/nsf.js +25 -0
- package/dist/adapters/nsf.js.map +1 -0
- package/dist/adapters/onrr.d.ts +9 -0
- package/dist/adapters/onrr.js +35 -0
- package/dist/adapters/onrr.js.map +1 -0
- package/dist/adapters/sba.d.ts +10 -0
- package/dist/adapters/sba.js +134 -0
- package/dist/adapters/sba.js.map +1 -0
- package/dist/adapters/treasury-tic.d.ts +30 -0
- package/dist/adapters/treasury-tic.js +149 -0
- package/dist/adapters/treasury-tic.js.map +1 -0
- package/dist/adapters/uscourts.d.ts +8 -0
- package/dist/adapters/uscourts.js +8 -0
- package/dist/adapters/uscourts.js.map +1 -0
- package/dist/analysis/cite.d.ts +9 -1
- package/dist/analysis/cite.js +9 -2
- package/dist/analysis/cite.js.map +1 -1
- package/dist/bin/cli.d.ts +2 -0
- package/dist/bin/cli.js +17 -0
- package/dist/bin/cli.js.map +1 -1
- package/dist/core/generic-rest.js +42 -17
- package/dist/core/generic-rest.js.map +1 -1
- package/dist/core/guard.d.ts +32 -0
- package/dist/core/guard.js +176 -0
- package/dist/core/guard.js.map +1 -0
- package/dist/core/http.d.ts +10 -0
- package/dist/core/http.js +36 -7
- package/dist/core/http.js.map +1 -1
- package/dist/core/keys.d.ts +13 -3
- package/dist/core/keys.js +67 -7
- package/dist/core/keys.js.map +1 -1
- package/dist/core/package.d.ts +12 -3
- package/dist/core/package.js +56 -12
- package/dist/core/package.js.map +1 -1
- package/dist/core/secret_patterns.d.ts +6 -0
- package/dist/core/secret_patterns.js +20 -0
- package/dist/core/secret_patterns.js.map +1 -0
- package/dist/core/tabular.d.ts +43 -1
- package/dist/core/tabular.js +149 -6
- package/dist/core/tabular.js.map +1 -1
- package/dist/envelope.js +3 -1
- package/dist/envelope.js.map +1 -1
- package/dist/index.d.ts +29 -22
- package/dist/index.js +45 -46
- package/dist/index.js.map +1 -1
- package/dist/remote/app.js +87 -31
- package/dist/remote/app.js.map +1 -1
- package/dist/remote/oauth.js +8 -1
- package/dist/remote/oauth.js.map +1 -1
- package/dist/remote/pages.d.ts +3 -7
- package/dist/remote/pages.js +49 -19
- package/dist/remote/pages.js.map +1 -1
- package/dist/remote/sign.d.ts +7 -1
- package/dist/remote/sign.js +33 -7
- package/dist/remote/sign.js.map +1 -1
- package/dist/remote/store.js +14 -5
- package/dist/remote/store.js.map +1 -1
- package/dist/report/audit_report.js +29 -7
- package/dist/report/audit_report.js.map +1 -1
- package/dist/report/build_report.js +34 -4
- package/dist/report/build_report.js.map +1 -1
- package/dist/report/method_audit.js +10 -1
- package/dist/report/method_audit.js.map +1 -1
- package/dist/report/scan_secrets.d.ts +4 -1
- package/dist/report/scan_secrets.js +38 -18
- package/dist/report/scan_secrets.js.map +1 -1
- package/dist/server.d.ts +3 -1
- package/dist/server.js +46 -13
- package/dist/server.js.map +1 -1
- package/dist/tools.js +27 -6
- package/dist/tools.js.map +1 -1
- package/guide/WORKFLOW.md +134 -134
- package/guide/assets/disclaimer.txt +1 -1
- package/guide/assets/download_data.ps1 +164 -162
- package/guide/assets/download_data.sh +8 -2
- package/guide/assets/report_template.html +597 -597
- package/guide/references/routing.md +198 -171
- package/guide/references/style_guide.md +398 -398
- package/package.json +56 -56
- package/packs/acf/endpoints.json +114 -114
- package/packs/acf/policy.json +104 -104
- package/packs/actuaries/endpoints.json +82 -82
- package/packs/actuaries/policy.json +109 -109
- package/packs/bea/assets/download_bea.ps1 +182 -182
- package/packs/bea/docs/bea_api.md +461 -461
- package/packs/bea/docs/datasets.md +157 -157
- package/packs/bea/docs/report_patterns.md +112 -112
- package/packs/bea/policy.json +170 -170
- package/packs/bea/source.md +152 -152
- package/packs/bjs/endpoints.json +150 -0
- package/packs/bjs/policy.json +114 -0
- package/packs/bjs/source.md +98 -0
- package/packs/bls/assets/download_bls.ps1 +152 -152
- package/packs/bls/docs/analysis_patterns.md +108 -108
- package/packs/bls/docs/bls_api.md +73 -73
- package/packs/bls/docs/series_ids.md +149 -149
- package/packs/bls/policy.json +238 -238
- package/packs/bts/endpoints.json +346 -346
- package/packs/bts/policy.json +94 -94
- package/packs/cbp/endpoints.json +31 -0
- package/packs/cbp/policy.json +85 -0
- package/packs/cbp/source.md +57 -0
- package/packs/cdc/endpoints.json +197 -197
- package/packs/cdc/policy.json +83 -83
- package/packs/cdc/source.md +1 -1
- package/packs/cdc-wonder/endpoints.json +26 -0
- package/packs/cdc-wonder/policy.json +84 -0
- package/packs/cdc-wonder/source.md +53 -0
- package/packs/census/endpoints.json +99 -99
- package/packs/cfpb/endpoints.json +326 -326
- package/packs/cfpb/policy.json +119 -119
- package/packs/clinical-trials/endpoints.json +166 -166
- package/packs/clinical-trials/policy.json +84 -84
- package/packs/cms/endpoints.json +69 -69
- package/packs/cms/policy.json +96 -96
- package/packs/college-scorecard/endpoints.json +252 -252
- package/packs/college-scorecard/policy.json +106 -106
- package/packs/college-scorecard/source.md +1 -1
- package/packs/congress/endpoints.json +1701 -1701
- package/packs/congress/policy.json +117 -117
- package/packs/cpsc/endpoints.json +91 -0
- package/packs/cpsc/policy.json +91 -0
- package/packs/cpsc/source.md +76 -0
- package/packs/dhs-ohss/endpoints.json +66 -66
- package/packs/dhs-ohss/policy.json +107 -107
- package/packs/dhs-ohss/source.md +1 -1
- package/packs/doj-news/endpoints.json +59 -59
- package/packs/doj-news/policy.json +96 -96
- package/packs/doj-news/source.md +1 -1
- package/packs/dol/endpoints.json +817 -817
- package/packs/dol/policy.json +135 -135
- package/packs/eia/endpoints.json +717 -717
- package/packs/eia/policy.json +112 -112
- package/packs/epa/endpoints.json +291 -291
- package/packs/epa/policy.json +99 -99
- package/packs/epa-aqs/endpoints.json +173 -173
- package/packs/epa-aqs/policy.json +96 -96
- package/packs/fbi/endpoints.json +1841 -1841
- package/packs/fbi/policy.json +118 -118
- package/packs/fbi/source.md +3 -3
- package/packs/fda/endpoints.json +237 -237
- package/packs/fda/policy.json +121 -121
- package/packs/fdic/endpoints.json +416 -416
- package/packs/fdic/policy.json +83 -83
- package/packs/fec/endpoints.json +707 -707
- package/packs/fec/policy.json +93 -93
- package/packs/fed-board/endpoints.json +89 -89
- package/packs/fed-board/policy.json +136 -136
- package/packs/federal-register/endpoints.json +143 -143
- package/packs/federal-register/policy.json +86 -86
- package/packs/fema/endpoints.json +645 -645
- package/packs/fema/policy.json +103 -103
- package/packs/fhfa/endpoints.json +105 -105
- package/packs/fhfa/policy.json +109 -109
- package/packs/fhwa/endpoints.json +48 -0
- package/packs/fhwa/policy.json +91 -0
- package/packs/fhwa/source.md +59 -0
- package/packs/foreign-assistance/endpoints.json +166 -0
- package/packs/foreign-assistance/policy.json +80 -0
- package/packs/foreign-assistance/source.md +76 -0
- package/packs/fred/assets/download_fred.ps1 +122 -122
- package/packs/fred/docs/analysis_patterns.md +124 -124
- package/packs/fred/docs/fred_api.md +94 -94
- package/packs/fred/docs/series.md +188 -188
- package/packs/fred/policy.json +253 -253
- package/packs/fsa/endpoints.json +150 -150
- package/packs/fsa/policy.json +99 -99
- package/packs/fsa/source.md +1 -1
- package/packs/govinfo/endpoints.json +519 -519
- package/packs/gsa-calc/endpoints.json +93 -93
- package/packs/gsa-calc/policy.json +97 -97
- package/packs/hud/endpoints.json +146 -146
- package/packs/hud/policy.json +84 -84
- package/packs/hud/source.md +1 -1
- package/packs/hud-ahar/endpoints.json +92 -0
- package/packs/hud-ahar/policy.json +88 -0
- package/packs/hud-ahar/source.md +72 -0
- package/packs/ipums/docs/report_patterns.md +117 -117
- package/packs/irs-soi/endpoints.json +114 -114
- package/packs/irs-soi/policy.json +107 -107
- package/packs/medicaid/endpoints.json +412 -412
- package/packs/medicaid/policy.json +108 -108
- package/packs/naep/endpoints.json +217 -217
- package/packs/naep/policy.json +100 -100
- package/packs/naep/source.md +1 -1
- package/packs/nces/endpoints.json +101 -0
- package/packs/nces/policy.json +99 -0
- package/packs/nces/source.md +75 -0
- package/packs/nhtsa/endpoints.json +132 -132
- package/packs/nhtsa/policy.json +93 -93
- package/packs/nhtsa/source.md +1 -1
- package/packs/nih/endpoints.json +452 -452
- package/packs/nih/policy.json +86 -86
- package/packs/noaa/endpoints.json +431 -431
- package/packs/noaa/policy.json +95 -95
- package/packs/nrel/endpoints.json +327 -327
- package/packs/nrel/policy.json +96 -96
- package/packs/nsf/endpoints.json +33 -0
- package/packs/nsf/policy.json +83 -0
- package/packs/nsf/source.md +59 -0
- package/packs/nws/endpoints.json +271 -271
- package/packs/nws/policy.json +82 -82
- package/packs/nyfed/endpoints.json +80 -80
- package/packs/nyfed/policy.json +92 -92
- package/packs/omb/endpoints.json +460 -460
- package/packs/omb/policy.json +106 -106
- package/packs/onrr/endpoints.json +198 -0
- package/packs/onrr/policy.json +91 -0
- package/packs/onrr/source.md +55 -0
- package/packs/open-payments/endpoints.json +297 -297
- package/packs/open-payments/policy.json +101 -101
- package/packs/opm/assets/download_opm.ps1 +125 -125
- package/packs/opm/docs/fields.md +184 -184
- package/packs/opm/docs/opm_api.md +71 -71
- package/packs/opm/docs/report_patterns.md +83 -83
- package/packs/opm/lib/data_dictionary.csv +73 -73
- package/packs/pbgc/endpoints.json +483 -483
- package/packs/pbgc/policy.json +94 -94
- package/packs/philfed/endpoints.json +85 -85
- package/packs/philfed/policy.json +94 -94
- package/packs/port.json +65 -0
- package/packs/regulations/endpoints.json +287 -287
- package/packs/regulations/policy.json +86 -86
- package/packs/sba/endpoints.json +34 -0
- package/packs/sba/policy.json +86 -0
- package/packs/sba/source.md +60 -0
- package/packs/sec/endpoints.json +236 -236
- package/packs/sec/policy.json +86 -86
- package/packs/senate-lobbying/endpoints.json +553 -553
- package/packs/senate-lobbying/policy.json +99 -99
- package/packs/snapqc/endpoints.json +1426 -1426
- package/packs/snapqc/policy.json +92 -92
- package/packs/treasury/catalog.json +2267 -2267
- package/packs/treasury/docs/api_guide.md +245 -245
- package/packs/treasury/docs/datasets.md +56 -56
- package/packs/treasury/docs/report_patterns.md +126 -126
- package/packs/treasury/policy.json +159 -159
- package/packs/treasury/source.md +170 -165
- package/packs/treasury-tic/endpoints.json +77 -0
- package/packs/treasury-tic/policy.json +89 -0
- package/packs/treasury-tic/source.md +80 -0
- package/packs/treasury-yields/endpoints.json +226 -226
- package/packs/treasury-yields/policy.json +86 -86
- package/packs/usaspending/endpoints.json +1315 -1315
- package/packs/usaspending/policy.json +109 -109
- package/packs/uscourts/endpoints.json +22 -0
- package/packs/uscourts/policy.json +87 -0
- package/packs/uscourts/source.md +56 -0
- package/packs/usda-fna/policy.json +133 -133
- package/packs/usda-fooddata/endpoints.json +180 -180
- package/packs/usda-fooddata/policy.json +94 -94
- package/packs/usda-nass/endpoints.json +323 -323
- package/packs/usda-nass/policy.json +95 -95
- package/packs/usda-outlook/endpoints.json +86 -86
- package/packs/usda-outlook/policy.json +109 -109
- package/packs/usgs/endpoints.json +313 -313
- package/packs/usgs/policy.json +95 -95
- package/packs/usitc/endpoints.json +74 -74
- package/packs/usitc/policy.json +111 -111
- package/packs/uspto/endpoints.json +141 -141
- package/packs/uspto/policy.json +95 -95
- package/packs/uspto/source.md +1 -1
- package/packs/va/endpoints.json +414 -414
- package/packs/va/policy.json +101 -101
- package/packs/world-bank/endpoints.json +122 -122
- package/packs/world-bank/policy.json +101 -101
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# HUD Annual Homelessness Assessment Report (AHAR) data: Point-in-Time and Housing Inventory Counts
|
|
2
|
+
Depth: standard. Key: none. Manual only. HUD's AHAR data workbooks from the 2025 AHAR Part 1 page on HUD User:
|
|
3
|
+
- Point-in-Time (PIT) counts of people experiencing homelessness on one night in January, by Continuum of Care
|
|
4
|
+
(CoC) and by state, 2007-2025.
|
|
5
|
+
- PIT counts of veterans, 2011-2025.
|
|
6
|
+
- The Housing Inventory Count (HIC) of beds and units, by CoC and by state.
|
|
7
|
+
The workbooks have one sheet per year. huduser.gov refuses scripts, so the user downloads the workbook and
|
|
8
|
+
uploads it.
|
|
9
|
+
|
|
10
|
+
## When to use
|
|
11
|
+
- How many people experience homelessness in a state or CoC, sheltered vs unsheltered; individuals vs
|
|
12
|
+
people in families; chronically homeless; veterans; unaccompanied youth; change over time.
|
|
13
|
+
- Shelter and housing capacity: emergency shelter, transitional housing, safe haven, rapid re-housing and
|
|
14
|
+
permanent supportive housing beds by CoC or state.
|
|
15
|
+
- Not here: Fair Market Rents and income limits (hud, with a HUD User token); renters' cost burden and
|
|
16
|
+
doubled-up households (census ACS); homelessness among students (school-year counts by the Department of
|
|
17
|
+
Education); public housing.
|
|
18
|
+
|
|
19
|
+
## Endpoints
|
|
20
|
+
Every endpoint is a workbook (fetch refuses them; `manual_bundle` gives the links, then `ingest`):
|
|
21
|
+
|
|
22
|
+
| Endpoint | Workbook (2025 edition) |
|
|
23
|
+
|---|---|
|
|
24
|
+
| `pit_by_coc` | 2007-2025-PIT-Counts-by-CoC.xlsb |
|
|
25
|
+
| `pit_by_state` | 2007-2025-PIT-Counts-by-State.xlsb |
|
|
26
|
+
| `hic_by_coc` | 2007-2025-HIC-Counts-by-CoC.xlsx |
|
|
27
|
+
| `hic_by_state` | 2007-2025-HIC-Counts-by-State.xlsx |
|
|
28
|
+
| `veterans_by_coc` | 2011-2025-PIT-Veteran-Counts-by-CoC.xlsx |
|
|
29
|
+
| `veterans_by_state` | 2011-2025-PIT-Veteran-Counts-by-State.xlsx |
|
|
30
|
+
|
|
31
|
+
Parameters:
|
|
32
|
+
- `year` (required): the sheet to read, named for the count year.
|
|
33
|
+
- `sheet`: any other sheet by name.
|
|
34
|
+
- `header_row`: the header line, 1-based (default 1).
|
|
35
|
+
|
|
36
|
+
## Spec examples
|
|
37
|
+
```json
|
|
38
|
+
{"description": "PIT counts by state, 2025 and 2020",
|
|
39
|
+
"requests": [{"name": "pit_2025", "endpoint": "pit_by_state", "params": {"year": "2025"}},
|
|
40
|
+
{"name": "pit_2020", "endpoint": "pit_by_state", "params": {"year": "2020"}}]}
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Data format and traps
|
|
44
|
+
- Read the columns before using them. The workbook layout was not checked here, because the files cannot be
|
|
45
|
+
downloaded by script. The pack reads the year's sheet with its header row as published; a missing sheet fails
|
|
46
|
+
with the list of sheet names. If the first rows are titles, set `header_row`. Columns are kept exactly as named
|
|
47
|
+
in the workbook, which typically embeds the year in each name, so the same measure has a different name in
|
|
48
|
+
each year's sheet: rename before joining years.
|
|
49
|
+
- The PIT count is one night in late January, not the number of people homeless over a year. HUD requires a
|
|
50
|
+
sheltered count every year but an unsheltered count only every other year (odd years), so a CoC's
|
|
51
|
+
even-year unsheltered number may be carried forward from the year before. In 2021 many CoCs did no
|
|
52
|
+
unsheltered count because of COVID-19, and the 2021 AHAR reported sheltered counts only: 2021 totals are not
|
|
53
|
+
comparable. Say which year and which population.
|
|
54
|
+
- Populations overlap: individuals plus people in families make up the total, while chronically homeless,
|
|
55
|
+
veterans and youth are subsets of it. Never add subpopulations to the total.
|
|
56
|
+
- CoCs are HUD's planning areas: one city, a county, several counties or a "balance of state". CoC numbers
|
|
57
|
+
(like "CA-500") and boundaries change; CoCs merge. Do not equate a CoC with a city without saying so. A state's
|
|
58
|
+
total is the sum of its CoCs.
|
|
59
|
+
- HIC counts beds and units on the PIT night, not people. Permanent supportive housing and rapid re-housing
|
|
60
|
+
beds hold formerly homeless people, who are not in the PIT count.
|
|
61
|
+
- Values are counts as published (numbers in the workbook). Blank cells are blanks: never fill them with 0.
|
|
62
|
+
|
|
63
|
+
## Access
|
|
64
|
+
No key. Manual only: www.huduser.gov answers scripts with an empty HTTP 202 challenge (verified 2026-11-07). The
|
|
65
|
+
user opens https://www.huduser.gov/portal/datasets/ahar/2025-ahar-part-1-pit-estimates-of-homelessness-in-the-us.html
|
|
66
|
+
in a web browser, downloads the workbook (the `manual_bundle` links point at the files) and uploads it as saved;
|
|
67
|
+
then `ingest`. Older AHAR editions have their own pages under https://www.huduser.gov/portal/datasets/ahar.html.
|
|
68
|
+
|
|
69
|
+
## Citation and terms
|
|
70
|
+
"U.S. Department of Housing and Urban Development, 2025 Annual Homelessness Assessment Report (AHAR) to Congress,
|
|
71
|
+
Part 1, 2007-2025 Point-in-Time Estimates by State (or by CoC; Housing Inventory Count), HUD User, downloaded
|
|
72
|
+
<date>." Public domain (17 U.S.C. 105).
|
|
@@ -1,117 +1,117 @@
|
|
|
1
|
-
# Report patterns for IPUMS data products
|
|
2
|
-
|
|
3
|
-
Use this file to shape the analysis for every product, and the layout when a report or paper is
|
|
4
|
-
built. The questions and conventions apply to quick answers too; the publication structure applies only
|
|
5
|
-
in report or paper mode. The aim is a product a careful reviewer would recognize: precise populations,
|
|
6
|
-
weighted estimates, a lead summary with numbers, numbered exhibits that each carry notes and a source
|
|
7
|
-
line, exhibit data shipped alongside, and limitations stated plainly.
|
|
8
|
-
|
|
9
|
-
## Typical questions and how they are framed
|
|
10
|
-
|
|
11
|
-
These recur in published work by statistical and policy agencies (Census Bureau, BLS, GAO, CRS,
|
|
12
|
-
state agencies, research institutes), so there is usually a precedent to follow and benchmark against.
|
|
13
|
-
|
|
14
|
-
- **Employment of immigrants** (ACS, CPS ASEC): people ages 25 to 54, split by sex, by nativity and
|
|
15
|
-
citizenship, by education and by birthplace. Measures: weighted population counts (millions),
|
|
16
|
-
employment rates (employed ÷ population) and average annual earnings of workers. Headline style:
|
|
17
|
-
"In 2018, 46 million people living in the United States, or 14 percent of the population, had been
|
|
18
|
-
born in other countries." Public IPUMS data **cannot** identify legal status (published work that
|
|
19
|
-
reports it imputes it), so report nativity and citizenship (naturalized vs. noncitizen) instead, and
|
|
20
|
-
say so.
|
|
21
|
-
- **Public-sector vs. private-sector pay** (CPS): full-time, full-year civilian workers, in 5 education
|
|
22
|
-
groups (high school diploma or less; some college; bachelor's degree; master's degree; professional
|
|
23
|
-
degree or doctorate). Measures: average hourly wages (annual wages ÷ annual hours), then
|
|
24
|
-
regression-adjusted differences that control for education, experience, occupation, employer size,
|
|
25
|
-
region, veteran status and demographics. With IPUMS alone: raw wage gaps, and optionally an OLS on
|
|
26
|
-
log hourly wage with controls. Benefits are **not** in IPUMS, so say so.
|
|
27
|
-
- **Veterans' labor market outcomes** (CPS, ACS): for example, men ages 22 to 44 who left active duty
|
|
28
|
-
after September 2001, compared with men who never served. Measures: labor force participation rate,
|
|
29
|
-
unemployment rate (unemployed ÷ labor force) and earnings, by age group and by disability rating.
|
|
30
|
-
- Other recurring uses of the same data:
|
|
31
|
-
- labor force projections by age and sex (CPS monthly);
|
|
32
|
-
- minimum wage analyses that count affected workers by hourly wage (CPS outgoing rotation groups: EARNWEEK, HOURWAGE, EARNWT);
|
|
33
|
-
- health insurance coverage by source (ACS HCOV*, CPS ASEC);
|
|
34
|
-
- distribution of household income and poverty (CPS ASEC);
|
|
35
|
-
- immigration surges (ACS and CPS: education and labor force participation of recent arrivals, using YRIMMIG / YRSUSA1);
|
|
36
|
-
- state-level profiles (ACS: large samples with STATEFIP and PUMA detail).
|
|
37
|
-
- **Health insurance and access by group** (NHIS): uninsured at the interview, private vs. public coverage,
|
|
38
|
-
no usual place of care, care delayed or forgone because of cost, by age (0-17, 18-64, 65+), poverty ratio,
|
|
39
|
-
race and Hispanic origin, nativity. Benchmark the headline against NCHS's annual Health Insurance Coverage
|
|
40
|
-
report (2024: 8.2% of all ages and 11.6% of adults 18-64 uninsured). NHIS has no state identifier in the
|
|
41
|
-
public files: state estimates come from the ACS.
|
|
42
|
-
- **Health care spending** (MEPS): mean and median spending per person, the share with any spending,
|
|
43
|
-
out-of-pocket spending and payer shares, by age, insurance type (`COVERTYPE`) and income group (`POVCAT`).
|
|
44
|
-
Benchmark against AHRQ's MEPS-HC summary tables or Statistical Briefs (2022: $6,765 per person). Name the dollar
|
|
45
|
-
year; per-person means hide extreme concentration, so pair them with medians or the top-percentile share.
|
|
46
|
-
|
|
47
|
-
## Conventions to follow
|
|
48
|
-
|
|
49
|
-
**Population and definitions**
|
|
50
|
-
- State the universe exactly: age range, civilian vs. all, noninstitutional (USA GQ 1, 2 or 5 = households), years and geography.
|
|
51
|
-
- The prime-age default is 25–54. It avoids school-age and retirement-age confounding.
|
|
52
|
-
- **Employment rate** = employed ÷ population. **Unemployment rate** = unemployed ÷ labor force. **Participation rate** = labor force ÷ population.
|
|
53
|
-
- **Earnings:** say whose earnings are averaged. Usually it is workers with positive wages, sometimes full-time, full-year workers only. Also say whether self-employment income is included (INCWAGE excludes it; INCEARN in the ACS includes it).
|
|
54
|
-
|
|
55
|
-
**Estimation**
|
|
56
|
-
- Always weight. In the USA collection use PERWT (HHWT for household measures). In the CPS ASEC use ASECWT, in CPS basic monthly use WTFINL (COMPWT for labor-force levels that match BLS), and in the CPS outgoing rotation groups use EARNWT. In NHIS (2019 on) use SAMPWEIGHT on sample adults or sample children; in MEPS use PERWEIGHT (SAQWEIGHT for questionnaire items).
|
|
57
|
-
- Report the unweighted cell size internally. Suppress or footnote cells with fewer than about 100 respondents; unstable estimates are not published.
|
|
58
|
-
- **Dollars:** convert to a single year's dollars and name the year and index ("in 2023 dollars, using the CPI-U-RS"; the PCE price index is a common choice for earnings and income), with CPI99 or `adjustMonetaryValues`. CPS ASEC income is for the **previous calendar year** (ASEC 2023 means 2022 income). ACS income is for the past 12 months, and IPUMS expresses it in survey-year dollars.
|
|
59
|
-
- **Top-coding:** INCWAGE is top-coded, which mostly matters for means at high education levels. Mention it.
|
|
60
|
-
- **Sampling error:** for differences that drive conclusions, note whether they are large relative to sampling error. ACS replicate weights (REPWTP, 80) and CPS ASEC replicate weights (REPWTP, 160) give proper standard errors. CPS basic monthly files have none (REPWTP is ASEC-only) and no design variables: an SE there can only be approximate (clustered by household, `SERIAL` within month). This server does not compute standard errors yet: report which weight was used and that standard errors were not computed.
|
|
61
|
-
|
|
62
|
-
**Publication structure (mirrored in `build_report`, `"mode": "report"`; only when a report is asked for)**
|
|
63
|
-
|
|
64
|
-
The report layout, the **exhibit chartbook**, runs in this order:
|
|
65
|
-
1. **Cover:** "Month Year", the title (with the period for trend reports), and a hero chart that has its own title and unit label.
|
|
66
|
-
2. **At a Glance** (one page of about 300–420 words):
|
|
67
|
-
- one framing paragraph saying what was examined, the population, the period and the dimensions;
|
|
68
|
-
- 3–5 bullets, each opening with a **bold lead phrase.** that names a section, followed by 2–4 sentences with the key numbers.
|
|
69
|
-
3. **Contents:** the Introduction subsections, each section with an "Exhibits a–b" sub-line, the appendixes, and About This Document when there is one.
|
|
70
|
-
4. **List of Exhibits:** grouped under section headings, each entry "N. Title, years".
|
|
71
|
-
5. **Notes About This Report** (also titled "Notes" or "Notes and Definitions"). Short unnumbered paragraphs covering:
|
|
72
|
-
- calendar years;
|
|
73
|
-
- "Numbers in the text and exhibits may not add up to totals because of rounding.";
|
|
74
|
-
- the dollar year and deflator;
|
|
75
|
-
- the data source and universe;
|
|
76
|
-
- key definitions;
|
|
77
|
-
- recession shading, if any;
|
|
78
|
-
- where the data are posted.
|
|
79
|
-
6. **Introduction** (or "Summary and Introduction"):
|
|
80
|
-
- a lead paragraph with the headline numbers;
|
|
81
|
-
- question-style subheadings, one per section ("How Do Demographic Factors Relate to …?");
|
|
82
|
-
- closing subsections such as "Why Focus on People Ages 25 to 54?", "What Data Were Used?" and, when updating earlier work, "What Is New in This Report?".
|
|
83
|
-
7. **Thematic sections** (2–5, unnumbered):
|
|
84
|
-
- Each opens on a full-width page with 1–3 summary paragraphs.
|
|
85
|
-
- Then comes **one exhibit per page**: "Exhibit N." with a title-case title ending in ", 1998 to 2018", a unit label above the chart, and 2–4 narrative paragraphs beside it (levels, changes, comparisons, "(see Exhibit 5)").
|
|
86
|
-
- Below the chart: the data source line ("Data source: `<organization>`, using data from …", or "Data source: Analyst calculations using data from …" when no organization is named), then the notes, then lettered notes ("a.").
|
|
87
|
-
8. **Appendix A: Data and Methods** (or "Appendix: Estimating …"): data, sample, universe, weights, imputations, reliability, comparison with other estimates. Sometimes followed by **Appendix B: Definitions**, an alphabetical glossary.
|
|
88
|
-
9. **About This Document** (optional), in this order:
|
|
89
|
-
- who requested it, or what question prompted it;
|
|
90
|
-
- who prepared it, with what data and tools;
|
|
91
|
-
- the reviewers, if any;
|
|
92
|
-
- the date.
|
|
93
|
-
10. **Footnotes** are numbered in each part (restarting in each section), and cite publications as: Organization, *Title* (Month Year), URL.
|
|
94
|
-
|
|
95
|
-
Other variants, not built by `build_report`:
|
|
96
|
-
- **Long-form**:
|
|
97
|
-
- a Summary with Figure S-1 / Table S-1;
|
|
98
|
-
- "Chapter N:" sections in two-column prose with embedded "Figure 1-1." / "Table 2-3.";
|
|
99
|
-
- Boxes;
|
|
100
|
-
- a List of Tables and Figures at the back.
|
|
101
|
-
- **Hybrid**: full-width pages each with a heading, a paragraph and unnumbered charts; sources pooled in an appendix.
|
|
102
|
-
- Use `"mode": "paper"` when a prose-led product is wanted.
|
|
103
|
-
|
|
104
|
-
**Reading PDFs from sites that block scripts:** some publishers refuse scripts and the in-app browser (HTTP 403, bot-check slider). Fetch `https://web.archive.org/web/<year>/<full pdf url>` and extract the text with pypdf. If the plain year URL returns HTML, use a CDX timestamp with `id_` (for example `https://web.archive.org/web/20240430034127id_/<url>`). Cite the original URL.
|
|
105
|
-
|
|
106
|
-
**Data files:** the codebook, the extract definition (`data/ipums/extract_spec.json`), `reproduce/ipums/`
|
|
107
|
-
(written by `build_report`: the request as `spec.json` and `REPRODUCE.md`, which says how to request the same
|
|
108
|
-
extract again), all attached to the page with Download buttons, and each exhibit's CSV ship with the product,
|
|
109
|
-
so every number can be traced and the data requested again. The extract CSV (microdata) stays local: the
|
|
110
|
-
IPUMS terms forbid redistributing it, so the report never links or embeds it, and `build_report` never puts
|
|
111
|
-
it in a package (`"replication": {"include_microdata": true}` is ignored).
|
|
112
|
-
|
|
113
|
-
**Chart forms**
|
|
114
|
-
- Trends over time: line charts with at most 4–5 series. Facet (one exhibit per sex) rather than drawing 8 lines.
|
|
115
|
-
- Group comparisons at a point in time: grouped columns, or horizontal bars when the category labels are long (education, birthplace).
|
|
116
|
-
- Composition (wages + benefits, population by status): stacked columns.
|
|
117
|
-
- No dual axes, no pie charts, no 3-D.
|
|
1
|
+
# Report patterns for IPUMS data products
|
|
2
|
+
|
|
3
|
+
Use this file to shape the analysis for every product, and the layout when a report or paper is
|
|
4
|
+
built. The questions and conventions apply to quick answers too; the publication structure applies only
|
|
5
|
+
in report or paper mode. The aim is a product a careful reviewer would recognize: precise populations,
|
|
6
|
+
weighted estimates, a lead summary with numbers, numbered exhibits that each carry notes and a source
|
|
7
|
+
line, exhibit data shipped alongside, and limitations stated plainly.
|
|
8
|
+
|
|
9
|
+
## Typical questions and how they are framed
|
|
10
|
+
|
|
11
|
+
These recur in published work by statistical and policy agencies (Census Bureau, BLS, GAO, CRS,
|
|
12
|
+
state agencies, research institutes), so there is usually a precedent to follow and benchmark against.
|
|
13
|
+
|
|
14
|
+
- **Employment of immigrants** (ACS, CPS ASEC): people ages 25 to 54, split by sex, by nativity and
|
|
15
|
+
citizenship, by education and by birthplace. Measures: weighted population counts (millions),
|
|
16
|
+
employment rates (employed ÷ population) and average annual earnings of workers. Headline style:
|
|
17
|
+
"In 2018, 46 million people living in the United States, or 14 percent of the population, had been
|
|
18
|
+
born in other countries." Public IPUMS data **cannot** identify legal status (published work that
|
|
19
|
+
reports it imputes it), so report nativity and citizenship (naturalized vs. noncitizen) instead, and
|
|
20
|
+
say so.
|
|
21
|
+
- **Public-sector vs. private-sector pay** (CPS): full-time, full-year civilian workers, in 5 education
|
|
22
|
+
groups (high school diploma or less; some college; bachelor's degree; master's degree; professional
|
|
23
|
+
degree or doctorate). Measures: average hourly wages (annual wages ÷ annual hours), then
|
|
24
|
+
regression-adjusted differences that control for education, experience, occupation, employer size,
|
|
25
|
+
region, veteran status and demographics. With IPUMS alone: raw wage gaps, and optionally an OLS on
|
|
26
|
+
log hourly wage with controls. Benefits are **not** in IPUMS, so say so.
|
|
27
|
+
- **Veterans' labor market outcomes** (CPS, ACS): for example, men ages 22 to 44 who left active duty
|
|
28
|
+
after September 2001, compared with men who never served. Measures: labor force participation rate,
|
|
29
|
+
unemployment rate (unemployed ÷ labor force) and earnings, by age group and by disability rating.
|
|
30
|
+
- Other recurring uses of the same data:
|
|
31
|
+
- labor force projections by age and sex (CPS monthly);
|
|
32
|
+
- minimum wage analyses that count affected workers by hourly wage (CPS outgoing rotation groups: EARNWEEK, HOURWAGE, EARNWT);
|
|
33
|
+
- health insurance coverage by source (ACS HCOV*, CPS ASEC);
|
|
34
|
+
- distribution of household income and poverty (CPS ASEC);
|
|
35
|
+
- immigration surges (ACS and CPS: education and labor force participation of recent arrivals, using YRIMMIG / YRSUSA1);
|
|
36
|
+
- state-level profiles (ACS: large samples with STATEFIP and PUMA detail).
|
|
37
|
+
- **Health insurance and access by group** (NHIS): uninsured at the interview, private vs. public coverage,
|
|
38
|
+
no usual place of care, care delayed or forgone because of cost, by age (0-17, 18-64, 65+), poverty ratio,
|
|
39
|
+
race and Hispanic origin, nativity. Benchmark the headline against NCHS's annual Health Insurance Coverage
|
|
40
|
+
report (2024: 8.2% of all ages and 11.6% of adults 18-64 uninsured). NHIS has no state identifier in the
|
|
41
|
+
public files: state estimates come from the ACS.
|
|
42
|
+
- **Health care spending** (MEPS): mean and median spending per person, the share with any spending,
|
|
43
|
+
out-of-pocket spending and payer shares, by age, insurance type (`COVERTYPE`) and income group (`POVCAT`).
|
|
44
|
+
Benchmark against AHRQ's MEPS-HC summary tables or Statistical Briefs (2022: $6,765 per person). Name the dollar
|
|
45
|
+
year; per-person means hide extreme concentration, so pair them with medians or the top-percentile share.
|
|
46
|
+
|
|
47
|
+
## Conventions to follow
|
|
48
|
+
|
|
49
|
+
**Population and definitions**
|
|
50
|
+
- State the universe exactly: age range, civilian vs. all, noninstitutional (USA GQ 1, 2 or 5 = households), years and geography.
|
|
51
|
+
- The prime-age default is 25–54. It avoids school-age and retirement-age confounding.
|
|
52
|
+
- **Employment rate** = employed ÷ population. **Unemployment rate** = unemployed ÷ labor force. **Participation rate** = labor force ÷ population.
|
|
53
|
+
- **Earnings:** say whose earnings are averaged. Usually it is workers with positive wages, sometimes full-time, full-year workers only. Also say whether self-employment income is included (INCWAGE excludes it; INCEARN in the ACS includes it).
|
|
54
|
+
|
|
55
|
+
**Estimation**
|
|
56
|
+
- Always weight. In the USA collection use PERWT (HHWT for household measures). In the CPS ASEC use ASECWT, in CPS basic monthly use WTFINL (COMPWT for labor-force levels that match BLS), and in the CPS outgoing rotation groups use EARNWT. In NHIS (2019 on) use SAMPWEIGHT on sample adults or sample children; in MEPS use PERWEIGHT (SAQWEIGHT for questionnaire items).
|
|
57
|
+
- Report the unweighted cell size internally. Suppress or footnote cells with fewer than about 100 respondents; unstable estimates are not published.
|
|
58
|
+
- **Dollars:** convert to a single year's dollars and name the year and index ("in 2023 dollars, using the CPI-U-RS"; the PCE price index is a common choice for earnings and income), with CPI99 or `adjustMonetaryValues`. CPS ASEC income is for the **previous calendar year** (ASEC 2023 means 2022 income). ACS income is for the past 12 months, and IPUMS expresses it in survey-year dollars.
|
|
59
|
+
- **Top-coding:** INCWAGE is top-coded, which mostly matters for means at high education levels. Mention it.
|
|
60
|
+
- **Sampling error:** for differences that drive conclusions, note whether they are large relative to sampling error. ACS replicate weights (REPWTP, 80) and CPS ASEC replicate weights (REPWTP, 160) give proper standard errors. CPS basic monthly files have none (REPWTP is ASEC-only) and no design variables: an SE there can only be approximate (clustered by household, `SERIAL` within month). This server does not compute standard errors yet: report which weight was used and that standard errors were not computed.
|
|
61
|
+
|
|
62
|
+
**Publication structure (mirrored in `build_report`, `"mode": "report"`; only when a report is asked for)**
|
|
63
|
+
|
|
64
|
+
The report layout, the **exhibit chartbook**, runs in this order:
|
|
65
|
+
1. **Cover:** "Month Year", the title (with the period for trend reports), and a hero chart that has its own title and unit label.
|
|
66
|
+
2. **At a Glance** (one page of about 300–420 words):
|
|
67
|
+
- one framing paragraph saying what was examined, the population, the period and the dimensions;
|
|
68
|
+
- 3–5 bullets, each opening with a **bold lead phrase.** that names a section, followed by 2–4 sentences with the key numbers.
|
|
69
|
+
3. **Contents:** the Introduction subsections, each section with an "Exhibits a–b" sub-line, the appendixes, and About This Document when there is one.
|
|
70
|
+
4. **List of Exhibits:** grouped under section headings, each entry "N. Title, years".
|
|
71
|
+
5. **Notes About This Report** (also titled "Notes" or "Notes and Definitions"). Short unnumbered paragraphs covering:
|
|
72
|
+
- calendar years;
|
|
73
|
+
- "Numbers in the text and exhibits may not add up to totals because of rounding.";
|
|
74
|
+
- the dollar year and deflator;
|
|
75
|
+
- the data source and universe;
|
|
76
|
+
- key definitions;
|
|
77
|
+
- recession shading, if any;
|
|
78
|
+
- where the data are posted.
|
|
79
|
+
6. **Introduction** (or "Summary and Introduction"):
|
|
80
|
+
- a lead paragraph with the headline numbers;
|
|
81
|
+
- question-style subheadings, one per section ("How Do Demographic Factors Relate to …?");
|
|
82
|
+
- closing subsections such as "Why Focus on People Ages 25 to 54?", "What Data Were Used?" and, when updating earlier work, "What Is New in This Report?".
|
|
83
|
+
7. **Thematic sections** (2–5, unnumbered):
|
|
84
|
+
- Each opens on a full-width page with 1–3 summary paragraphs.
|
|
85
|
+
- Then comes **one exhibit per page**: "Exhibit N." with a title-case title ending in ", 1998 to 2018", a unit label above the chart, and 2–4 narrative paragraphs beside it (levels, changes, comparisons, "(see Exhibit 5)").
|
|
86
|
+
- Below the chart: the data source line ("Data source: `<organization>`, using data from …", or "Data source: Analyst calculations using data from …" when no organization is named), then the notes, then lettered notes ("a.").
|
|
87
|
+
8. **Appendix A: Data and Methods** (or "Appendix: Estimating …"): data, sample, universe, weights, imputations, reliability, comparison with other estimates. Sometimes followed by **Appendix B: Definitions**, an alphabetical glossary.
|
|
88
|
+
9. **About This Document** (optional), in this order:
|
|
89
|
+
- who requested it, or what question prompted it;
|
|
90
|
+
- who prepared it, with what data and tools;
|
|
91
|
+
- the reviewers, if any;
|
|
92
|
+
- the date.
|
|
93
|
+
10. **Footnotes** are numbered in each part (restarting in each section), and cite publications as: Organization, *Title* (Month Year), URL.
|
|
94
|
+
|
|
95
|
+
Other variants, not built by `build_report`:
|
|
96
|
+
- **Long-form**:
|
|
97
|
+
- a Summary with Figure S-1 / Table S-1;
|
|
98
|
+
- "Chapter N:" sections in two-column prose with embedded "Figure 1-1." / "Table 2-3.";
|
|
99
|
+
- Boxes;
|
|
100
|
+
- a List of Tables and Figures at the back.
|
|
101
|
+
- **Hybrid**: full-width pages each with a heading, a paragraph and unnumbered charts; sources pooled in an appendix.
|
|
102
|
+
- Use `"mode": "paper"` when a prose-led product is wanted.
|
|
103
|
+
|
|
104
|
+
**Reading PDFs from sites that block scripts:** some publishers refuse scripts and the in-app browser (HTTP 403, bot-check slider). Fetch `https://web.archive.org/web/<year>/<full pdf url>` and extract the text with pypdf. If the plain year URL returns HTML, use a CDX timestamp with `id_` (for example `https://web.archive.org/web/20240430034127id_/<url>`). Cite the original URL.
|
|
105
|
+
|
|
106
|
+
**Data files:** the codebook, the extract definition (`data/ipums/extract_spec.json`), `reproduce/ipums/`
|
|
107
|
+
(written by `build_report`: the request as `spec.json` and `REPRODUCE.md`, which says how to request the same
|
|
108
|
+
extract again), all attached to the page with Download buttons, and each exhibit's CSV ship with the product,
|
|
109
|
+
so every number can be traced and the data requested again. The extract CSV (microdata) stays local: the
|
|
110
|
+
IPUMS terms forbid redistributing it, so the report never links or embeds it, and `build_report` never puts
|
|
111
|
+
it in a package (`"replication": {"include_microdata": true}` is ignored).
|
|
112
|
+
|
|
113
|
+
**Chart forms**
|
|
114
|
+
- Trends over time: line charts with at most 4–5 series. Facet (one exhibit per sex) rather than drawing 8 lines.
|
|
115
|
+
- Group comparisons at a point in time: grouped columns, or horizontal bars when the category labels are long (education, birthplace).
|
|
116
|
+
- Composition (wages + benefits, population by status): stacked columns.
|
|
117
|
+
- No dual axes, no pie charts, no 3-D.
|
|
@@ -1,114 +1,114 @@
|
|
|
1
|
-
{
|
|
2
|
-
"base_url": "https://www.irs.gov/pub/irs-soi",
|
|
3
|
-
"timeout": 120,
|
|
4
|
-
"endpoints": {
|
|
5
|
-
"zip_agi": {
|
|
6
|
-
"path": "/{yy}zpallagi.csv",
|
|
7
|
-
"method": "GET",
|
|
8
|
-
"description": "Individual income tax returns by ZIP code and AGI size class (6 classes): number of returns (N*), amounts in thousands of dollars (A*), about 160 items. One all-states file per tax year (216 MB for TY2022), streamed to a temporary file and filtered by state while reading. agi_classes=false reads {yy}zpallnoagi.csv (one row per ZIP, AGI_STUB 0). Tax years 2011-2023.",
|
|
9
|
-
"first_year": 2011,
|
|
10
|
-
"last_year_verified": 2023,
|
|
11
|
-
"stream": true,
|
|
12
|
-
"params": {
|
|
13
|
-
"tax_year": {"in": "spec", "desc": "tax year, e.g. 2022 (or start_year/end_year for several; each is one file)"},
|
|
14
|
-
"start_year": {"in": "spec", "desc": "first tax year of a range"},
|
|
15
|
-
"end_year": {"in": "spec", "desc": "last tax year of a range"},
|
|
16
|
-
"state": {"in": "spec", "desc": "state(s): USPS code, FIPS code or name, or a list (required unless zipcode is given)"},
|
|
17
|
-
"zipcode": {"in": "spec", "desc": "5-digit ZIP code(s) to keep; 00000 = state total, 99999 = small and nonresidential ZIPs combined"},
|
|
18
|
-
"agi_stub": {"in": "spec", "desc": "AGI size class(es) to keep, 1-6 (see source.md)"},
|
|
19
|
-
"agi_classes": {"in": "spec", "desc": "false: read the file without AGI classes (one row per ZIP); default true"},
|
|
20
|
-
"columns": {"in": "spec", "desc": "variables to keep, e.g. [\"N1\", \"A00100\"]; the identifier columns are always kept"}
|
|
21
|
-
},
|
|
22
|
-
"records": "$csv",
|
|
23
|
-
"paging": {"type": "none"},
|
|
24
|
-
"max_rows": 200000,
|
|
25
|
-
"flatten": false
|
|
26
|
-
},
|
|
27
|
-
"county_agi": {
|
|
28
|
-
"path": "/{yy}incyallagi.csv",
|
|
29
|
-
"method": "GET",
|
|
30
|
-
"description": "Individual income tax returns by county and AGI size class (8 classes), with each state's total as COUNTYFIPS 000: number of returns (N*), amounts in thousands of dollars (A*), about 160 items. One all-states file per tax year (36 MB). agi_classes=false reads {yy}incyallnoagi.csv (one row per county, AGI_STUB 0). Tax years 2011-2023.",
|
|
31
|
-
"first_year": 2011,
|
|
32
|
-
"last_year_verified": 2023,
|
|
33
|
-
"params": {
|
|
34
|
-
"tax_year": {"in": "spec", "desc": "tax year, e.g. 2022 (or start_year/end_year)"},
|
|
35
|
-
"start_year": {"in": "spec", "desc": "first tax year of a range"},
|
|
36
|
-
"end_year": {"in": "spec", "desc": "last tax year of a range"},
|
|
37
|
-
"state": {"in": "spec", "desc": "state(s): USPS code, FIPS code or name, or a list (default: all states)"},
|
|
38
|
-
"county": {"in": "spec", "desc": "county(ies): 3-digit county FIPS (with state), 5-digit state+county FIPS, or the county name as published (\"Montgomery County\"); 000 = the state total"},
|
|
39
|
-
"agi_stub": {"in": "spec", "desc": "AGI size class(es) to keep, 1-8 (see source.md)"},
|
|
40
|
-
"agi_classes": {"in": "spec", "desc": "false: read the file without AGI classes; default true"},
|
|
41
|
-
"columns": {"in": "spec", "desc": "variables to keep; the identifier columns are always kept"}
|
|
42
|
-
},
|
|
43
|
-
"records": "$csv",
|
|
44
|
-
"paging": {"type": "none"},
|
|
45
|
-
"max_rows": 200000,
|
|
46
|
-
"flatten": false
|
|
47
|
-
},
|
|
48
|
-
"state_agi": {
|
|
49
|
-
"path": "/{yy}in55cmcsv.csv",
|
|
50
|
-
"method": "GET",
|
|
51
|
-
"description": "Historic Table 2: individual income tax returns by state and AGI size class (AGI_STUB 0 = all returns, 1-10 = classes), for 50 states, DC, Puerto Rico (PR), other areas (OA) and the United States (US); amounts in thousands of dollars. One file per tax year (0.8 MB): tax years 2012 and 2014-2023 (2013 is published only as .xlsx). File names vary by year (12/14-17in54cmcsv.csv, 18in55cmagi.csv, 19-23in55cmcsv.csv); the adapter picks the right one.",
|
|
52
|
-
"first_year": 2012,
|
|
53
|
-
"last_year_verified": 2023,
|
|
54
|
-
"params": {
|
|
55
|
-
"tax_year": {"in": "spec", "desc": "tax year, e.g. 2022 (or start_year/end_year)"},
|
|
56
|
-
"start_year": {"in": "spec", "desc": "first tax year of a range"},
|
|
57
|
-
"end_year": {"in": "spec", "desc": "last tax year of a range"},
|
|
58
|
-
"state": {"in": "spec", "desc": "state(s): USPS code, FIPS code or name, or US / PR / OA; default all"},
|
|
59
|
-
"agi_stub": {"in": "spec", "desc": "AGI size class(es) to keep, 0-10 (0 = all returns)"},
|
|
60
|
-
"columns": {"in": "spec", "desc": "variables to keep; STATE and AGI_STUB are always kept"}
|
|
61
|
-
},
|
|
62
|
-
"records": "$csv",
|
|
63
|
-
"paging": {"type": "none"},
|
|
64
|
-
"max_rows": 200000,
|
|
65
|
-
"flatten": false
|
|
66
|
-
},
|
|
67
|
-
"migration": {
|
|
68
|
-
"path": "/{level}{direction}{y1y2}.csv",
|
|
69
|
-
"method": "GET",
|
|
70
|
-
"description": "Taxpayer migration from matched returns of two consecutive filing years: number of returns (N1, about households), individuals (N2) and AGI (thousands of dollars, year-2 return) moving between states or counties. level state|county, direction inflow (into the state/county, by origin) | outflow (out of it, by destination); y1 = first filing year (2022 = 2022-2023). Filing-year pairs 2011-2012 to 2022-2023.",
|
|
71
|
-
"first_year": 2011,
|
|
72
|
-
"last_year_verified": 2022,
|
|
73
|
-
"params": {
|
|
74
|
-
"level": {"in": "spec", "required": true, "enum": ["state", "county"], "desc": "state-to-state or county-to-county flows"},
|
|
75
|
-
"direction": {"in": "spec", "required": true, "enum": ["inflow", "outflow"], "desc": "inflow: into the state/county by origin; outflow: out of it by destination"},
|
|
76
|
-
"y1": {"in": "spec", "desc": "first filing year of the pair, e.g. 2022 for 2022-2023 (or start_y1/end_y1)"},
|
|
77
|
-
"start_y1": {"in": "spec", "desc": "first y1 of a range of pairs"},
|
|
78
|
-
"end_y1": {"in": "spec", "desc": "last y1 of a range of pairs"},
|
|
79
|
-
"state": {"in": "spec", "desc": "the state(s) whose inflow or outflow to keep (destination for inflow, origin for outflow); default all"},
|
|
80
|
-
"county": {"in": "spec", "desc": "county level: the county(ies) whose flows to keep: 3-digit county FIPS (with state) or 5-digit FIPS"}
|
|
81
|
-
},
|
|
82
|
-
"records": "$csv",
|
|
83
|
-
"paging": {"type": "none"},
|
|
84
|
-
"max_rows": 200000,
|
|
85
|
-
"flatten": false
|
|
86
|
-
}
|
|
87
|
-
},
|
|
88
|
-
"catalog": {
|
|
89
|
-
"zip": {
|
|
90
|
-
"id": "zpallagi",
|
|
91
|
-
"name": "SOI ZIP Code Data (all states, by AGI class)",
|
|
92
|
-
"endpoint": "zip_agi",
|
|
93
|
-
"description": "income tax returns, adjusted gross income AGI, wages, credits by ZIP code and income size class; zip code income"
|
|
94
|
-
},
|
|
95
|
-
"county": {
|
|
96
|
-
"id": "incyallagi",
|
|
97
|
-
"name": "SOI County Data (all states, by AGI class)",
|
|
98
|
-
"endpoint": "county_agi",
|
|
99
|
-
"description": "income tax returns, adjusted gross income AGI, wages, earned income credit by county and income size class; county income"
|
|
100
|
-
},
|
|
101
|
-
"state": {
|
|
102
|
-
"id": "in55cmcsv",
|
|
103
|
-
"name": "SOI Historic Table 2 (state data by AGI class)",
|
|
104
|
-
"endpoint": "state_agi",
|
|
105
|
-
"description": "income tax returns and AGI by state and income size class including $1 million or more; national total"
|
|
106
|
-
},
|
|
107
|
-
"migration": {
|
|
108
|
-
"id": "migration",
|
|
109
|
-
"name": "SOI Migration Data (state-to-state and county-to-county)",
|
|
110
|
-
"endpoint": "migration",
|
|
111
|
-
"description": "taxpayer migration inflow outflow between states and counties; households, people and income moving; net migration"
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"base_url": "https://www.irs.gov/pub/irs-soi",
|
|
3
|
+
"timeout": 120,
|
|
4
|
+
"endpoints": {
|
|
5
|
+
"zip_agi": {
|
|
6
|
+
"path": "/{yy}zpallagi.csv",
|
|
7
|
+
"method": "GET",
|
|
8
|
+
"description": "Individual income tax returns by ZIP code and AGI size class (6 classes): number of returns (N*), amounts in thousands of dollars (A*), about 160 items. One all-states file per tax year (216 MB for TY2022), streamed to a temporary file and filtered by state while reading. agi_classes=false reads {yy}zpallnoagi.csv (one row per ZIP, AGI_STUB 0). Tax years 2011-2023.",
|
|
9
|
+
"first_year": 2011,
|
|
10
|
+
"last_year_verified": 2023,
|
|
11
|
+
"stream": true,
|
|
12
|
+
"params": {
|
|
13
|
+
"tax_year": {"in": "spec", "desc": "tax year, e.g. 2022 (or start_year/end_year for several; each is one file)"},
|
|
14
|
+
"start_year": {"in": "spec", "desc": "first tax year of a range"},
|
|
15
|
+
"end_year": {"in": "spec", "desc": "last tax year of a range"},
|
|
16
|
+
"state": {"in": "spec", "desc": "state(s): USPS code, FIPS code or name, or a list (required unless zipcode is given)"},
|
|
17
|
+
"zipcode": {"in": "spec", "desc": "5-digit ZIP code(s) to keep; 00000 = state total, 99999 = small and nonresidential ZIPs combined"},
|
|
18
|
+
"agi_stub": {"in": "spec", "desc": "AGI size class(es) to keep, 1-6 (see source.md)"},
|
|
19
|
+
"agi_classes": {"in": "spec", "desc": "false: read the file without AGI classes (one row per ZIP); default true"},
|
|
20
|
+
"columns": {"in": "spec", "desc": "variables to keep, e.g. [\"N1\", \"A00100\"]; the identifier columns are always kept"}
|
|
21
|
+
},
|
|
22
|
+
"records": "$csv",
|
|
23
|
+
"paging": {"type": "none"},
|
|
24
|
+
"max_rows": 200000,
|
|
25
|
+
"flatten": false
|
|
26
|
+
},
|
|
27
|
+
"county_agi": {
|
|
28
|
+
"path": "/{yy}incyallagi.csv",
|
|
29
|
+
"method": "GET",
|
|
30
|
+
"description": "Individual income tax returns by county and AGI size class (8 classes), with each state's total as COUNTYFIPS 000: number of returns (N*), amounts in thousands of dollars (A*), about 160 items. One all-states file per tax year (36 MB). agi_classes=false reads {yy}incyallnoagi.csv (one row per county, AGI_STUB 0). Tax years 2011-2023.",
|
|
31
|
+
"first_year": 2011,
|
|
32
|
+
"last_year_verified": 2023,
|
|
33
|
+
"params": {
|
|
34
|
+
"tax_year": {"in": "spec", "desc": "tax year, e.g. 2022 (or start_year/end_year)"},
|
|
35
|
+
"start_year": {"in": "spec", "desc": "first tax year of a range"},
|
|
36
|
+
"end_year": {"in": "spec", "desc": "last tax year of a range"},
|
|
37
|
+
"state": {"in": "spec", "desc": "state(s): USPS code, FIPS code or name, or a list (default: all states)"},
|
|
38
|
+
"county": {"in": "spec", "desc": "county(ies): 3-digit county FIPS (with state), 5-digit state+county FIPS, or the county name as published (\"Montgomery County\"); 000 = the state total"},
|
|
39
|
+
"agi_stub": {"in": "spec", "desc": "AGI size class(es) to keep, 1-8 (see source.md)"},
|
|
40
|
+
"agi_classes": {"in": "spec", "desc": "false: read the file without AGI classes; default true"},
|
|
41
|
+
"columns": {"in": "spec", "desc": "variables to keep; the identifier columns are always kept"}
|
|
42
|
+
},
|
|
43
|
+
"records": "$csv",
|
|
44
|
+
"paging": {"type": "none"},
|
|
45
|
+
"max_rows": 200000,
|
|
46
|
+
"flatten": false
|
|
47
|
+
},
|
|
48
|
+
"state_agi": {
|
|
49
|
+
"path": "/{yy}in55cmcsv.csv",
|
|
50
|
+
"method": "GET",
|
|
51
|
+
"description": "Historic Table 2: individual income tax returns by state and AGI size class (AGI_STUB 0 = all returns, 1-10 = classes), for 50 states, DC, Puerto Rico (PR), other areas (OA) and the United States (US); amounts in thousands of dollars. One file per tax year (0.8 MB): tax years 2012 and 2014-2023 (2013 is published only as .xlsx). File names vary by year (12/14-17in54cmcsv.csv, 18in55cmagi.csv, 19-23in55cmcsv.csv); the adapter picks the right one.",
|
|
52
|
+
"first_year": 2012,
|
|
53
|
+
"last_year_verified": 2023,
|
|
54
|
+
"params": {
|
|
55
|
+
"tax_year": {"in": "spec", "desc": "tax year, e.g. 2022 (or start_year/end_year)"},
|
|
56
|
+
"start_year": {"in": "spec", "desc": "first tax year of a range"},
|
|
57
|
+
"end_year": {"in": "spec", "desc": "last tax year of a range"},
|
|
58
|
+
"state": {"in": "spec", "desc": "state(s): USPS code, FIPS code or name, or US / PR / OA; default all"},
|
|
59
|
+
"agi_stub": {"in": "spec", "desc": "AGI size class(es) to keep, 0-10 (0 = all returns)"},
|
|
60
|
+
"columns": {"in": "spec", "desc": "variables to keep; STATE and AGI_STUB are always kept"}
|
|
61
|
+
},
|
|
62
|
+
"records": "$csv",
|
|
63
|
+
"paging": {"type": "none"},
|
|
64
|
+
"max_rows": 200000,
|
|
65
|
+
"flatten": false
|
|
66
|
+
},
|
|
67
|
+
"migration": {
|
|
68
|
+
"path": "/{level}{direction}{y1y2}.csv",
|
|
69
|
+
"method": "GET",
|
|
70
|
+
"description": "Taxpayer migration from matched returns of two consecutive filing years: number of returns (N1, about households), individuals (N2) and AGI (thousands of dollars, year-2 return) moving between states or counties. level state|county, direction inflow (into the state/county, by origin) | outflow (out of it, by destination); y1 = first filing year (2022 = 2022-2023). Filing-year pairs 2011-2012 to 2022-2023.",
|
|
71
|
+
"first_year": 2011,
|
|
72
|
+
"last_year_verified": 2022,
|
|
73
|
+
"params": {
|
|
74
|
+
"level": {"in": "spec", "required": true, "enum": ["state", "county"], "desc": "state-to-state or county-to-county flows"},
|
|
75
|
+
"direction": {"in": "spec", "required": true, "enum": ["inflow", "outflow"], "desc": "inflow: into the state/county by origin; outflow: out of it by destination"},
|
|
76
|
+
"y1": {"in": "spec", "desc": "first filing year of the pair, e.g. 2022 for 2022-2023 (or start_y1/end_y1)"},
|
|
77
|
+
"start_y1": {"in": "spec", "desc": "first y1 of a range of pairs"},
|
|
78
|
+
"end_y1": {"in": "spec", "desc": "last y1 of a range of pairs"},
|
|
79
|
+
"state": {"in": "spec", "desc": "the state(s) whose inflow or outflow to keep (destination for inflow, origin for outflow); default all"},
|
|
80
|
+
"county": {"in": "spec", "desc": "county level: the county(ies) whose flows to keep: 3-digit county FIPS (with state) or 5-digit FIPS"}
|
|
81
|
+
},
|
|
82
|
+
"records": "$csv",
|
|
83
|
+
"paging": {"type": "none"},
|
|
84
|
+
"max_rows": 200000,
|
|
85
|
+
"flatten": false
|
|
86
|
+
}
|
|
87
|
+
},
|
|
88
|
+
"catalog": {
|
|
89
|
+
"zip": {
|
|
90
|
+
"id": "zpallagi",
|
|
91
|
+
"name": "SOI ZIP Code Data (all states, by AGI class)",
|
|
92
|
+
"endpoint": "zip_agi",
|
|
93
|
+
"description": "income tax returns, adjusted gross income AGI, wages, credits by ZIP code and income size class; zip code income"
|
|
94
|
+
},
|
|
95
|
+
"county": {
|
|
96
|
+
"id": "incyallagi",
|
|
97
|
+
"name": "SOI County Data (all states, by AGI class)",
|
|
98
|
+
"endpoint": "county_agi",
|
|
99
|
+
"description": "income tax returns, adjusted gross income AGI, wages, earned income credit by county and income size class; county income"
|
|
100
|
+
},
|
|
101
|
+
"state": {
|
|
102
|
+
"id": "in55cmcsv",
|
|
103
|
+
"name": "SOI Historic Table 2 (state data by AGI class)",
|
|
104
|
+
"endpoint": "state_agi",
|
|
105
|
+
"description": "income tax returns and AGI by state and income size class including $1 million or more; national total"
|
|
106
|
+
},
|
|
107
|
+
"migration": {
|
|
108
|
+
"id": "migration",
|
|
109
|
+
"name": "SOI Migration Data (state-to-state and county-to-county)",
|
|
110
|
+
"endpoint": "migration",
|
|
111
|
+
"description": "taxpayer migration inflow outflow between states and counties; households, people and income moving; net migration"
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|