data-analyst-mcp 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/README.md +28 -4
  2. package/dist/adapters/bjs.d.ts +13 -0
  3. package/dist/adapters/bjs.js +28 -0
  4. package/dist/adapters/bjs.js.map +1 -0
  5. package/dist/adapters/cbp.d.ts +5 -0
  6. package/dist/adapters/cbp.js +64 -0
  7. package/dist/adapters/cbp.js.map +1 -0
  8. package/dist/adapters/cdc-wonder.d.ts +5 -0
  9. package/dist/adapters/cdc-wonder.js +68 -0
  10. package/dist/adapters/cdc-wonder.js.map +1 -0
  11. package/dist/adapters/cpsc.d.ts +4 -0
  12. package/dist/adapters/cpsc.js +125 -0
  13. package/dist/adapters/cpsc.js.map +1 -0
  14. package/dist/adapters/fhwa.d.ts +9 -0
  15. package/dist/adapters/fhwa.js +16 -0
  16. package/dist/adapters/fhwa.js.map +1 -0
  17. package/dist/adapters/foreign-assistance.d.ts +9 -0
  18. package/dist/adapters/foreign-assistance.js +42 -0
  19. package/dist/adapters/foreign-assistance.js.map +1 -0
  20. package/dist/adapters/hud-ahar.d.ts +3 -0
  21. package/dist/adapters/hud-ahar.js +80 -0
  22. package/dist/adapters/hud-ahar.js.map +1 -0
  23. package/dist/adapters/nces.d.ts +10 -0
  24. package/dist/adapters/nces.js +32 -0
  25. package/dist/adapters/nces.js.map +1 -0
  26. package/dist/adapters/nsf.d.ts +11 -0
  27. package/dist/adapters/nsf.js +25 -0
  28. package/dist/adapters/nsf.js.map +1 -0
  29. package/dist/adapters/onrr.d.ts +9 -0
  30. package/dist/adapters/onrr.js +35 -0
  31. package/dist/adapters/onrr.js.map +1 -0
  32. package/dist/adapters/sba.d.ts +10 -0
  33. package/dist/adapters/sba.js +134 -0
  34. package/dist/adapters/sba.js.map +1 -0
  35. package/dist/adapters/treasury-tic.d.ts +30 -0
  36. package/dist/adapters/treasury-tic.js +149 -0
  37. package/dist/adapters/treasury-tic.js.map +1 -0
  38. package/dist/adapters/uscourts.d.ts +8 -0
  39. package/dist/adapters/uscourts.js +8 -0
  40. package/dist/adapters/uscourts.js.map +1 -0
  41. package/dist/analysis/cite.d.ts +9 -1
  42. package/dist/analysis/cite.js +9 -2
  43. package/dist/analysis/cite.js.map +1 -1
  44. package/dist/bin/cli.d.ts +2 -0
  45. package/dist/bin/cli.js +17 -0
  46. package/dist/bin/cli.js.map +1 -1
  47. package/dist/core/generic-rest.js +42 -17
  48. package/dist/core/generic-rest.js.map +1 -1
  49. package/dist/core/guard.d.ts +32 -0
  50. package/dist/core/guard.js +176 -0
  51. package/dist/core/guard.js.map +1 -0
  52. package/dist/core/http.d.ts +10 -0
  53. package/dist/core/http.js +36 -7
  54. package/dist/core/http.js.map +1 -1
  55. package/dist/core/keys.d.ts +13 -3
  56. package/dist/core/keys.js +67 -7
  57. package/dist/core/keys.js.map +1 -1
  58. package/dist/core/package.d.ts +12 -3
  59. package/dist/core/package.js +56 -12
  60. package/dist/core/package.js.map +1 -1
  61. package/dist/core/secret_patterns.d.ts +6 -0
  62. package/dist/core/secret_patterns.js +20 -0
  63. package/dist/core/secret_patterns.js.map +1 -0
  64. package/dist/core/tabular.d.ts +43 -1
  65. package/dist/core/tabular.js +149 -6
  66. package/dist/core/tabular.js.map +1 -1
  67. package/dist/envelope.js +3 -1
  68. package/dist/envelope.js.map +1 -1
  69. package/dist/index.d.ts +29 -22
  70. package/dist/index.js +45 -46
  71. package/dist/index.js.map +1 -1
  72. package/dist/remote/app.js +87 -31
  73. package/dist/remote/app.js.map +1 -1
  74. package/dist/remote/oauth.js +8 -1
  75. package/dist/remote/oauth.js.map +1 -1
  76. package/dist/remote/pages.d.ts +3 -7
  77. package/dist/remote/pages.js +49 -19
  78. package/dist/remote/pages.js.map +1 -1
  79. package/dist/remote/sign.d.ts +7 -1
  80. package/dist/remote/sign.js +33 -7
  81. package/dist/remote/sign.js.map +1 -1
  82. package/dist/remote/store.js +14 -5
  83. package/dist/remote/store.js.map +1 -1
  84. package/dist/report/audit_report.js +29 -7
  85. package/dist/report/audit_report.js.map +1 -1
  86. package/dist/report/build_report.js +34 -4
  87. package/dist/report/build_report.js.map +1 -1
  88. package/dist/report/method_audit.js +3 -0
  89. package/dist/report/method_audit.js.map +1 -1
  90. package/dist/report/scan_secrets.js +2 -13
  91. package/dist/report/scan_secrets.js.map +1 -1
  92. package/dist/server.d.ts +3 -1
  93. package/dist/server.js +44 -12
  94. package/dist/server.js.map +1 -1
  95. package/dist/tools.js +27 -6
  96. package/dist/tools.js.map +1 -1
  97. package/guide/assets/download_data.ps1 +2 -0
  98. package/guide/assets/download_data.sh +8 -2
  99. package/guide/references/routing.md +30 -3
  100. package/package.json +1 -1
  101. package/packs/bjs/endpoints.json +150 -0
  102. package/packs/bjs/policy.json +114 -0
  103. package/packs/bjs/source.md +98 -0
  104. package/packs/cbp/endpoints.json +31 -0
  105. package/packs/cbp/policy.json +85 -0
  106. package/packs/cbp/source.md +57 -0
  107. package/packs/cdc/source.md +1 -1
  108. package/packs/cdc-wonder/endpoints.json +26 -0
  109. package/packs/cdc-wonder/policy.json +84 -0
  110. package/packs/cdc-wonder/source.md +53 -0
  111. package/packs/college-scorecard/source.md +1 -1
  112. package/packs/cpsc/endpoints.json +91 -0
  113. package/packs/cpsc/policy.json +91 -0
  114. package/packs/cpsc/source.md +76 -0
  115. package/packs/dhs-ohss/source.md +1 -1
  116. package/packs/doj-news/source.md +1 -1
  117. package/packs/fbi/source.md +3 -3
  118. package/packs/fhwa/endpoints.json +48 -0
  119. package/packs/fhwa/policy.json +91 -0
  120. package/packs/fhwa/source.md +59 -0
  121. package/packs/foreign-assistance/endpoints.json +166 -0
  122. package/packs/foreign-assistance/policy.json +80 -0
  123. package/packs/foreign-assistance/source.md +76 -0
  124. package/packs/fsa/source.md +1 -1
  125. package/packs/hud/source.md +1 -1
  126. package/packs/hud-ahar/endpoints.json +92 -0
  127. package/packs/hud-ahar/policy.json +88 -0
  128. package/packs/hud-ahar/source.md +72 -0
  129. package/packs/naep/source.md +1 -1
  130. package/packs/nces/endpoints.json +101 -0
  131. package/packs/nces/policy.json +99 -0
  132. package/packs/nces/source.md +75 -0
  133. package/packs/nhtsa/source.md +1 -1
  134. package/packs/nsf/endpoints.json +33 -0
  135. package/packs/nsf/policy.json +83 -0
  136. package/packs/nsf/source.md +59 -0
  137. package/packs/onrr/endpoints.json +198 -0
  138. package/packs/onrr/policy.json +91 -0
  139. package/packs/onrr/source.md +55 -0
  140. package/packs/port.json +65 -0
  141. package/packs/sba/endpoints.json +34 -0
  142. package/packs/sba/policy.json +86 -0
  143. package/packs/sba/source.md +60 -0
  144. package/packs/treasury/source.md +1 -1
  145. package/packs/treasury-tic/endpoints.json +77 -0
  146. package/packs/treasury-tic/policy.json +89 -0
  147. package/packs/treasury-tic/source.md +80 -0
  148. package/packs/uscourts/endpoints.json +22 -0
  149. package/packs/uscourts/policy.json +87 -0
  150. package/packs/uscourts/source.md +56 -0
  151. package/packs/uspto/source.md +1 -1
@@ -0,0 +1,92 @@
1
+ {
2
+ "base_url": "https://www.huduser.gov/portal/sites/default/files/xls",
3
+ "timeout": 60,
4
+ "files": {
5
+ "pit_by_coc": "2007-2025-PIT-Counts-by-CoC.xlsb",
6
+ "pit_by_state": "2007-2025-PIT-Counts-by-State.xlsb",
7
+ "hic_by_coc": "2007-2025-HIC-Counts-by-CoC.xlsx",
8
+ "hic_by_state": "2007-2025-HIC-Counts-by-State.xlsx",
9
+ "veterans_by_coc": "2011-2025-PIT-Veteran-Counts-by-CoC.xlsx",
10
+ "veterans_by_state": "2011-2025-PIT-Veteran-Counts-by-State.xlsx"
11
+ },
12
+ "endpoints": {
13
+ "pit_by_coc": {
14
+ "path": "/2007-2025-PIT-Counts-by-CoC.xlsb",
15
+ "method": "GET",
16
+ "description": "Point-in-Time (PIT) counts of people experiencing homelessness by Continuum of Care (CoC), one sheet per year 2007-2025 (2025 AHAR edition). Manual: huduser.gov refuses scripts; the user downloads the workbook and uploads it.",
17
+ "params": {
18
+ "year": {"in": "spec", "required": true, "desc": "the count year, i.e. the workbook sheet to read (2007-2025)"},
19
+ "sheet": {"in": "spec", "desc": "a sheet name other than the year (the workbook's other sheets, e.g. a change table)"},
20
+ "header_row": {"in": "spec", "desc": "the header row of the sheet, 1-based (default 1)"}
21
+ },
22
+ "records": "$xls",
23
+ "paging": {"type": "none"},
24
+ "max_rows": 100000
25
+ },
26
+ "pit_by_state": {
27
+ "path": "/2007-2025-PIT-Counts-by-State.xlsb",
28
+ "method": "GET",
29
+ "description": "PIT counts of people experiencing homelessness by state (and the national total), one sheet per year 2007-2025. Manual.",
30
+ "params": {
31
+ "year": {"in": "spec", "required": true, "desc": "the count year (sheet), 2007-2025"},
32
+ "sheet": {"in": "spec", "desc": "a sheet name other than the year"},
33
+ "header_row": {"in": "spec", "desc": "the header row, 1-based (default 1)"}
34
+ },
35
+ "records": "$xls",
36
+ "paging": {"type": "none"},
37
+ "max_rows": 100000
38
+ },
39
+ "hic_by_coc": {
40
+ "path": "/2007-2025-HIC-Counts-by-CoC.xlsx",
41
+ "method": "GET",
42
+ "description": "Housing Inventory Count (HIC): beds and units for people experiencing homelessness (emergency shelter, transitional, safe haven, rapid re-housing, permanent supportive housing) by CoC, one sheet per year. Manual.",
43
+ "params": {
44
+ "year": {"in": "spec", "required": true, "desc": "the count year (sheet)"},
45
+ "sheet": {"in": "spec", "desc": "a sheet name other than the year"},
46
+ "header_row": {"in": "spec", "desc": "the header row, 1-based (default 1)"}
47
+ },
48
+ "records": "$xlsx",
49
+ "paging": {"type": "none"},
50
+ "max_rows": 100000
51
+ },
52
+ "hic_by_state": {
53
+ "path": "/2007-2025-HIC-Counts-by-State.xlsx",
54
+ "method": "GET",
55
+ "description": "Housing Inventory Count by state, one sheet per year. Manual.",
56
+ "params": {
57
+ "year": {"in": "spec", "required": true, "desc": "the count year (sheet)"},
58
+ "sheet": {"in": "spec", "desc": "a sheet name other than the year"},
59
+ "header_row": {"in": "spec", "desc": "the header row, 1-based (default 1)"}
60
+ },
61
+ "records": "$xlsx",
62
+ "paging": {"type": "none"},
63
+ "max_rows": 100000
64
+ },
65
+ "veterans_by_coc": {
66
+ "path": "/2011-2025-PIT-Veteran-Counts-by-CoC.xlsx",
67
+ "method": "GET",
68
+ "description": "PIT counts of veterans experiencing homelessness by CoC, one sheet per year 2011-2025. Manual.",
69
+ "params": {
70
+ "year": {"in": "spec", "required": true, "desc": "the count year (sheet), 2011-2025"},
71
+ "sheet": {"in": "spec", "desc": "a sheet name other than the year"},
72
+ "header_row": {"in": "spec", "desc": "the header row, 1-based (default 1)"}
73
+ },
74
+ "records": "$xlsx",
75
+ "paging": {"type": "none"},
76
+ "max_rows": 100000
77
+ },
78
+ "veterans_by_state": {
79
+ "path": "/2011-2025-PIT-Veteran-Counts-by-State.xlsx",
80
+ "method": "GET",
81
+ "description": "PIT counts of veterans experiencing homelessness by state, one sheet per year 2011-2025. Manual.",
82
+ "params": {
83
+ "year": {"in": "spec", "required": true, "desc": "the count year (sheet), 2011-2025"},
84
+ "sheet": {"in": "spec", "desc": "a sheet name other than the year"},
85
+ "header_row": {"in": "spec", "desc": "the header row, 1-based (default 1)"}
86
+ },
87
+ "records": "$xlsx",
88
+ "paging": {"type": "none"},
89
+ "max_rows": 100000
90
+ }
91
+ }
92
+ }
@@ -0,0 +1,88 @@
1
+ {
2
+ "id": "hud-ahar",
3
+ "title": "HUD Annual Homelessness Assessment Report (AHAR) data: Point-in-Time and Housing Inventory Counts",
4
+ "agency": "U.S. Department of Housing and Urban Development",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "homelessness",
8
+ "housing"
9
+ ],
10
+ "keywords": [
11
+ "homeless",
12
+ "homelessness",
13
+ "people experiencing homelessness",
14
+ "point-in-time count",
15
+ "pit count",
16
+ "ahar",
17
+ "annual homelessness assessment report",
18
+ "unsheltered",
19
+ "sheltered homeless",
20
+ "chronically homeless",
21
+ "homeless veterans",
22
+ "veteran homelessness",
23
+ "homeless youth",
24
+ "continuum of care",
25
+ "coc",
26
+ "housing inventory count",
27
+ "hic",
28
+ "shelter beds",
29
+ "emergency shelter",
30
+ "permanent supportive housing",
31
+ "rapid re-housing"
32
+ ],
33
+ "questions": [
34
+ "number of people experiencing homelessness by state",
35
+ "unsheltered homeless count in a city or Continuum of Care",
36
+ "change in homelessness since 2020",
37
+ "homeless veterans by state",
38
+ "chronically homeless individuals",
39
+ "emergency shelter and permanent supportive housing beds by CoC"
40
+ ],
41
+ "homepage": "https://www.huduser.gov/portal/datasets/ahar.html",
42
+ "terms_url": "https://www.huduser.gov/portal/datasets/ahar/2025-ahar-part-1-pit-estimates-of-homelessness-in-the-us.html",
43
+ "auth": {
44
+ "env": null,
45
+ "required": "none",
46
+ "placement": "none",
47
+ "signup_url": null,
48
+ "echoes_key": false
49
+ },
50
+ "limits": {
51
+ "rps": 1,
52
+ "burst": 2,
53
+ "max_rows": 100000
54
+ },
55
+ "routes": {
56
+ "manual": {
57
+ "how": "Open https://www.huduser.gov/portal/datasets/ahar/2025-ahar-part-1-pit-estimates-of-homelessness-in-the-us.html in a web browser and download the workbook (2007 - 2025 Point-in-Time Estimates by CoC or by State, Housing Inventory Count by CoC or State, PIT Veteran Counts); huduser.gov refuses scripts. Upload the workbook as saved (.xlsb or .xlsx) with ingest."
58
+ }
59
+ },
60
+ "cache": {
61
+ "mode": "run_only",
62
+ "ttl_hours": null,
63
+ "why": "Workbooks are replaced once a year with the new AHAR edition."
64
+ },
65
+ "redistribution": {
66
+ "mode": "allowed",
67
+ "why": "HUD's AHAR data are U.S. government works in the public domain (17 U.S.C. 105)."
68
+ },
69
+ "notices": [],
70
+ "retention": null,
71
+ "citation": "U.S. Department of Housing and Urban Development, Annual Homelessness Assessment Report (AHAR), {dataset}, HUD User, downloaded {retrieved_long}.",
72
+ "audit_hooks": [],
73
+ "clarify": [
74
+ "Which year? Each sheet is one count year (a single night in late January). Default: the latest year, with 2020 or the prior year for change.",
75
+ "Which population? Total, sheltered, unsheltered, individuals, people in families, chronically homeless, veterans, youth: these overlap. Default: total, split into sheltered and unsheltered.",
76
+ "Which geography? CoCs are HUD's planning areas (a city, a county or a balance of state), not cities or counties as such. Default: states, or the CoC that contains the place named, said as such."
77
+ ],
78
+ "formats": [
79
+ "xlsb",
80
+ "xlsx"
81
+ ],
82
+ "aliases": [
83
+ "AHAR",
84
+ "PIT count",
85
+ "Point-in-Time count",
86
+ "Housing Inventory Count"
87
+ ]
88
+ }
@@ -0,0 +1,72 @@
1
+ # HUD Annual Homelessness Assessment Report (AHAR) data: Point-in-Time and Housing Inventory Counts
2
+ Depth: standard. Key: none. Manual only. HUD's AHAR data workbooks from the 2025 AHAR Part 1 page on HUD User:
3
+ - Point-in-Time (PIT) counts of people experiencing homelessness on one night in January, by Continuum of Care
4
+ (CoC) and by state, 2007-2025.
5
+ - PIT counts of veterans, 2011-2025.
6
+ - The Housing Inventory Count (HIC) of beds and units, by CoC and by state.
7
+ The workbooks have one sheet per year. huduser.gov refuses scripts, so the user downloads the workbook and
8
+ uploads it.
9
+
10
+ ## When to use
11
+ - How many people experience homelessness in a state or CoC, sheltered vs unsheltered; individuals vs
12
+ people in families; chronically homeless; veterans; unaccompanied youth; change over time.
13
+ - Shelter and housing capacity: emergency shelter, transitional housing, safe haven, rapid re-housing and
14
+ permanent supportive housing beds by CoC or state.
15
+ - Not here: Fair Market Rents and income limits (hud, with a HUD User token); renters' cost burden and
16
+ doubled-up households (census ACS); homelessness among students (school-year counts by the Department of
17
+ Education); public housing.
18
+
19
+ ## Endpoints
20
+ Every endpoint is a workbook (fetch refuses them; `manual_bundle` gives the links, then `ingest`):
21
+
22
+ | Endpoint | Workbook (2025 edition) |
23
+ |---|---|
24
+ | `pit_by_coc` | 2007-2025-PIT-Counts-by-CoC.xlsb |
25
+ | `pit_by_state` | 2007-2025-PIT-Counts-by-State.xlsb |
26
+ | `hic_by_coc` | 2007-2025-HIC-Counts-by-CoC.xlsx |
27
+ | `hic_by_state` | 2007-2025-HIC-Counts-by-State.xlsx |
28
+ | `veterans_by_coc` | 2011-2025-PIT-Veteran-Counts-by-CoC.xlsx |
29
+ | `veterans_by_state` | 2011-2025-PIT-Veteran-Counts-by-State.xlsx |
30
+
31
+ Parameters:
32
+ - `year` (required): the sheet to read, named for the count year.
33
+ - `sheet`: any other sheet by name.
34
+ - `header_row`: the header line, 1-based (default 1).
35
+
36
+ ## Spec examples
37
+ ```json
38
+ {"description": "PIT counts by state, 2025 and 2020",
39
+ "requests": [{"name": "pit_2025", "endpoint": "pit_by_state", "params": {"year": "2025"}},
40
+ {"name": "pit_2020", "endpoint": "pit_by_state", "params": {"year": "2020"}}]}
41
+ ```
42
+
43
+ ## Data format and traps
44
+ - Read the columns before using them. The workbook layout was not checked here, because the files cannot be
45
+ downloaded by script. The pack reads the year's sheet with its header row as published; a missing sheet fails
46
+ with the list of sheet names. If the first rows are titles, set `header_row`. Columns are kept exactly as named
47
+ in the workbook, which typically embeds the year in each name, so the same measure has a different name in
48
+ each year's sheet: rename before joining years.
49
+ - The PIT count is one night in late January, not the number of people homeless over a year. HUD requires a
50
+ sheltered count every year but an unsheltered count only every other year (odd years), so a CoC's
51
+ even-year unsheltered number may be carried forward from the year before. In 2021 many CoCs did no
52
+ unsheltered count because of COVID-19, and the 2021 AHAR reported sheltered counts only: 2021 totals are not
53
+ comparable. Say which year and which population.
54
+ - Populations overlap: individuals plus people in families make up the total, while chronically homeless,
55
+ veterans and youth are subsets of it. Never add subpopulations to the total.
56
+ - CoCs are HUD's planning areas: one city, a county, several counties or a "balance of state". CoC numbers
57
+ (like "CA-500") and boundaries change; CoCs merge. Do not equate a CoC with a city without saying so. A state's
58
+ total is the sum of its CoCs.
59
+ - HIC counts beds and units on the PIT night, not people. Permanent supportive housing and rapid re-housing
60
+ beds hold formerly homeless people, who are not in the PIT count.
61
+ - Values are counts as published (numbers in the workbook). Blank cells are blanks: never fill them with 0.
62
+
63
+ ## Access
64
+ No key. Manual only: www.huduser.gov answers scripts with an empty HTTP 202 challenge (verified 2026-11-07). The
65
+ user opens https://www.huduser.gov/portal/datasets/ahar/2025-ahar-part-1-pit-estimates-of-homelessness-in-the-us.html
66
+ in a web browser, downloads the workbook (the `manual_bundle` links point at the files) and uploads it as saved;
67
+ then `ingest`. Older AHAR editions have their own pages under https://www.huduser.gov/portal/datasets/ahar.html.
68
+
69
+ ## Citation and terms
70
+ "U.S. Department of Housing and Urban Development, 2025 Annual Homelessness Assessment Report (AHAR) to Congress,
71
+ Part 1, 2007-2025 Point-in-Time Estimates by State (or by CoC; Housing Inventory Count), HUD User, downloaded
72
+ <date>." Public domain (17 U.S.C. 105).
@@ -8,7 +8,7 @@ student group, with standard errors and NAEP's own significance tests.
8
8
  science, writing, civics, U.S. history, geography, economics, technology and engineering literacy,
9
9
  music. States versus the nation, change since 2019, gaps by race, sex, economic disadvantage.
10
10
  - Not here: state test results (each state's own exams, not comparable across states), graduation rates
11
- and school finance (nces or ed packs if present), college outcomes (college-scorecard).
11
+ and school finance (nces), college outcomes (college-scorecard).
12
12
  - NAEP is a sample survey: every number is an estimate. It reports groups, never students or schools.
13
13
 
14
14
  ## Endpoints and datasets
@@ -0,0 +1,101 @@
1
+ {
2
+ "base_url": "https://nces.ed.gov",
3
+ "timeout": 180,
4
+ "endpoints": {
5
+ "digest_table": {
6
+ "path": "/programs/digest/d{edition}/tables/xls/tabn{table}.xlsx",
7
+ "method": "GET",
8
+ "description": "One table of the Digest of Education Statistics as its Excel file: enrollment, teachers, finances, degrees, outcomes, from pre-K to postsecondary. Default: every line numbered (row, col1...) to find the header lines; then header_lines (usually '3-4') names the columns.",
9
+ "params": {
10
+ "edition": {
11
+ "in": "path",
12
+ "required": true,
13
+ "desc": "two-digit Digest edition, e.g. 25 (Digest 2025); not every table is in every edition"
14
+ },
15
+ "table": {
16
+ "in": "path",
17
+ "required": true,
18
+ "desc": "table number, e.g. 203.10 (public school enrollment by grade)"
19
+ },
20
+ "header_lines": {
21
+ "in": "records",
22
+ "desc": "the header line numbers, e.g. '3-4'"
23
+ },
24
+ "header_row": {
25
+ "in": "records",
26
+ "desc": "one header line instead; 0 (default) returns every line numbered"
27
+ },
28
+ "sheet": {
29
+ "in": "records",
30
+ "desc": "sheet name or number (default 1)"
31
+ },
32
+ "carry": {
33
+ "in": "records",
34
+ "desc": "which header lines span to the right over blank cells: top (default), all, none; check the names against the numbered lines"
35
+ }
36
+ },
37
+ "defaults": {
38
+ "header_row": 0
39
+ },
40
+ "records": "$xlsx",
41
+ "paging": {
42
+ "type": "none"
43
+ },
44
+ "max_rows": 20000
45
+ },
46
+ "ipeds": {
47
+ "path": "/ipeds/datacenter/data/{file}.zip",
48
+ "method": "GET",
49
+ "description": "An IPEDS complete data file (one row per institution, or institution by category), e.g. HD2023 directory, EFFY2023 12-month enrollment, C2023_A completions by CIP code, ADM2023 admissions, SFA2223 student aid, F2223_F2 finance. The revised (_rv) version is read when the zip has one.",
50
+ "params": {
51
+ "file": {
52
+ "in": "path",
53
+ "required": true,
54
+ "desc": "IPEDS file name as on the Data Center's Complete Data Files page, e.g. HD2023, EFFY2023, C2023_A"
55
+ },
56
+ "unitid": {
57
+ "in": "spec",
58
+ "desc": "keep only these institutions (UNITID, a list works)"
59
+ },
60
+ "state": {
61
+ "in": "spec",
62
+ "desc": "keep only these states (two-letter STABBR; files that have that column, e.g. HD)"
63
+ },
64
+ "member": {
65
+ "in": "records",
66
+ "desc": "the CSV in the zip, e.g. c2023_a.csv for the original instead of the revised file"
67
+ }
68
+ },
69
+ "records": "$zip:*.csv",
70
+ "prefer": "_rv[.]csv$",
71
+ "paging": {
72
+ "type": "none"
73
+ },
74
+ "max_rows": 1000000
75
+ },
76
+ "ipeds_dictionary": {
77
+ "path": "/ipeds/datacenter/data/{file}_Dict.zip",
78
+ "method": "GET",
79
+ "description": "The data dictionary of an IPEDS file: variable names and titles (sheet Varlist), code values and labels (Frequencies), long descriptions (Description).",
80
+ "params": {
81
+ "file": {
82
+ "in": "path",
83
+ "required": true,
84
+ "desc": "the IPEDS file name, e.g. HD2023"
85
+ },
86
+ "sheet": {
87
+ "in": "records",
88
+ "desc": "Varlist (default), Frequencies, Description, Introduction"
89
+ }
90
+ },
91
+ "defaults": {
92
+ "sheet": "Varlist"
93
+ },
94
+ "records": "$zip:*.xlsx",
95
+ "paging": {
96
+ "type": "none"
97
+ },
98
+ "max_rows": 100000
99
+ }
100
+ }
101
+ }
@@ -0,0 +1,99 @@
1
+ {
2
+ "id": "nces",
3
+ "title": "National Center for Education Statistics: Digest of Education Statistics tables and IPEDS data files",
4
+ "agency": "U.S. Department of Education, National Center for Education Statistics",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "education",
8
+ "higher education"
9
+ ],
10
+ "keywords": [
11
+ "nces",
12
+ "national center for education statistics",
13
+ "digest of education statistics",
14
+ "school enrollment",
15
+ "public school enrollment",
16
+ "enrollment by grade",
17
+ "teachers",
18
+ "pupil teacher ratio",
19
+ "per-pupil expenditure",
20
+ "school finance",
21
+ "high school graduates",
22
+ "graduation rate",
23
+ "degrees conferred",
24
+ "ipeds",
25
+ "college enrollment",
26
+ "fall enrollment",
27
+ "completions",
28
+ "degrees by field",
29
+ "cip code",
30
+ "institutional characteristics",
31
+ "tuition and fees",
32
+ "admissions",
33
+ "student financial aid",
34
+ "college finance"
35
+ ],
36
+ "questions": [
37
+ "public school enrollment by grade over time",
38
+ "number of teachers and pupil/teacher ratio",
39
+ "current expenditure per pupil by state",
40
+ "bachelor's degrees conferred by field",
41
+ "enrollment at a college or all colleges in a state (IPEDS)",
42
+ "completions by CIP code and award level",
43
+ "admissions and yield at a university"
44
+ ],
45
+ "homepage": "https://nces.ed.gov/",
46
+ "terms_url": "https://nces.ed.gov/ipeds/datacenter/DataFiles.aspx",
47
+ "auth": {
48
+ "env": null,
49
+ "required": "none",
50
+ "placement": "none",
51
+ "signup_url": null,
52
+ "echoes_key": false
53
+ },
54
+ "limits": {
55
+ "rps": 2,
56
+ "burst": 4,
57
+ "max_rows": 1000000
58
+ },
59
+ "routes": {
60
+ "api": {
61
+ "base": "https://nces.ed.gov",
62
+ "probe": "https://nces.ed.gov/ipeds/datacenter/data/HD2023_Dict.zip",
63
+ "hosts": [
64
+ "nces.ed.gov"
65
+ ]
66
+ },
67
+ "manual": {
68
+ "how": "Digest: open the table page (https://nces.ed.gov/programs/digest/current_tables.asp, then the table) and click its Excel link, or https://nces.ed.gov/programs/digest/d<yy>/tables/xls/tabn<table>.xlsx. IPEDS: download https://nces.ed.gov/ipeds/datacenter/data/<FILE>.zip from the Complete Data Files page (https://nces.ed.gov/ipeds/datacenter/DataFiles.aspx). Upload the file as saved with ingest."
69
+ }
70
+ },
71
+ "cache": {
72
+ "mode": "run_only",
73
+ "ttl_hours": null,
74
+ "why": "Files are replaced when NCES revises them; data/ is reused within a run."
75
+ },
76
+ "redistribution": {
77
+ "mode": "allowed",
78
+ "why": "NCES publications and IPEDS data are U.S. government works in the public domain (17 U.S.C. 105)."
79
+ },
80
+ "notices": [],
81
+ "retention": null,
82
+ "citation": "U.S. Department of Education, National Center for Education Statistics, {dataset}, nces.ed.gov, retrieved {retrieved_long}.",
83
+ "audit_hooks": [],
84
+ "clarify": [
85
+ "Published table or institution file? Digest tables give national (and often state) totals as NCES published them; IPEDS files give one row per institution to filter and sum. Default: a Digest table for national or state totals, IPEDS for named colleges.",
86
+ "Which year? Digest tables are labelled by school year or fall; IPEDS files by collection year (EFFY2023 = 12-month enrollment for 2022-23). Say which, and whether the IPEDS file is provisional or revised (_rv)."
87
+ ],
88
+ "formats": [
89
+ "xlsx",
90
+ "zip",
91
+ "csv"
92
+ ],
93
+ "aliases": [
94
+ "NCES",
95
+ "IPEDS",
96
+ "Digest of Education Statistics",
97
+ "Integrated Postsecondary Education Data System"
98
+ ]
99
+ }
@@ -0,0 +1,75 @@
1
+ # National Center for Education Statistics: Digest of Education Statistics tables and IPEDS data files
2
+ Depth: standard. Key: none.
3
+ - The Digest of Education Statistics: NCES's compendium of published tables (enrollment, teachers, finances,
4
+ graduates, degrees, outcomes, pre-K to postsecondary), as each table's Excel file.
5
+ - IPEDS complete data files: one row per postsecondary institution, or institution by category, for every
6
+ survey component (directory, enrollment, completions, admissions, student aid, finance, staff), with their
7
+ data dictionaries.
8
+
9
+ ## When to use
10
+ - National or state totals as NCES published them: public school enrollment by grade, teachers and
11
+ pupil/teacher ratios, current expenditure per pupil, high school completers, degrees conferred by field and
12
+ level, enrollment trends.
13
+ - A named college or every college in a state: fall or 12-month enrollment, completions by CIP code and award
14
+ level, admissions and test scores, tuition, aid, finance (IPEDS).
15
+ - Not here: earnings and debt of graduates by program and institution (college-scorecard); federal student
16
+ loans and Pell totals (fsa); test scores of K-12 students (naep); school-age population, poverty or
17
+ attainment from surveys (census ACS).
18
+
19
+ ## Endpoints
20
+ | Endpoint | Returns | Parameters |
21
+ |---|---|---|
22
+ | `digest_table` | one Digest table: without `header_lines`, every line numbered (`row`, `col1`...); with it, named columns | `edition` (two digits, e.g. `25`), `table` (e.g. `203.10`), `header_lines` (usually `3-4`), `header_row`, `sheet` |
23
+ | `ipeds` | an IPEDS complete data file (the revised `_rv` CSV when the zip has one) | `file` (e.g. `HD2023`, `EFFY2023`, `C2023_A`), `unitid`, `state` (filters), `member` |
24
+ | `ipeds_dictionary` | that file's dictionary sheet | `file`, `sheet` (`Varlist` default, `Frequencies` for code labels, `Description`) |
25
+
26
+ ## Spec examples
27
+ ```json
28
+ {"description": "Public school enrollment by grade, fall 1980-2024 (Digest 2025 table 203.10)",
29
+ "requests": [{"name": "t203_10", "endpoint": "digest_table", "params": {"edition": "25", "table": "203.10", "header_lines": "3-4"}}]}
30
+ ```
31
+ ```json
32
+ {"description": "Colleges in Vermont (IPEDS directory 2023-24) and the code labels",
33
+ "requests": [{"name": "vt_colleges", "endpoint": "ipeds", "params": {"file": "HD2023", "state": "VT"}},
34
+ {"name": "hd_codes", "endpoint": "ipeds_dictionary", "params": {"file": "HD2023", "sheet": "Frequencies"}}]}
35
+ ```
36
+
37
+ ## Data format and traps
38
+ - Digest tables:
39
+ - Line 1 is the title, line 2 the units ("[In thousands]": multiply), lines 3-4 the stacked header, and the
40
+ first data line is the row of column numbers ("1", "2", ...): drop it. Notes and the source follow the last
41
+ data line. Verified on table 203.10, Digest 2025; other tables differ, so read the raw lines first.
42
+ - Columns named `<header>_colN` hold footnote marks for the value to their left, or nothing. In table
43
+ 203.10's 2024 row, `\3,4\` points to footnotes 3 and 4 below the table. Keep the marks with the value they
44
+ qualify.
45
+ - Values are text with thousands separators. Footnote marks also sit inside header cells (`Ungraded\1\`), and
46
+ some cells start with a line break ("\n1993"): trim them. Symbols such as `†` and `—`, used in many
47
+ tables, are defined in each table's notes; keep them and never treat them as 0. Projections are marked in
48
+ the notes.
49
+ - Each edition renumbers or drops tables, and a table that is not in that edition answers 404. Use the
50
+ current edition's table list (https://nces.ed.gov/programs/digest/current_tables.asp) to find the number.
51
+ - IPEDS:
52
+ - Names. Files are named by collection year: HD2023 is the 2023-24 directory, EFFY2023 the 12-month enrollment
53
+ of 2022-23, C2023_A the completions of July 2022-June 2023. Read the dictionary's Introduction sheet to be
54
+ sure.
55
+ - Revisions. When a zip has a revised file (`_rv`, the final release), it is read instead of the provisional
56
+ one; `member` picks the other.
57
+ - Codes. Columns are coded (`SECTOR`, `CONTROL`, `ICLEVEL`; `X`-prefixed imputation flags): label them from
58
+ the dictionary's Frequencies sheet. Negative codes such as -1 and -2 mean not reported or not applicable;
59
+ never sum them.
60
+ - Joining files. UNITID joins files; a parent/child system (`F1SYSTYP` ...) can report for several campuses.
61
+ - Size. Completions files are large (C2023_A has two 60 MB CSVs): filter by `unitid` or `state`. `state`
62
+ works only on files with `STABBR`, so join HD for state.
63
+ - Counting. Enrollment and completions rows are by category (all students, by race, by sex, by CIP and award
64
+ level), and totals are rows of their own (CIPCODE 99, MAJORNUM 1): pick the total row; never sum across
65
+ categories.
66
+
67
+ ## Access
68
+ No key. `fetch` reads nces.ed.gov directly (verified 2026-11-07). Manual: the Digest table's Excel link, or the
69
+ IPEDS Complete Data Files page (https://nces.ed.gov/ipeds/datacenter/DataFiles.aspx); upload the file as saved;
70
+ then `ingest`.
71
+
72
+ ## Citation and terms
73
+ "U.S. Department of Education, National Center for Education Statistics, Digest of Education Statistics 2025,
74
+ table 203.10" or "... Integrated Postsecondary Education Data System (IPEDS), <file> (<release>), nces.ed.gov,
75
+ retrieved <date>." Public domain (17 U.S.C. 105).
@@ -6,7 +6,7 @@ Investigation (ODI), NCAP 5-Star crash ratings, VIN decoding (vPIC), and child c
6
6
  - Recalls of a make/model/model year or a campaign; complaints about a model (crashes, fires, injuries,
7
7
  components); crash-test star ratings and rollover risk; decoding a VIN; models a make built in a year.
8
8
  - Not here: crash fatality statistics (FARS/CRSS: not covered by this pack; say so), traffic volumes and
9
- transport statistics (bts or dot packs if present), product recalls outside vehicles (cpsc, fda).
9
+ transport statistics (bts, fhwa), product recalls outside vehicles (cpsc, fda).
10
10
  - Records are text and counts of reports. They are not defect rates: there is no fleet size or sales data.
11
11
 
12
12
  ## Endpoints and datasets
@@ -0,0 +1,33 @@
1
+ {
2
+ "base_url": "https://api.nsf.gov/services/v1",
3
+ "timeout": 60,
4
+ "endpoints": {
5
+ "awards": {
6
+ "path": "/awards.json",
7
+ "method": "GET",
8
+ "description": "NSF awards (grants and cooperative agreements) from the NSF Award Search API: title, awardee institution and location, PI, amounts obligated and estimated, dates, NSF directorate, division and program, abstract. Filter by keyword, institution, state, PI, program or date range.",
9
+ "params": {
10
+ "keyword": {"desc": "words in the title or abstract; quote a phrase, e.g. \"machine learning\""},
11
+ "awardeeName": {"desc": "institution name or part of it"},
12
+ "awardeeStateCode": {"desc": "two-letter state of the awardee institution"},
13
+ "perfStateCode": {"desc": "two-letter state where the work is performed"},
14
+ "pdPIName": {"desc": "PI name (last name, or first and last)"},
15
+ "fundProgramName": {"desc": "NSF funding program name"},
16
+ "id": {"desc": "award number(s), comma-separated"},
17
+ "dateStart": {"desc": "award date from, MM/DD/YYYY"},
18
+ "dateEnd": {"desc": "award date to, MM/DD/YYYY"},
19
+ "startDateStart": {"desc": "award start date from, MM/DD/YYYY"},
20
+ "startDateEnd": {"desc": "award start date to, MM/DD/YYYY"},
21
+ "expDateStart": {"desc": "expiration date from, MM/DD/YYYY"},
22
+ "expDateEnd": {"desc": "expiration date to, MM/DD/YYYY"}
23
+ },
24
+ "defaults": {"offset": 1},
25
+ "records": "response.award",
26
+ "total": "response.metadata.totalCount",
27
+ "paging": {"type": "offset", "limit_param": "rpp", "offset_param": "offset", "size": 25, "manual_size": 25},
28
+ "max_rows": 10000,
29
+ "unique_key": "id",
30
+ "flatten": true
31
+ }
32
+ }
33
+ }