data-analyst-mcp 0.3.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/README.md +31 -6
  2. package/dist/adapters/_census_micro.js +2 -2
  3. package/dist/adapters/_census_micro.js.map +1 -1
  4. package/dist/adapters/bjs.d.ts +13 -0
  5. package/dist/adapters/bjs.js +28 -0
  6. package/dist/adapters/bjs.js.map +1 -0
  7. package/dist/adapters/cbp.d.ts +5 -0
  8. package/dist/adapters/cbp.js +64 -0
  9. package/dist/adapters/cbp.js.map +1 -0
  10. package/dist/adapters/cdc-wonder.d.ts +5 -0
  11. package/dist/adapters/cdc-wonder.js +68 -0
  12. package/dist/adapters/cdc-wonder.js.map +1 -0
  13. package/dist/adapters/cpsc.d.ts +4 -0
  14. package/dist/adapters/cpsc.js +125 -0
  15. package/dist/adapters/cpsc.js.map +1 -0
  16. package/dist/adapters/fhwa.d.ts +9 -0
  17. package/dist/adapters/fhwa.js +16 -0
  18. package/dist/adapters/fhwa.js.map +1 -0
  19. package/dist/adapters/foreign-assistance.d.ts +9 -0
  20. package/dist/adapters/foreign-assistance.js +42 -0
  21. package/dist/adapters/foreign-assistance.js.map +1 -0
  22. package/dist/adapters/hud-ahar.d.ts +3 -0
  23. package/dist/adapters/hud-ahar.js +80 -0
  24. package/dist/adapters/hud-ahar.js.map +1 -0
  25. package/dist/adapters/nces.d.ts +10 -0
  26. package/dist/adapters/nces.js +32 -0
  27. package/dist/adapters/nces.js.map +1 -0
  28. package/dist/adapters/nsf.d.ts +11 -0
  29. package/dist/adapters/nsf.js +25 -0
  30. package/dist/adapters/nsf.js.map +1 -0
  31. package/dist/adapters/onrr.d.ts +9 -0
  32. package/dist/adapters/onrr.js +35 -0
  33. package/dist/adapters/onrr.js.map +1 -0
  34. package/dist/adapters/sba.d.ts +10 -0
  35. package/dist/adapters/sba.js +134 -0
  36. package/dist/adapters/sba.js.map +1 -0
  37. package/dist/adapters/treasury-tic.d.ts +30 -0
  38. package/dist/adapters/treasury-tic.js +149 -0
  39. package/dist/adapters/treasury-tic.js.map +1 -0
  40. package/dist/adapters/uscourts.d.ts +8 -0
  41. package/dist/adapters/uscourts.js +8 -0
  42. package/dist/adapters/uscourts.js.map +1 -0
  43. package/dist/analysis/cite.d.ts +9 -1
  44. package/dist/analysis/cite.js +9 -2
  45. package/dist/analysis/cite.js.map +1 -1
  46. package/dist/analysis/compute.d.ts +22 -10
  47. package/dist/analysis/compute.js +75 -16
  48. package/dist/analysis/compute.js.map +1 -1
  49. package/dist/analysis/expr.d.ts +2 -0
  50. package/dist/analysis/expr.js +17 -0
  51. package/dist/analysis/expr.js.map +1 -1
  52. package/dist/analysis/replicate.d.ts +15 -0
  53. package/dist/analysis/replicate.js +228 -0
  54. package/dist/analysis/replicate.js.map +1 -0
  55. package/dist/analysis/rerun.js +2 -1
  56. package/dist/analysis/rerun.js.map +1 -1
  57. package/dist/analysis/table.d.ts +14 -0
  58. package/dist/analysis/table.js +79 -28
  59. package/dist/analysis/table.js.map +1 -1
  60. package/dist/analysis/tools.js +16 -4
  61. package/dist/analysis/tools.js.map +1 -1
  62. package/dist/bin/cli.d.ts +2 -0
  63. package/dist/bin/cli.js +17 -0
  64. package/dist/bin/cli.js.map +1 -1
  65. package/dist/core/generic-rest.js +42 -17
  66. package/dist/core/generic-rest.js.map +1 -1
  67. package/dist/core/guard.d.ts +32 -0
  68. package/dist/core/guard.js +176 -0
  69. package/dist/core/guard.js.map +1 -0
  70. package/dist/core/http.d.ts +10 -0
  71. package/dist/core/http.js +36 -7
  72. package/dist/core/http.js.map +1 -1
  73. package/dist/core/keys.d.ts +13 -3
  74. package/dist/core/keys.js +67 -7
  75. package/dist/core/keys.js.map +1 -1
  76. package/dist/core/package.d.ts +12 -3
  77. package/dist/core/package.js +56 -12
  78. package/dist/core/package.js.map +1 -1
  79. package/dist/core/packs.d.ts +14 -5
  80. package/dist/core/packs.js.map +1 -1
  81. package/dist/{deep/_opm_parquet.d.ts → core/parquet.d.ts} +2 -1
  82. package/dist/{deep/_opm_parquet.js → core/parquet.js} +7 -5
  83. package/dist/core/parquet.js.map +1 -0
  84. package/dist/core/secret_patterns.d.ts +6 -0
  85. package/dist/core/secret_patterns.js +20 -0
  86. package/dist/core/secret_patterns.js.map +1 -0
  87. package/dist/core/tabular.d.ts +43 -1
  88. package/dist/core/tabular.js +149 -6
  89. package/dist/core/tabular.js.map +1 -1
  90. package/dist/deep/_opm_io.js +2 -2
  91. package/dist/deep/_opm_io.js.map +1 -1
  92. package/dist/deep/bea.js +1 -0
  93. package/dist/deep/bea.js.map +1 -1
  94. package/dist/deep/fred.js +40 -9
  95. package/dist/deep/fred.js.map +1 -1
  96. package/dist/deep/ipums.js +1 -0
  97. package/dist/deep/ipums.js.map +1 -1
  98. package/dist/deep/opm.js +1 -1
  99. package/dist/deep/opm.js.map +1 -1
  100. package/dist/envelope.js +3 -1
  101. package/dist/envelope.js.map +1 -1
  102. package/dist/index.d.ts +29 -22
  103. package/dist/index.js +45 -46
  104. package/dist/index.js.map +1 -1
  105. package/dist/remote/app.js +87 -31
  106. package/dist/remote/app.js.map +1 -1
  107. package/dist/remote/oauth.js +8 -1
  108. package/dist/remote/oauth.js.map +1 -1
  109. package/dist/remote/pages.d.ts +3 -7
  110. package/dist/remote/pages.js +49 -19
  111. package/dist/remote/pages.js.map +1 -1
  112. package/dist/remote/sign.d.ts +7 -1
  113. package/dist/remote/sign.js +33 -7
  114. package/dist/remote/sign.js.map +1 -1
  115. package/dist/remote/store.js +14 -5
  116. package/dist/remote/store.js.map +1 -1
  117. package/dist/report/audit_report.js +29 -7
  118. package/dist/report/audit_report.js.map +1 -1
  119. package/dist/report/build_report.js +34 -4
  120. package/dist/report/build_report.js.map +1 -1
  121. package/dist/report/method_audit.js +3 -0
  122. package/dist/report/method_audit.js.map +1 -1
  123. package/dist/report/scan_secrets.js +2 -13
  124. package/dist/report/scan_secrets.js.map +1 -1
  125. package/dist/report/tools.js +2 -1
  126. package/dist/report/tools.js.map +1 -1
  127. package/dist/server.d.ts +3 -1
  128. package/dist/server.js +53 -17
  129. package/dist/server.js.map +1 -1
  130. package/dist/tools.d.ts +3 -1
  131. package/dist/tools.js +82 -9
  132. package/dist/tools.js.map +1 -1
  133. package/guide/WORKFLOW.md +6 -5
  134. package/guide/assets/download_data.ps1 +2 -0
  135. package/guide/assets/download_data.sh +8 -2
  136. package/guide/references/report_patterns.md +5 -2
  137. package/guide/references/report_schema.md +140 -0
  138. package/guide/references/routing.md +30 -3
  139. package/package.json +1 -1
  140. package/packs/bjs/endpoints.json +150 -0
  141. package/packs/bjs/policy.json +114 -0
  142. package/packs/bjs/source.md +98 -0
  143. package/packs/cbp/endpoints.json +31 -0
  144. package/packs/cbp/policy.json +85 -0
  145. package/packs/cbp/source.md +57 -0
  146. package/packs/cdc/source.md +1 -1
  147. package/packs/cdc-wonder/endpoints.json +26 -0
  148. package/packs/cdc-wonder/policy.json +84 -0
  149. package/packs/cdc-wonder/source.md +53 -0
  150. package/packs/census/source.md +8 -4
  151. package/packs/college-scorecard/source.md +1 -1
  152. package/packs/cpsc/endpoints.json +91 -0
  153. package/packs/cpsc/policy.json +91 -0
  154. package/packs/cpsc/source.md +76 -0
  155. package/packs/dhs-ohss/source.md +1 -1
  156. package/packs/doj-news/source.md +1 -1
  157. package/packs/fbi/source.md +3 -3
  158. package/packs/fhwa/endpoints.json +48 -0
  159. package/packs/fhwa/policy.json +91 -0
  160. package/packs/fhwa/source.md +59 -0
  161. package/packs/foreign-assistance/endpoints.json +166 -0
  162. package/packs/foreign-assistance/policy.json +80 -0
  163. package/packs/foreign-assistance/source.md +76 -0
  164. package/packs/fred/source.md +2 -1
  165. package/packs/fsa/source.md +1 -1
  166. package/packs/hud/source.md +1 -1
  167. package/packs/hud-ahar/endpoints.json +92 -0
  168. package/packs/hud-ahar/policy.json +88 -0
  169. package/packs/hud-ahar/source.md +72 -0
  170. package/packs/ipums/docs/report_patterns.md +1 -1
  171. package/packs/ipums/docs/variables.md +4 -3
  172. package/packs/ipums/source.md +10 -8
  173. package/packs/naep/source.md +1 -1
  174. package/packs/nces/endpoints.json +101 -0
  175. package/packs/nces/policy.json +99 -0
  176. package/packs/nces/source.md +75 -0
  177. package/packs/nhtsa/source.md +1 -1
  178. package/packs/nsf/endpoints.json +33 -0
  179. package/packs/nsf/policy.json +83 -0
  180. package/packs/nsf/source.md +59 -0
  181. package/packs/onrr/endpoints.json +198 -0
  182. package/packs/onrr/policy.json +91 -0
  183. package/packs/onrr/source.md +55 -0
  184. package/packs/opm/docs/opm_api.md +2 -1
  185. package/packs/opm/source.md +2 -1
  186. package/packs/port.json +65 -0
  187. package/packs/sba/endpoints.json +34 -0
  188. package/packs/sba/policy.json +86 -0
  189. package/packs/sba/source.md +60 -0
  190. package/packs/treasury/source.md +1 -1
  191. package/packs/treasury-tic/endpoints.json +77 -0
  192. package/packs/treasury-tic/policy.json +89 -0
  193. package/packs/treasury-tic/source.md +80 -0
  194. package/packs/uscourts/endpoints.json +22 -0
  195. package/packs/uscourts/policy.json +87 -0
  196. package/packs/uscourts/source.md +56 -0
  197. package/packs/uspto/source.md +1 -1
  198. package/dist/deep/_opm_parquet.js.map +0 -1
@@ -109,11 +109,13 @@ weights, terms).
109
109
  - **Extract gotchas**: IPUMS adds detailed versions automatically (EDUC brings EDUCD); a data-quality
110
110
  flag or case-selection code that a sample lacks returns HTTP 400 (fix the spec and fetch again);
111
111
  ACS 2020 1-year has experimental weights.
112
- - **Standard errors**: this server has no replicate-weight or design-based standard errors yet. Report
113
- which weight each estimate uses and that standard errors were not computed; without them, never call a
114
- difference "statistically different" (when standard errors exist, the rule is beyond 1.645 standard errors,
115
- 90 percent; otherwise "about the same"). The methods, for reference: ACS person estimates use REPWTP (80
116
- replicates), household estimates (tenure, household income) REPWT with HHWT, Census's own method; a
112
+ - **Standard errors**: request the replicate weights (`REPWTP`, `REPWT`) and use `compute`'s `replicate_se`
113
+ step: `{"replicate_se": {"estimates": {...}, "weight": "PERWT", "replicates": "REPWTP1:REPWTP80", "method": "sdr"}}`
114
+ (4/80, Census's method; ASEC `ASECWT` with `REPWTP1:REPWTP160`). Design-based (STRATA/CLUSTER) standard errors
115
+ are not computed here. Without standard errors, report which weight each estimate uses and that standard errors
116
+ were not computed, and never call a difference "statistically different" (with them, the rule is beyond 1.645
117
+ standard errors of the difference, 90 percent; otherwise "about the same"). The methods: ACS person estimates
118
+ use REPWTP (80 replicates), household estimates (tenure, household income) REPWT with HHWT, Census's own method; a
117
119
  linearized SE with STRATA and CLUSTER (added automatically) can understate a PUMS margin (half of the
118
120
  replicate one in a 2024 tenure check), so replicates are preferred. CPS ASEC: REPWTP (160) for person
119
121
  estimates, or REPWT (160) with ASECWTH on householders (`RELATE == 101`) for household estimates such as
@@ -132,9 +134,9 @@ weights, terms).
132
134
  every threshold or definition for sensitivity; benchmark a headline number against a published figure (Census, BLS).
133
135
  - Public IPUMS data cannot show legal status, benefits, or full administrative controls: say so.
134
136
 
135
- Analysis: build tables with `compute` on `data/ipums/ipums_extract.csv`, decoding values with `codebook.csv`. This
136
- server has no IPUMS recode helpers or replicate-weight standard errors yet: say which weight you used, and that
137
- standard errors were not computed.
137
+ Analysis: build tables with `compute` on `data/ipums/ipums_extract.csv`, decoding values with `codebook.csv`, and
138
+ standard errors with its `replicate_se` step (replicate weights in the extract). This server has no IPUMS recode
139
+ helpers.
138
140
  Typical questions, conventions (populations, definitions, benchmarks) and chart forms:
139
141
  `docs/report_patterns.md`.
140
142
 
@@ -8,7 +8,7 @@ student group, with standard errors and NAEP's own significance tests.
8
8
  science, writing, civics, U.S. history, geography, economics, technology and engineering literacy,
9
9
  music. States versus the nation, change since 2019, gaps by race, sex, economic disadvantage.
10
10
  - Not here: state test results (each state's own exams, not comparable across states), graduation rates
11
- and school finance (nces or ed packs if present), college outcomes (college-scorecard).
11
+ and school finance (nces), college outcomes (college-scorecard).
12
12
  - NAEP is a sample survey: every number is an estimate. It reports groups, never students or schools.
13
13
 
14
14
  ## Endpoints and datasets
@@ -0,0 +1,101 @@
1
+ {
2
+ "base_url": "https://nces.ed.gov",
3
+ "timeout": 180,
4
+ "endpoints": {
5
+ "digest_table": {
6
+ "path": "/programs/digest/d{edition}/tables/xls/tabn{table}.xlsx",
7
+ "method": "GET",
8
+ "description": "One table of the Digest of Education Statistics as its Excel file: enrollment, teachers, finances, degrees, outcomes, from pre-K to postsecondary. Default: every line numbered (row, col1...) to find the header lines; then header_lines (usually '3-4') names the columns.",
9
+ "params": {
10
+ "edition": {
11
+ "in": "path",
12
+ "required": true,
13
+ "desc": "two-digit Digest edition, e.g. 25 (Digest 2025); not every table is in every edition"
14
+ },
15
+ "table": {
16
+ "in": "path",
17
+ "required": true,
18
+ "desc": "table number, e.g. 203.10 (public school enrollment by grade)"
19
+ },
20
+ "header_lines": {
21
+ "in": "records",
22
+ "desc": "the header line numbers, e.g. '3-4'"
23
+ },
24
+ "header_row": {
25
+ "in": "records",
26
+ "desc": "one header line instead; 0 (default) returns every line numbered"
27
+ },
28
+ "sheet": {
29
+ "in": "records",
30
+ "desc": "sheet name or number (default 1)"
31
+ },
32
+ "carry": {
33
+ "in": "records",
34
+ "desc": "which header lines span to the right over blank cells: top (default), all, none; check the names against the numbered lines"
35
+ }
36
+ },
37
+ "defaults": {
38
+ "header_row": 0
39
+ },
40
+ "records": "$xlsx",
41
+ "paging": {
42
+ "type": "none"
43
+ },
44
+ "max_rows": 20000
45
+ },
46
+ "ipeds": {
47
+ "path": "/ipeds/datacenter/data/{file}.zip",
48
+ "method": "GET",
49
+ "description": "An IPEDS complete data file (one row per institution, or institution by category), e.g. HD2023 directory, EFFY2023 12-month enrollment, C2023_A completions by CIP code, ADM2023 admissions, SFA2223 student aid, F2223_F2 finance. The revised (_rv) version is read when the zip has one.",
50
+ "params": {
51
+ "file": {
52
+ "in": "path",
53
+ "required": true,
54
+ "desc": "IPEDS file name as on the Data Center's Complete Data Files page, e.g. HD2023, EFFY2023, C2023_A"
55
+ },
56
+ "unitid": {
57
+ "in": "spec",
58
+ "desc": "keep only these institutions (UNITID, a list works)"
59
+ },
60
+ "state": {
61
+ "in": "spec",
62
+ "desc": "keep only these states (two-letter STABBR; files that have that column, e.g. HD)"
63
+ },
64
+ "member": {
65
+ "in": "records",
66
+ "desc": "the CSV in the zip, e.g. c2023_a.csv for the original instead of the revised file"
67
+ }
68
+ },
69
+ "records": "$zip:*.csv",
70
+ "prefer": "_rv[.]csv$",
71
+ "paging": {
72
+ "type": "none"
73
+ },
74
+ "max_rows": 1000000
75
+ },
76
+ "ipeds_dictionary": {
77
+ "path": "/ipeds/datacenter/data/{file}_Dict.zip",
78
+ "method": "GET",
79
+ "description": "The data dictionary of an IPEDS file: variable names and titles (sheet Varlist), code values and labels (Frequencies), long descriptions (Description).",
80
+ "params": {
81
+ "file": {
82
+ "in": "path",
83
+ "required": true,
84
+ "desc": "the IPEDS file name, e.g. HD2023"
85
+ },
86
+ "sheet": {
87
+ "in": "records",
88
+ "desc": "Varlist (default), Frequencies, Description, Introduction"
89
+ }
90
+ },
91
+ "defaults": {
92
+ "sheet": "Varlist"
93
+ },
94
+ "records": "$zip:*.xlsx",
95
+ "paging": {
96
+ "type": "none"
97
+ },
98
+ "max_rows": 100000
99
+ }
100
+ }
101
+ }
@@ -0,0 +1,99 @@
1
+ {
2
+ "id": "nces",
3
+ "title": "National Center for Education Statistics: Digest of Education Statistics tables and IPEDS data files",
4
+ "agency": "U.S. Department of Education, National Center for Education Statistics",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "education",
8
+ "higher education"
9
+ ],
10
+ "keywords": [
11
+ "nces",
12
+ "national center for education statistics",
13
+ "digest of education statistics",
14
+ "school enrollment",
15
+ "public school enrollment",
16
+ "enrollment by grade",
17
+ "teachers",
18
+ "pupil teacher ratio",
19
+ "per-pupil expenditure",
20
+ "school finance",
21
+ "high school graduates",
22
+ "graduation rate",
23
+ "degrees conferred",
24
+ "ipeds",
25
+ "college enrollment",
26
+ "fall enrollment",
27
+ "completions",
28
+ "degrees by field",
29
+ "cip code",
30
+ "institutional characteristics",
31
+ "tuition and fees",
32
+ "admissions",
33
+ "student financial aid",
34
+ "college finance"
35
+ ],
36
+ "questions": [
37
+ "public school enrollment by grade over time",
38
+ "number of teachers and pupil/teacher ratio",
39
+ "current expenditure per pupil by state",
40
+ "bachelor's degrees conferred by field",
41
+ "enrollment at a college or all colleges in a state (IPEDS)",
42
+ "completions by CIP code and award level",
43
+ "admissions and yield at a university"
44
+ ],
45
+ "homepage": "https://nces.ed.gov/",
46
+ "terms_url": "https://nces.ed.gov/ipeds/datacenter/DataFiles.aspx",
47
+ "auth": {
48
+ "env": null,
49
+ "required": "none",
50
+ "placement": "none",
51
+ "signup_url": null,
52
+ "echoes_key": false
53
+ },
54
+ "limits": {
55
+ "rps": 2,
56
+ "burst": 4,
57
+ "max_rows": 1000000
58
+ },
59
+ "routes": {
60
+ "api": {
61
+ "base": "https://nces.ed.gov",
62
+ "probe": "https://nces.ed.gov/ipeds/datacenter/data/HD2023_Dict.zip",
63
+ "hosts": [
64
+ "nces.ed.gov"
65
+ ]
66
+ },
67
+ "manual": {
68
+ "how": "Digest: open the table page (https://nces.ed.gov/programs/digest/current_tables.asp, then the table) and click its Excel link, or https://nces.ed.gov/programs/digest/d<yy>/tables/xls/tabn<table>.xlsx. IPEDS: download https://nces.ed.gov/ipeds/datacenter/data/<FILE>.zip from the Complete Data Files page (https://nces.ed.gov/ipeds/datacenter/DataFiles.aspx). Upload the file as saved with ingest."
69
+ }
70
+ },
71
+ "cache": {
72
+ "mode": "run_only",
73
+ "ttl_hours": null,
74
+ "why": "Files are replaced when NCES revises them; data/ is reused within a run."
75
+ },
76
+ "redistribution": {
77
+ "mode": "allowed",
78
+ "why": "NCES publications and IPEDS data are U.S. government works in the public domain (17 U.S.C. 105)."
79
+ },
80
+ "notices": [],
81
+ "retention": null,
82
+ "citation": "U.S. Department of Education, National Center for Education Statistics, {dataset}, nces.ed.gov, retrieved {retrieved_long}.",
83
+ "audit_hooks": [],
84
+ "clarify": [
85
+ "Published table or institution file? Digest tables give national (and often state) totals as NCES published them; IPEDS files give one row per institution to filter and sum. Default: a Digest table for national or state totals, IPEDS for named colleges.",
86
+ "Which year? Digest tables are labelled by school year or fall; IPEDS files by collection year (EFFY2023 = 12-month enrollment for 2022-23). Say which, and whether the IPEDS file is provisional or revised (_rv)."
87
+ ],
88
+ "formats": [
89
+ "xlsx",
90
+ "zip",
91
+ "csv"
92
+ ],
93
+ "aliases": [
94
+ "NCES",
95
+ "IPEDS",
96
+ "Digest of Education Statistics",
97
+ "Integrated Postsecondary Education Data System"
98
+ ]
99
+ }
@@ -0,0 +1,75 @@
1
+ # National Center for Education Statistics: Digest of Education Statistics tables and IPEDS data files
2
+ Depth: standard. Key: none.
3
+ - The Digest of Education Statistics: NCES's compendium of published tables (enrollment, teachers, finances,
4
+ graduates, degrees, outcomes, pre-K to postsecondary), as each table's Excel file.
5
+ - IPEDS complete data files: one row per postsecondary institution, or institution by category, for every
6
+ survey component (directory, enrollment, completions, admissions, student aid, finance, staff), with their
7
+ data dictionaries.
8
+
9
+ ## When to use
10
+ - National or state totals as NCES published them: public school enrollment by grade, teachers and
11
+ pupil/teacher ratios, current expenditure per pupil, high school completers, degrees conferred by field and
12
+ level, enrollment trends.
13
+ - A named college or every college in a state: fall or 12-month enrollment, completions by CIP code and award
14
+ level, admissions and test scores, tuition, aid, finance (IPEDS).
15
+ - Not here: earnings and debt of graduates by program and institution (college-scorecard); federal student
16
+ loans and Pell totals (fsa); test scores of K-12 students (naep); school-age population, poverty or
17
+ attainment from surveys (census ACS).
18
+
19
+ ## Endpoints
20
+ | Endpoint | Returns | Parameters |
21
+ |---|---|---|
22
+ | `digest_table` | one Digest table: without `header_lines`, every line numbered (`row`, `col1`...); with it, named columns | `edition` (two digits, e.g. `25`), `table` (e.g. `203.10`), `header_lines` (usually `3-4`), `header_row`, `sheet` |
23
+ | `ipeds` | an IPEDS complete data file (the revised `_rv` CSV when the zip has one) | `file` (e.g. `HD2023`, `EFFY2023`, `C2023_A`), `unitid`, `state` (filters), `member` |
24
+ | `ipeds_dictionary` | that file's dictionary sheet | `file`, `sheet` (`Varlist` default, `Frequencies` for code labels, `Description`) |
25
+
26
+ ## Spec examples
27
+ ```json
28
+ {"description": "Public school enrollment by grade, fall 1980-2024 (Digest 2025 table 203.10)",
29
+ "requests": [{"name": "t203_10", "endpoint": "digest_table", "params": {"edition": "25", "table": "203.10", "header_lines": "3-4"}}]}
30
+ ```
31
+ ```json
32
+ {"description": "Colleges in Vermont (IPEDS directory 2023-24) and the code labels",
33
+ "requests": [{"name": "vt_colleges", "endpoint": "ipeds", "params": {"file": "HD2023", "state": "VT"}},
34
+ {"name": "hd_codes", "endpoint": "ipeds_dictionary", "params": {"file": "HD2023", "sheet": "Frequencies"}}]}
35
+ ```
36
+
37
+ ## Data format and traps
38
+ - Digest tables:
39
+ - Line 1 is the title, line 2 the units ("[In thousands]": multiply), lines 3-4 the stacked header, and the
40
+ first data line is the row of column numbers ("1", "2", ...): drop it. Notes and the source follow the last
41
+ data line. Verified on table 203.10, Digest 2025; other tables differ, so read the raw lines first.
42
+ - Columns named `<header>_colN` hold footnote marks for the value to their left, or nothing. In table
43
+ 203.10's 2024 row, `\3,4\` points to footnotes 3 and 4 below the table. Keep the marks with the value they
44
+ qualify.
45
+ - Values are text with thousands separators. Footnote marks also sit inside header cells (`Ungraded\1\`), and
46
+ some cells start with a line break ("\n1993"): trim them. Symbols such as `†` and `—`, used in many
47
+ tables, are defined in each table's notes; keep them and never treat them as 0. Projections are marked in
48
+ the notes.
49
+ - Each edition renumbers or drops tables, and a table that is not in that edition answers 404. Use the
50
+ current edition's table list (https://nces.ed.gov/programs/digest/current_tables.asp) to find the number.
51
+ - IPEDS:
52
+ - Names. Files are named by collection year: HD2023 is the 2023-24 directory, EFFY2023 the 12-month enrollment
53
+ of 2022-23, C2023_A the completions of July 2022-June 2023. Read the dictionary's Introduction sheet to be
54
+ sure.
55
+ - Revisions. When a zip has a revised file (`_rv`, the final release), it is read instead of the provisional
56
+ one; `member` picks the other.
57
+ - Codes. Columns are coded (`SECTOR`, `CONTROL`, `ICLEVEL`; `X`-prefixed imputation flags): label them from
58
+ the dictionary's Frequencies sheet. Negative codes such as -1 and -2 mean not reported or not applicable;
59
+ never sum them.
60
+ - Joining files. UNITID joins files; a parent/child system (`F1SYSTYP` ...) can report for several campuses.
61
+ - Size. Completions files are large (C2023_A has two 60 MB CSVs): filter by `unitid` or `state`. `state`
62
+ works only on files with `STABBR`, so join HD for state.
63
+ - Counting. Enrollment and completions rows are by category (all students, by race, by sex, by CIP and award
64
+ level), and totals are rows of their own (CIPCODE 99, MAJORNUM 1): pick the total row; never sum across
65
+ categories.
66
+
67
+ ## Access
68
+ No key. `fetch` reads nces.ed.gov directly (verified 2026-11-07). Manual: the Digest table's Excel link, or the
69
+ IPEDS Complete Data Files page (https://nces.ed.gov/ipeds/datacenter/DataFiles.aspx); upload the file as saved;
70
+ then `ingest`.
71
+
72
+ ## Citation and terms
73
+ "U.S. Department of Education, National Center for Education Statistics, Digest of Education Statistics 2025,
74
+ table 203.10" or "... Integrated Postsecondary Education Data System (IPEDS), <file> (<release>), nces.ed.gov,
75
+ retrieved <date>." Public domain (17 U.S.C. 105).
@@ -6,7 +6,7 @@ Investigation (ODI), NCAP 5-Star crash ratings, VIN decoding (vPIC), and child c
6
6
  - Recalls of a make/model/model year or a campaign; complaints about a model (crashes, fires, injuries,
7
7
  components); crash-test star ratings and rollover risk; decoding a VIN; models a make built in a year.
8
8
  - Not here: crash fatality statistics (FARS/CRSS: not covered by this pack; say so), traffic volumes and
9
- transport statistics (bts or dot packs if present), product recalls outside vehicles (cpsc, fda).
9
+ transport statistics (bts, fhwa), product recalls outside vehicles (cpsc, fda).
10
10
  - Records are text and counts of reports. They are not defect rates: there is no fleet size or sales data.
11
11
 
12
12
  ## Endpoints and datasets
@@ -0,0 +1,33 @@
1
+ {
2
+ "base_url": "https://api.nsf.gov/services/v1",
3
+ "timeout": 60,
4
+ "endpoints": {
5
+ "awards": {
6
+ "path": "/awards.json",
7
+ "method": "GET",
8
+ "description": "NSF awards (grants and cooperative agreements) from the NSF Award Search API: title, awardee institution and location, PI, amounts obligated and estimated, dates, NSF directorate, division and program, abstract. Filter by keyword, institution, state, PI, program or date range.",
9
+ "params": {
10
+ "keyword": {"desc": "words in the title or abstract; quote a phrase, e.g. \"machine learning\""},
11
+ "awardeeName": {"desc": "institution name or part of it"},
12
+ "awardeeStateCode": {"desc": "two-letter state of the awardee institution"},
13
+ "perfStateCode": {"desc": "two-letter state where the work is performed"},
14
+ "pdPIName": {"desc": "PI name (last name, or first and last)"},
15
+ "fundProgramName": {"desc": "NSF funding program name"},
16
+ "id": {"desc": "award number(s), comma-separated"},
17
+ "dateStart": {"desc": "award date from, MM/DD/YYYY"},
18
+ "dateEnd": {"desc": "award date to, MM/DD/YYYY"},
19
+ "startDateStart": {"desc": "award start date from, MM/DD/YYYY"},
20
+ "startDateEnd": {"desc": "award start date to, MM/DD/YYYY"},
21
+ "expDateStart": {"desc": "expiration date from, MM/DD/YYYY"},
22
+ "expDateEnd": {"desc": "expiration date to, MM/DD/YYYY"}
23
+ },
24
+ "defaults": {"offset": 1},
25
+ "records": "response.award",
26
+ "total": "response.metadata.totalCount",
27
+ "paging": {"type": "offset", "limit_param": "rpp", "offset_param": "offset", "size": 25, "manual_size": 25},
28
+ "max_rows": 10000,
29
+ "unique_key": "id",
30
+ "flatten": true
31
+ }
32
+ }
33
+ }
@@ -0,0 +1,83 @@
1
+ {
2
+ "id": "nsf",
3
+ "title": "National Science Foundation awards (NSF Award Search API)",
4
+ "agency": "U.S. National Science Foundation",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "science",
8
+ "research funding",
9
+ "federal grants"
10
+ ],
11
+ "keywords": [
12
+ "nsf",
13
+ "national science foundation",
14
+ "nsf grants",
15
+ "nsf awards",
16
+ "research grants",
17
+ "science funding",
18
+ "principal investigator",
19
+ "epscor",
20
+ "career award",
21
+ "graduate research fellowship",
22
+ "directorate",
23
+ "research funding by university"
24
+ ],
25
+ "questions": [
26
+ "NSF awards to a university in a year",
27
+ "NSF grants on a research topic (e.g. quantum computing)",
28
+ "NSF funding by state",
29
+ "awards from an NSF program",
30
+ "a researcher's NSF awards"
31
+ ],
32
+ "homepage": "https://www.nsf.gov/awardsearch/",
33
+ "terms_url": "https://resources.research.gov/common/webapi/awardapisearch-v1.htm",
34
+ "auth": {
35
+ "env": null,
36
+ "required": "none",
37
+ "placement": "none",
38
+ "signup_url": null,
39
+ "echoes_key": false
40
+ },
41
+ "limits": {
42
+ "rps": 2,
43
+ "burst": 4,
44
+ "max_rows": 10000
45
+ },
46
+ "routes": {
47
+ "api": {
48
+ "base": "https://api.nsf.gov/services/v1",
49
+ "probe": "https://api.nsf.gov/services/v1/awards.json?id=2600000",
50
+ "hosts": [
51
+ "api.nsf.gov"
52
+ ]
53
+ },
54
+ "manual": {
55
+ "how": "Search at https://www.nsf.gov/awardsearch/ (Advanced Search), export the results as CSV, and upload the file; or open the API links in INSTRUCTIONS.md (25 awards per page) and save the JSON."
56
+ }
57
+ },
58
+ "cache": {
59
+ "mode": "run_only",
60
+ "ttl_hours": null,
61
+ "why": "Awards are amended daily; data/ is reused within a run."
62
+ },
63
+ "redistribution": {
64
+ "mode": "allowed",
65
+ "why": "NSF award data are U.S. government works in the public domain (17 U.S.C. 105). PI and program officer e-mail and phone are dropped."
66
+ },
67
+ "notices": [],
68
+ "retention": null,
69
+ "citation": "U.S. National Science Foundation, NSF Award Search ({dataset}), api.nsf.gov, retrieved {retrieved_long}.",
70
+ "audit_hooks": [],
71
+ "clarify": [
72
+ "Which dollars? fundsObligatedAmt is what NSF has obligated so far; estimatedTotalAmt is the expected total over the award's life. Default: obligated, said as such.",
73
+ "Which dates? dateStart/dateEnd filter the award date (when NSF made the award); startDate and expDate are the award's period. Default: awards made in the period asked."
74
+ ],
75
+ "formats": [
76
+ "json"
77
+ ],
78
+ "aliases": [
79
+ "NSF",
80
+ "National Science Foundation",
81
+ "NSF Award Search"
82
+ ]
83
+ }
@@ -0,0 +1,59 @@
1
+ # National Science Foundation awards (NSF Award Search API)
2
+ Depth: standard. Key: none. NSF's awards (grants, cooperative agreements, fellowships to institutions) from the
3
+ Award Search API, one row per award:
4
+ - title and abstract;
5
+ - awardee institution, location and congressional district, and the place of performance;
6
+ - PI and co-PIs;
7
+ - NSF directorate, division and program;
8
+ - amounts obligated and estimated; award, start and expiration dates.
9
+
10
+ ## When to use
11
+ - NSF funding to a university, a state or a program; NSF awards on a research topic; a researcher's NSF
12
+ awards; new awards in a period.
13
+ - Not here: NIH grants (nih); all federal grants by agency and recipient, including NSF in USAspending's
14
+ accounting (usaspending); national R&D spending statistics (NSF NCSES surveys such as HERD and Federal Funds for
15
+ R&D are not covered here); patents (uspto).
16
+
17
+ ## Endpoints
18
+ | Endpoint | Returns | Parameters (at least one filter) |
19
+ |---|---|---|
20
+ | `awards` | one row per award: `id`, `title`, `awardeeName`, `awardeeCity`, `awardeeStateCode`, `awardeeDistrictCode`, `perfStateCode`, `pdPIName`, `piFirstName`, `piLastName`, `coPDPI`, `dirAbbr`, `divAbbr`, `orgLongName`, `fundProgramName`, `fundsObligatedAmt`, `estimatedTotalAmt`, `date`, `startDate`, `expDate`, `transType`, `cfdaNumber`, `abstractText` ... | `keyword`, `awardeeName`, `awardeeStateCode`, `perfStateCode`, `pdPIName`, `fundProgramName`, `id`, and date ranges `dateStart`/`dateEnd` (award date), `startDateStart`/`startDateEnd`, `expDateStart`/`expDateEnd` (MM/DD/YYYY) |
21
+
22
+ ## Spec examples
23
+ ```json
24
+ {"description": "NSF awards mentioning quantum, made in January 2026",
25
+ "requests": [{"name": "quantum_jan26", "endpoint": "awards",
26
+ "params": {"keyword": "quantum", "dateStart": "01/01/2026", "dateEnd": "01/31/2026"}}]}
27
+ ```
28
+ ```json
29
+ {"description": "NSF awards to Vermont institutions, FY2026",
30
+ "requests": [{"name": "vt_fy26", "endpoint": "awards",
31
+ "params": {"awardeeStateCode": "VT", "dateStart": "10/01/2025", "dateEnd": "09/30/2026"}}]}
32
+ ```
33
+
34
+ ## Data format and traps
35
+ - Dollars:
36
+ - `fundsObligatedAmt` is the money obligated to date (it grows with continuing increments).
37
+ - `estimatedTotalAmt` is the expected total over the award's life.
38
+ - Name which one you use. Summing `estimatedTotalAmt` over awards made in a year gives commitments, not
39
+ spending.
40
+ - `fundsObligated` lists the obligations by fiscal year as text.
41
+ - Dates are MM/DD/YYYY, both in filters and in the rows. `dateStart`/`dateEnd` filter the award date (`date`),
42
+ not the period of performance.
43
+ - Collaborative research: each collaborating institution has its own award with the same title. Count awards, or
44
+ group by title, and say which.
45
+ - `totalCount` can exceed the awards returned: 12 counted and 11 returned for keyword "quantum", January 2026, on
46
+ 2026-11-07. Report the rows you have.
47
+ - Keyword search matches title and abstract text, including unrelated uses of the word: read the titles before
48
+ counting.
49
+ - Paging is 25 awards a page (1-based offset); requests stop at 10,000 rows. Narrow by date or state.
50
+ - People's contact fields (PI and program officer e-mail and phone) are dropped; names stay.
51
+
52
+ ## Access
53
+ No key. `fetch` uses api.nsf.gov (verified 2026-11-07). Manual: https://www.nsf.gov/awardsearch/ (Advanced Search,
54
+ export CSV), or the API links in INSTRUCTIONS.md; then `ingest`. API documentation:
55
+ https://resources.research.gov/common/webapi/awardapisearch-v1.htm.
56
+
57
+ ## Citation and terms
58
+ "U.S. National Science Foundation, NSF Award Search, <query>, api.nsf.gov, retrieved <date>." Public domain (17
59
+ U.S.C. 105).