data-analyst-mcp 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/README.md +28 -4
  2. package/dist/adapters/bjs.d.ts +13 -0
  3. package/dist/adapters/bjs.js +28 -0
  4. package/dist/adapters/bjs.js.map +1 -0
  5. package/dist/adapters/cbp.d.ts +5 -0
  6. package/dist/adapters/cbp.js +64 -0
  7. package/dist/adapters/cbp.js.map +1 -0
  8. package/dist/adapters/cdc-wonder.d.ts +5 -0
  9. package/dist/adapters/cdc-wonder.js +68 -0
  10. package/dist/adapters/cdc-wonder.js.map +1 -0
  11. package/dist/adapters/cpsc.d.ts +4 -0
  12. package/dist/adapters/cpsc.js +125 -0
  13. package/dist/adapters/cpsc.js.map +1 -0
  14. package/dist/adapters/fhwa.d.ts +9 -0
  15. package/dist/adapters/fhwa.js +16 -0
  16. package/dist/adapters/fhwa.js.map +1 -0
  17. package/dist/adapters/foreign-assistance.d.ts +9 -0
  18. package/dist/adapters/foreign-assistance.js +42 -0
  19. package/dist/adapters/foreign-assistance.js.map +1 -0
  20. package/dist/adapters/hud-ahar.d.ts +3 -0
  21. package/dist/adapters/hud-ahar.js +80 -0
  22. package/dist/adapters/hud-ahar.js.map +1 -0
  23. package/dist/adapters/nces.d.ts +10 -0
  24. package/dist/adapters/nces.js +32 -0
  25. package/dist/adapters/nces.js.map +1 -0
  26. package/dist/adapters/nsf.d.ts +11 -0
  27. package/dist/adapters/nsf.js +25 -0
  28. package/dist/adapters/nsf.js.map +1 -0
  29. package/dist/adapters/onrr.d.ts +9 -0
  30. package/dist/adapters/onrr.js +35 -0
  31. package/dist/adapters/onrr.js.map +1 -0
  32. package/dist/adapters/sba.d.ts +10 -0
  33. package/dist/adapters/sba.js +134 -0
  34. package/dist/adapters/sba.js.map +1 -0
  35. package/dist/adapters/treasury-tic.d.ts +30 -0
  36. package/dist/adapters/treasury-tic.js +149 -0
  37. package/dist/adapters/treasury-tic.js.map +1 -0
  38. package/dist/adapters/uscourts.d.ts +8 -0
  39. package/dist/adapters/uscourts.js +8 -0
  40. package/dist/adapters/uscourts.js.map +1 -0
  41. package/dist/analysis/cite.d.ts +9 -1
  42. package/dist/analysis/cite.js +9 -2
  43. package/dist/analysis/cite.js.map +1 -1
  44. package/dist/bin/cli.d.ts +2 -0
  45. package/dist/bin/cli.js +17 -0
  46. package/dist/bin/cli.js.map +1 -1
  47. package/dist/core/generic-rest.js +42 -17
  48. package/dist/core/generic-rest.js.map +1 -1
  49. package/dist/core/guard.d.ts +32 -0
  50. package/dist/core/guard.js +176 -0
  51. package/dist/core/guard.js.map +1 -0
  52. package/dist/core/http.d.ts +10 -0
  53. package/dist/core/http.js +36 -7
  54. package/dist/core/http.js.map +1 -1
  55. package/dist/core/keys.d.ts +13 -3
  56. package/dist/core/keys.js +67 -7
  57. package/dist/core/keys.js.map +1 -1
  58. package/dist/core/package.d.ts +12 -3
  59. package/dist/core/package.js +56 -12
  60. package/dist/core/package.js.map +1 -1
  61. package/dist/core/secret_patterns.d.ts +6 -0
  62. package/dist/core/secret_patterns.js +20 -0
  63. package/dist/core/secret_patterns.js.map +1 -0
  64. package/dist/core/tabular.d.ts +43 -1
  65. package/dist/core/tabular.js +149 -6
  66. package/dist/core/tabular.js.map +1 -1
  67. package/dist/envelope.js +3 -1
  68. package/dist/envelope.js.map +1 -1
  69. package/dist/index.d.ts +29 -22
  70. package/dist/index.js +45 -46
  71. package/dist/index.js.map +1 -1
  72. package/dist/remote/app.js +87 -31
  73. package/dist/remote/app.js.map +1 -1
  74. package/dist/remote/oauth.js +8 -1
  75. package/dist/remote/oauth.js.map +1 -1
  76. package/dist/remote/pages.d.ts +3 -7
  77. package/dist/remote/pages.js +49 -19
  78. package/dist/remote/pages.js.map +1 -1
  79. package/dist/remote/sign.d.ts +7 -1
  80. package/dist/remote/sign.js +33 -7
  81. package/dist/remote/sign.js.map +1 -1
  82. package/dist/remote/store.js +14 -5
  83. package/dist/remote/store.js.map +1 -1
  84. package/dist/report/audit_report.js +29 -7
  85. package/dist/report/audit_report.js.map +1 -1
  86. package/dist/report/build_report.js +34 -4
  87. package/dist/report/build_report.js.map +1 -1
  88. package/dist/report/method_audit.js +3 -0
  89. package/dist/report/method_audit.js.map +1 -1
  90. package/dist/report/scan_secrets.js +2 -13
  91. package/dist/report/scan_secrets.js.map +1 -1
  92. package/dist/server.d.ts +3 -1
  93. package/dist/server.js +44 -12
  94. package/dist/server.js.map +1 -1
  95. package/dist/tools.js +27 -6
  96. package/dist/tools.js.map +1 -1
  97. package/guide/assets/download_data.ps1 +2 -0
  98. package/guide/assets/download_data.sh +8 -2
  99. package/guide/references/routing.md +30 -3
  100. package/package.json +1 -1
  101. package/packs/bjs/endpoints.json +150 -0
  102. package/packs/bjs/policy.json +114 -0
  103. package/packs/bjs/source.md +98 -0
  104. package/packs/cbp/endpoints.json +31 -0
  105. package/packs/cbp/policy.json +85 -0
  106. package/packs/cbp/source.md +57 -0
  107. package/packs/cdc/source.md +1 -1
  108. package/packs/cdc-wonder/endpoints.json +26 -0
  109. package/packs/cdc-wonder/policy.json +84 -0
  110. package/packs/cdc-wonder/source.md +53 -0
  111. package/packs/college-scorecard/source.md +1 -1
  112. package/packs/cpsc/endpoints.json +91 -0
  113. package/packs/cpsc/policy.json +91 -0
  114. package/packs/cpsc/source.md +76 -0
  115. package/packs/dhs-ohss/source.md +1 -1
  116. package/packs/doj-news/source.md +1 -1
  117. package/packs/fbi/source.md +3 -3
  118. package/packs/fhwa/endpoints.json +48 -0
  119. package/packs/fhwa/policy.json +91 -0
  120. package/packs/fhwa/source.md +59 -0
  121. package/packs/foreign-assistance/endpoints.json +166 -0
  122. package/packs/foreign-assistance/policy.json +80 -0
  123. package/packs/foreign-assistance/source.md +76 -0
  124. package/packs/fsa/source.md +1 -1
  125. package/packs/hud/source.md +1 -1
  126. package/packs/hud-ahar/endpoints.json +92 -0
  127. package/packs/hud-ahar/policy.json +88 -0
  128. package/packs/hud-ahar/source.md +72 -0
  129. package/packs/naep/source.md +1 -1
  130. package/packs/nces/endpoints.json +101 -0
  131. package/packs/nces/policy.json +99 -0
  132. package/packs/nces/source.md +75 -0
  133. package/packs/nhtsa/source.md +1 -1
  134. package/packs/nsf/endpoints.json +33 -0
  135. package/packs/nsf/policy.json +83 -0
  136. package/packs/nsf/source.md +59 -0
  137. package/packs/onrr/endpoints.json +198 -0
  138. package/packs/onrr/policy.json +91 -0
  139. package/packs/onrr/source.md +55 -0
  140. package/packs/port.json +65 -0
  141. package/packs/sba/endpoints.json +34 -0
  142. package/packs/sba/policy.json +86 -0
  143. package/packs/sba/source.md +60 -0
  144. package/packs/treasury/source.md +1 -1
  145. package/packs/treasury-tic/endpoints.json +77 -0
  146. package/packs/treasury-tic/policy.json +89 -0
  147. package/packs/treasury-tic/source.md +80 -0
  148. package/packs/uscourts/endpoints.json +22 -0
  149. package/packs/uscourts/policy.json +87 -0
  150. package/packs/uscourts/source.md +56 -0
  151. package/packs/uspto/source.md +1 -1
@@ -0,0 +1,98 @@
1
+ # Bureau of Justice Statistics: National Crime Victimization Survey and corrections statistics
2
+ Depth: standard. Key: none. Two kinds of data:
3
+ - NCVS Select records from BJS's API at api.ojp.gov: the person and household files of the National Crime
4
+ Victimization Survey, 1993-2024 (collection years), one row per victimization, person or household, with
5
+ weights.
6
+ - The CSV tables of BJS reports from bjs.ojp.gov: Criminal Victimization (official NCVS numbers, rates and
7
+ standard errors), Prisoners (National Prisoner Statistics), Jail Inmates, Probation and Parole, Capital
8
+ Punishment.
9
+
10
+ ## When to use
11
+ - Crime as victims report it, including crimes never reported to police: violent and property victimization
12
+ numbers and rates; the share reported to police; victim characteristics (age, sex, race, income); weapon,
13
+ injury and victim-offender relationship.
14
+ - People in prison or jail, on probation or parole, under sentence of death; imprisonment rates by state.
15
+ - Not here: crimes known to police, arrests and hate crimes (fbi); federal prosecutions and DOJ press releases
16
+ (doj-news); homicide (NCVS covers nonfatal crimes only; deaths by assault are in cdc); state court cases.
17
+
18
+ ## Endpoints
19
+ | Endpoint | Returns | Parameters |
20
+ |---|---|---|
21
+ | `query` | NCVS Select records (JSON rows, every value text) | `dataset` (catalog alias or 4x4 id), `$select` (plain columns), `$where`, `$order` |
22
+ | `statistical_table` | one CSV table of a report package | `report` (code or alias), `member` (file, e.g. `cv24t01.csv`), `header_lines` or `header_row` |
23
+
24
+ NCVS Select datasets: `personal_victimization` (gcuy-rt5g), `personal_population` (r4j4-fdwx),
25
+ `household_victimization` (gkck-euys), `household_population` (ya4e-n9zp). Report packages (verified
26
+ 2026-11-07): `cv24` and `cv23` (Criminal Victimization, 2024 and 2023), `p23st` (Prisoners in 2023), `ji24st`
27
+ and `ji23st` (Jail Inmates), `ppus23` (Probation and Parole, 2023), `cp22st` (Capital Punishment, 2022). In a
28
+ package, `<code>t01.csv` is table 1, `at01` appendix table 1 (estimates with standard errors and confidence
29
+ intervals), `f01` figure 1. A `member` that matches nothing fails with the list of files in the zip.
30
+
31
+ ## Spec examples
32
+ ```json
33
+ {"description": "Violent victimization numbers and rates, 2020-2024, as BJS published them",
34
+ "requests": [{"name": "cv24_t01", "endpoint": "statistical_table",
35
+ "params": {"report": "cv24", "member": "cv24t01.csv", "header_lines": "11-12"}}]}
36
+ ```
37
+ ```json
38
+ {"description": "2024 violent victimizations by type and reporting to police, from NCVS Select records",
39
+ "requests": [{"name": "violent_2024", "endpoint": "query", "params": {"dataset": "personal_victimization",
40
+ "$select": "year, newoff, notify, newwgt", "$where": "year = '2024' AND newcrime = '1'"}},
41
+ {"name": "persons_2024", "endpoint": "query", "params": {"dataset": "personal_population",
42
+ "$select": "year, wgtpercy", "$where": "year = '2024'"}}]}
43
+ ```
44
+ Then `compute`: the sum of `newwgt` by `newoff` gives victimizations. Divide by the sum of `wgtpercy` for the
45
+ year and multiply by 1,000 for the rate per 1,000 persons age 12 or older.
46
+
47
+ ## Data format and traps
48
+ - Report tables (`statistical_table`): the first fetch, without `header_lines`, gives every line numbered (`row`,
49
+ `col1`...). Lines 1-8 are the BJS banner, file name, title, report, data source, authors and version date;
50
+ the title repeats above the header. Read the header line numbers, then fetch again with `header_lines` (e.g.
51
+ `11-12`). Column names join the stacked header (`2024* | Number`, `2024* | Rate per 1,000/a`). A column of
52
+ footnote marks next to a value is named `<parent>_colN`: `†` or `‡` mean the difference from the comparison year
53
+ (marked `*`, e.g. `2024*`) is significant at the 95% or 90% level (table 1's notes).
54
+ Values are text with thousands separators (`6,671,640`). Row labels carry footnote letters (`Total violent
55
+ crime/b`), and the notes below the table explain them. Sub-rows are indented into the next column (`col2`).
56
+ Keep the footnote and significance notes with any number you quote.
57
+ - NCVS codes, from BJS's NCVS Select codebooks (person and household level):
58
+ - `newcrime` (person) is 1 violent crime, 2 personal theft/larceny. `newoff` (person) is 1 rape/sexual
59
+ assault, 2 robbery, 3 aggravated assault, 4 simple assault, 5 personal theft/larceny. `seriousviolent` 1
60
+ is violent crime excluding simple assault.
61
+ - Household: `newcrime` 3 is property crime; `newoff` is 6 burglary/trespassing, 7 motor-vehicle theft,
62
+ 8 theft.
63
+ - Demographic codes (`ager` 1-6, `sex`, `race_ethnicity` 1-6, `hincome1`, ...), the `notify` codes (reported to
64
+ police) and residue codes (98, -1 "invalid until ...") are in the codebooks:
65
+ https://bjs.ojp.gov/document/NCVS_Select_person_level_codebook.pdf and
66
+ https://bjs.ojp.gov/document/NCVS_Select_household_level_codebook.pdf. Read the code before filtering on it.
67
+ - Weights:
68
+ - `newwgt` is the series-adjusted victimization weight; use it for victimization counts (`wgtviccy` is the
69
+ unadjusted annual weight).
70
+ - `wgtpercy` (persons) and `wgthhcy` (households) are the population weights, the rate denominators.
71
+ - For prevalence (unique victims), use `wgt_ovam_per` and the `victim` flag.
72
+ - Never count rows: each row stands for thousands of victimizations.
73
+ - Reproducing BJS: the sum of `newwgt` over 2024 violent records is 6,671,636, and BJS publishes 6,671,640
74
+ (rounded to tens). The types match the same way (rape/sexual assault 560,890; robbery 642,150). Standard errors
75
+ and significance tests need the design variables (`pseudostratum`, `secucode`) and BJS's generalized variance
76
+ method; they are not computed here. Take them from the report's appendix tables, or say that none were computed.
77
+ - 2024 is the legacy-instrument half of a split sample (109,341 persons). Weights are larger and estimates less
78
+ precise (the codebook). Collection year: crimes are counted in the interview year, not the
79
+ year they happened. Victimizations outside the U.S. are excluded.
80
+ - Every column is text: quote values in `$where` (`year = '2024'`). Aggregate SoQL (`sum`, `count`, `$group`)
81
+ is refused before the request is sent, because the site answers it with a bot check. Fetch plain columns and
82
+ aggregate with `compute`. A year of `personal_population` has well over 100,000 rows, so select only the
83
+ columns you need.
84
+ - Small cells: a weighted sum over a handful of records is not a reliable estimate. When a tabulation splits a
85
+ year finely (by place size, race and crime type), say how many records are behind each cell, and prefer the
86
+ report tables, whose notes flag unreliable estimates. The person populations behind the rates are in
87
+ Criminal Victimization appendix tables (2024: appendix table 19).
88
+
89
+ ## Access
90
+ No key. `fetch` uses api.ojp.gov (records) and bjs.ojp.gov (report zips). Keep requests few: the site has a bot
91
+ check. Manual: `https://api.ojp.gov/bjsdataset/v1/<id>.csv?$where=year='2024'&$limit=200000` in a browser, or
92
+ the "Data tables" zip on the report's page (https://bjs.ojp.gov/library/publications/...), then `ingest`. API
93
+ documentation: https://bjs.ojp.gov/national-crime-victimization-survey-ncvs-api.
94
+
95
+ ## Citation and terms
96
+ "U.S. Department of Justice, Bureau of Justice Statistics, National Crime Victimization Survey, NCVS Select
97
+ <dataset> (or <report title and NCJ number>, table <n>), bjs.ojp.gov, retrieved <date>." Public domain (17 U.S.C.
98
+ 105). A tabulation of NCVS Select records is the analyst's own estimate, not a BJS estimate: say so.
@@ -0,0 +1,31 @@
1
+ {
2
+ "base_url": "https://www.cbp.gov",
3
+ "timeout": 120,
4
+ "document_page": "/document/stats/nationwide-encounters",
5
+ "endpoints": {
6
+ "list_files": {
7
+ "path": "/document/stats/nationwide-encounters",
8
+ "method": "GET",
9
+ "description": "The encounter data files CBP currently publishes (read from its Nationwide Encounters document page): one row per CSV with its path for the encounters request. The files are replaced every month.",
10
+ "params": {},
11
+ "records": "",
12
+ "paging": {"type": "none"},
13
+ "max_rows": 1000
14
+ },
15
+ "encounters": {
16
+ "path": "/sites/default/files/{file}.csv",
17
+ "method": "GET",
18
+ "description": "Nationwide encounters (OFO inadmissibles and Border Patrol apprehensions under Title 8) by fiscal year and month, component, land border region, area of responsibility or state, demographic and citizenship. Use list_files for the current file paths.",
19
+ "params": {
20
+ "file": {"in": "path", "raw": true, "required": true, "desc": "the file's path under /sites/default/files without .csv, from list_files, e.g. 2026-11/nationwide-encounters-fy24-fy27-oct-aor"},
21
+ "region": {"in": "spec", "desc": "keep these Land Border Region values: Southwest Land Border, Northern Land Border, Other"},
22
+ "component": {"in": "spec", "desc": "keep Office of Field Operations or U.S. Border Patrol"},
23
+ "fiscal_year": {"in": "spec", "desc": "keep these fiscal years as published (2025; the current year is '2027 (FYTD)')"},
24
+ "citizenship": {"in": "spec", "desc": "keep these citizenships as published (MEXICO, VENEZUELA ...)"}
25
+ },
26
+ "records": "$csv",
27
+ "paging": {"type": "none"},
28
+ "max_rows": 500000
29
+ }
30
+ }
31
+ }
@@ -0,0 +1,85 @@
1
+ {
2
+ "id": "cbp",
3
+ "title": "CBP nationwide encounters: border apprehensions and inadmissibles by month, region, demographic and citizenship",
4
+ "agency": "U.S. Department of Homeland Security, Customs and Border Protection",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "immigration",
8
+ "border security"
9
+ ],
10
+ "keywords": [
11
+ "cbp",
12
+ "customs and border protection",
13
+ "border encounters",
14
+ "southwest border",
15
+ "border crossings",
16
+ "border apprehensions",
17
+ "border patrol",
18
+ "inadmissibles",
19
+ "migrant encounters",
20
+ "unaccompanied children",
21
+ "uac",
22
+ "family units",
23
+ "northern border",
24
+ "office of field operations"
25
+ ],
26
+ "questions": [
27
+ "southwest border encounters by month",
28
+ "border apprehensions this year vs last year",
29
+ "encounters by citizenship (e.g. Venezuela, Mexico)",
30
+ "unaccompanied children encountered at the border",
31
+ "encounters by state or Border Patrol sector"
32
+ ],
33
+ "homepage": "https://www.cbp.gov/newsroom/stats/nationwide-encounters",
34
+ "terms_url": "https://www.cbp.gov/document/stats/nationwide-encounters",
35
+ "auth": {
36
+ "env": null,
37
+ "required": "none",
38
+ "placement": "none",
39
+ "signup_url": null,
40
+ "echoes_key": false
41
+ },
42
+ "limits": {
43
+ "rps": 1,
44
+ "burst": 2,
45
+ "max_rows": 500000
46
+ },
47
+ "routes": {
48
+ "api": {
49
+ "base": "https://www.cbp.gov",
50
+ "probe": "https://www.cbp.gov/document/stats/nationwide-encounters",
51
+ "hosts": [
52
+ "www.cbp.gov"
53
+ ]
54
+ },
55
+ "manual": {
56
+ "how": "Open https://www.cbp.gov/document/stats/nationwide-encounters, download the newest 'Nationwide Encounters' CSV (by AOR or by state), and upload it with ingest."
57
+ }
58
+ },
59
+ "cache": {
60
+ "mode": "run_only",
61
+ "ttl_hours": null,
62
+ "why": "Files are replaced monthly, with revisions to earlier months."
63
+ },
64
+ "redistribution": {
65
+ "mode": "allowed",
66
+ "why": "CBP statistics are U.S. government works in the public domain (17 U.S.C. 105)."
67
+ },
68
+ "notices": [],
69
+ "retention": null,
70
+ "citation": "U.S. Customs and Border Protection, Nationwide Encounters ({dataset}), cbp.gov, retrieved {retrieved_long}.",
71
+ "audit_hooks": [],
72
+ "clarify": [
73
+ "Which encounters? Southwest Land Border, Northern Land Border, or nationwide (including Other: airports and seaports); Border Patrol apprehensions between ports or OFO inadmissibles at ports, or both. Default: Southwest Land Border, both components, said as such.",
74
+ "Which months? Federal fiscal years (October-September); the current year is partial (FYTD). Default: the latest month and the same month a year earlier."
75
+ ],
76
+ "formats": [
77
+ "csv",
78
+ "html"
79
+ ],
80
+ "aliases": [
81
+ "CBP",
82
+ "Customs and Border Protection",
83
+ "Border Patrol"
84
+ ]
85
+ }
@@ -0,0 +1,57 @@
1
+ # CBP nationwide encounters: border apprehensions and inadmissibles by month, region, demographic and citizenship
2
+ Depth: standard. Key: none. U.S. Customs and Border Protection's monthly Nationwide Encounters data files (the CSVs
3
+ behind CBP's encounters dashboard). They count Border Patrol apprehensions (between ports of entry) and Office of
4
+ Field Operations inadmissibles (at ports of entry), under Title 8. Breakdowns: fiscal year and month, land border
5
+ region, area of responsibility (Border Patrol sector or field office) or state, demographic and citizenship.
6
+
7
+ ## When to use
8
+ - Southwest border encounters by month; year-over-year change; encounters by citizenship, demographic (single
9
+ adults, family units, unaccompanied children, accompanied minors), component, sector or field office, or state.
10
+ - Not here: annual immigration statistics (green cards, refugees, naturalizations, removals: dhs-ohss); asylum
11
+ and immigration court cases (not covered); trade and customs duties (census, usitc, treasury).
12
+
13
+ ## Endpoints
14
+ | Endpoint | Returns | Parameters |
15
+ |---|---|---|
16
+ | `list_files` | the encounter CSVs on CBP's document page, newest first: `file`, `by` (area of responsibility or state), `fiscal_years`, `url` | none |
17
+ | `encounters` | one file's rows as published | `file` (from `list_files`), filters `region`, `component`, `fiscal_year`, `citizenship` |
18
+
19
+ Columns:
20
+ - By area of responsibility (`-aor`): Fiscal Year, Month Grouping, Month (abbv), Component, Land Border Region,
21
+ Area of Responsibility, AOR (Abbv), Demographic, Citizenship, Title of Authority, Encounter Type, Encounter
22
+ Count.
23
+ - By state (`-state`): Fiscal Year, Month Grouping, Month (abbv), Land Border Region, State, Demographic,
24
+ Citizenship, Title of Authority, Encounter Count.
25
+
26
+ ## Spec example
27
+ ```json
28
+ {"description": "Southwest border encounters by month, FY2024 to date (file of November 2026)",
29
+ "requests": [{"name": "files", "endpoint": "list_files"},
30
+ {"name": "swb", "endpoint": "encounters",
31
+ "params": {"file": "2026-11/nationwide-encounters-fy24-fy27-oct-aor", "region": "Southwest Land Border"}}]}
32
+ ```
33
+ Run `list_files` first: the newest file is replaced every month (paths verified 2026-11-07).
34
+
35
+ ## Data format and traps
36
+ - Each file covers about four fiscal years, the last one partial (`2027 (FYTD)`). Earlier months are revised in
37
+ later files: use the newest file, and do not mix files.
38
+ - `Month Grouping` is `FYTD` for the months that fall in the current fiscal year-to-date period, marked in every
39
+ year for comparison (only OCT in the November 2026 file), and `Remaining` for the others. Months are abbreviations (`OCT` ... `SEP`), and the fiscal year starts in October. Build a
40
+ date before plotting (FY2025 OCT = October 2024).
41
+ - Encounters are events, not people: one person can be encountered more than once.
42
+ - OFO inadmissibles and Border Patrol apprehensions are different events. Add them only for "total
43
+ encounters", and say so.
44
+ - `Land Border Region` `Other` holds encounters away from land borders (airports, seaports, coastal).
45
+ - `Citizenship` lists the main countries and groups the rest as `OTHER`: a country absent from the list is
46
+ inside OTHER, not zero. Demographic codes: `FMUA` is individuals in family units, `UAC` unaccompanied children,
47
+ plus Single Adults and Accompanied Minors.
48
+ - Counts are text; sum with `Number()`. Every row of the November 2026 files is Title 8: older files also have
49
+ Title 42 expulsions (March 2020 to May 2023), which are encounters too.
50
+
51
+ ## Access
52
+ No key. `fetch` reads www.cbp.gov (verified 2026-11-07; the files are 2-5 MB). Manual: the document page's CSV
53
+ links, then `ingest`.
54
+
55
+ ## Citation and terms
56
+ "U.S. Customs and Border Protection, Nationwide Encounters, <file name>, cbp.gov, retrieved <date>." Public
57
+ domain (17 U.S.C. 105).
@@ -53,7 +53,7 @@ narrow `$where`), or open the dataset page.
53
53
  - Provisional data (weekly deaths, VSRR overdose, quarterly mortality) are revised for months and may be
54
54
  incomplete for recent periods; report the data date. Small counts are suppressed in several tables.
55
55
  - Leading causes of death ends in 2017 and life expectancy in 2018 on this portal; newer years are in CDC
56
- WONDER (not covered here). Say so instead of mixing sources silently.
56
+ WONDER (the cdc-wonder pack, by manual export). Say so instead of mixing sources silently.
57
57
  - Paging uses `$limit`/`$offset` with a stable `$order` (default `:id`); without a key Socrata throttles
58
58
  heavy use (about 1,000 requests an hour). A `$group` query fails with the default order ("Column ':id'
59
59
  is not in group by"): give it an explicit `$order` on a selected or grouped column.
@@ -0,0 +1,26 @@
1
+ {
2
+ "base_url": "https://wonder.cdc.gov",
3
+ "timeout": 60,
4
+ "endpoints": {
5
+ "export": {
6
+ "path": "https://wonder.cdc.gov/",
7
+ "method": "GET",
8
+ "description": "A CDC WONDER query result, exported from the WONDER web site (Export, tab-delimited .txt) and uploaded: deaths, populations and rates by the Group By fields chosen, e.g. underlying cause of death by year, state, age, sex, race. Manual only: wonder.cdc.gov refuses this client.",
9
+ "params": {
10
+ "database": {
11
+ "in": "spec",
12
+ "required": true,
13
+ "desc": "the WONDER database the query ran on, as WONDER names it (e.g. 'Underlying Cause of Death, 2018-2023, Single Race'), cited with the result"
14
+ },
15
+ "query": {
16
+ "in": "spec",
17
+ "desc": "what the query selected and grouped by, in words (for INSTRUCTIONS.md and the citation)"
18
+ }
19
+ },
20
+ "records": "$tsv",
21
+ "header_row": 1,
22
+ "paging": {"type": "none"},
23
+ "max_rows": 500000
24
+ }
25
+ }
26
+ }
@@ -0,0 +1,84 @@
1
+ {
2
+ "id": "cdc-wonder",
3
+ "title": "CDC WONDER: mortality, natality and population query results (manual export)",
4
+ "agency": "Centers for Disease Control and Prevention, National Center for Health Statistics",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "health",
8
+ "mortality",
9
+ "vital statistics"
10
+ ],
11
+ "keywords": [
12
+ "cdc wonder",
13
+ "wonder",
14
+ "underlying cause of death",
15
+ "multiple cause of death",
16
+ "death certificates",
17
+ "mortality rate",
18
+ "age-adjusted death rate",
19
+ "deaths by cause",
20
+ "deaths by county",
21
+ "icd-10",
22
+ "natality",
23
+ "births",
24
+ "infant mortality",
25
+ "linked birth infant death",
26
+ "compressed mortality"
27
+ ],
28
+ "questions": [
29
+ "age-adjusted death rate for a cause of death by state and year",
30
+ "deaths by ICD-10 cause, age group and sex",
31
+ "county death rates for a cause",
32
+ "births by mother's age or state",
33
+ "infant mortality rate by race"
34
+ ],
35
+ "homepage": "https://wonder.cdc.gov/",
36
+ "terms_url": "https://wonder.cdc.gov/datause.html",
37
+ "auth": {
38
+ "env": null,
39
+ "required": "none",
40
+ "placement": "none",
41
+ "signup_url": null,
42
+ "echoes_key": false
43
+ },
44
+ "limits": {
45
+ "rps": 1,
46
+ "burst": 1,
47
+ "max_rows": 500000
48
+ },
49
+ "routes": {
50
+ "manual": {
51
+ "how": "Open https://wonder.cdc.gov/ in a web browser, choose the database (e.g. Mortality: Underlying Cause of Death), read and accept its data use restrictions yourself, set Group By and the filters, click Send, then Export (tab-delimited .txt) with 'Show Totals' and notes as wanted. Upload the exported .txt file with ingest."
52
+ }
53
+ },
54
+ "cache": {
55
+ "mode": "none",
56
+ "ttl_hours": null,
57
+ "why": "Nothing is fetched; the uploaded export is the data."
58
+ },
59
+ "redistribution": {
60
+ "mode": "aggregates_only",
61
+ "microdata_globs": [
62
+ "raw/*"
63
+ ],
64
+ "why": "CDC WONDER's data use restrictions (wonder.cdc.gov/datause.html, read 2026-11-07): statistical reporting and analysis only; no attempt to identify any person or establishment; do not present or publish statistics representing nine or fewer births or deaths, including rates based on them. Exported tables are aggregates; the uploaded export stays in raw/."
65
+ },
66
+ "notices": [
67
+ "CDC WONDER data use restrictions (https://wonder.cdc.gov/datause.html): use for statistical reporting and analysis only; make no attempt to learn the identity of any person or establishment; do not present or publish statistics representing nine or fewer births or deaths, including rates based on counts of nine or fewer. The user accepts these on the WONDER site before running a query."
68
+ ],
69
+ "retention": null,
70
+ "citation": "Centers for Disease Control and Prevention, National Center for Health Statistics, {dataset}, CDC WONDER Online Database, accessed {retrieved_long}.",
71
+ "audit_hooks": [],
72
+ "clarify": [
73
+ "Which WONDER database and years? Final mortality (e.g. Underlying Cause of Death by single race, 2018 on; the bridged-race file for 1999-2020) or provisional; natality. The cause is coded in ICD-10. Default: the newest final underlying-cause database.",
74
+ "Crude or age-adjusted rates? Comparing places or years needs age-adjusted rates (WONDER's 2000 U.S. standard population). Default: age-adjusted, with the counts."
75
+ ],
76
+ "formats": [
77
+ "txt"
78
+ ],
79
+ "aliases": [
80
+ "WONDER",
81
+ "CDC WONDER",
82
+ "Wide-ranging Online Data for Epidemiologic Research"
83
+ ]
84
+ }
@@ -0,0 +1,53 @@
1
+ # CDC WONDER: mortality, natality and population query results (manual export)
2
+ Depth: standard. Key: none. Manual only. CDC WONDER (NCHS vital statistics) query results that the user runs on
3
+ the WONDER web site and exports. Deaths by underlying or multiple cause (ICD-10), births (natality), infant
4
+ deaths, with populations and crude and age-adjusted rates, by year, state, county, age, sex, race and Hispanic
5
+ origin, as grouped in the query.
6
+
7
+ ## When to use
8
+ - Deaths and death rates for causes, years and places that the cdc pack's Socrata tables do not have: cdc's
9
+ leading causes end in 2017 and life expectancy in 2018, while WONDER has final data to the latest year;
10
+ county rates; any ICD-10 code or group; age-adjusted rates by demographic group.
11
+ - Births by mother's characteristics, infant mortality.
12
+ - Not here: provisional weekly deaths and overdose counts that cdc's Socrata tables carry (cdc); health
13
+ insurance (census); injuries treated in emergency departments (cpsc NEISS).
14
+
15
+ ## Endpoints
16
+ | Endpoint | Returns | Parameters |
17
+ |---|---|---|
18
+ | `export` | the uploaded WONDER export as published: one column per Group By field (name and code columns, e.g. `Year`, `Year Code`), then `Deaths`, `Population`, `Crude Rate`, `Age Adjusted Rate` ... as chosen; `Notes` (`Total` on total rows) | `database` (required: the WONDER database name), `query` (the query in words) |
19
+
20
+ ## Spec example
21
+ ```json
22
+ {"description": "Age-adjusted suicide death rates by state, 2023 (WONDER export)",
23
+ "requests": [{"name": "suicide_2023", "endpoint": "export",
24
+ "params": {"database": "Underlying Cause of Death, 2018-2023, Single Race",
25
+ "query": "Group By State; Year 2023; ICD-10 113 Cause List: Intentional self-harm (suicide); age-adjusted rates"}}]}
26
+ ```
27
+
28
+ ## Data format and traps
29
+ - The export layout was not checked against a real file here. The reader takes the first line as the header
30
+ and stops at the "---" line, where WONDER's notes begin. Read the columns and the notes (the uploaded file is
31
+ kept in `raw/`) before using the table. Cite the query criteria the notes list.
32
+ - `Notes` = "Total" marks total rows: drop them before summing groups.
33
+ - Suppression and reliability: WONDER shows small counts as "Suppressed" and flags some rates "Unreliable". Keep
34
+ them as text and never fill them with 0. The data use restrictions forbid presenting or publishing any
35
+ statistic representing nine or fewer births or deaths, including rates based on them: aggregate further
36
+ instead.
37
+ - Compare places or years with age-adjusted rates, not crude ones, and say which standard population (WONDER
38
+ states it in the notes).
39
+ - Databases differ: single-race (2018 on) and bridged-race (1999-2020) files use different race categories.
40
+ Final and provisional data differ. Do not join years across databases without saying so. Causes are ICD-10;
41
+ name the codes.
42
+ - Values are text with WONDER's formatting; convert with `Number()` after removing thousands separators.
43
+
44
+ ## Access
45
+ Manual only: wonder.cdc.gov answers this client with HTTP 403 (verified 2026-11-07). The WONDER API would also
46
+ require every query to accept the data use restrictions on the user's behalf. The user opens
47
+ https://wonder.cdc.gov/, chooses the database, reads and accepts the data use restrictions, sets the query,
48
+ clicks Send, then Export, and uploads the saved .txt; then `ingest`.
49
+
50
+ ## Citation and terms
51
+ "Centers for Disease Control and Prevention, National Center for Health Statistics. <database name> on CDC
52
+ WONDER Online Database, accessed <date>." Data use restrictions: https://wonder.cdc.gov/datause.html (statistical
53
+ reporting and analysis only; no re-identification; no statistics on nine or fewer births or deaths).
@@ -6,7 +6,7 @@ cost, aid, debt, completion, earnings and admissions data for every U.S. college
6
6
  - Compare colleges on tuition, net price, admission rate, completion rate, median debt, Pell share and
7
7
  median earnings 6, 7 or 10 years after entry; list schools in a state, by ownership or by degree type.
8
8
  - Not here: K-12 achievement (naep), enrollment and finance surveys beyond what Scorecard carries (IPEDS
9
- itself, not covered), student loan portfolio totals (fred, treasury), earnings by occupation (bls).
9
+ itself: nces), student loan portfolio totals (fred, treasury), earnings by occupation (bls).
10
10
  - One row per institution (UNITID). Field-of-study (program) data exist but are very large per school.
11
11
 
12
12
  ## Endpoints and datasets
@@ -0,0 +1,91 @@
1
+ {
2
+ "base_url": "https://www.saferproducts.gov/RestWebServices",
3
+ "timeout": 60,
4
+ "endpoints": {
5
+ "recalls": {
6
+ "path": "/Recall",
7
+ "method": "GET",
8
+ "description": "CPSC product recalls (SaferProducts.gov Recall API): one row per recall with products, units, hazards, remedies, injuries, manufacturers, retailers and importers. Give at least one filter; text filters match part of the field.",
9
+ "params": {
10
+ "RecallDateStart": {
11
+ "desc": "recalls announced on or after this date, YYYY-MM-DD"
12
+ },
13
+ "RecallDateEnd": {
14
+ "desc": "recalls announced on or before this date, YYYY-MM-DD"
15
+ },
16
+ "LastPublishDateStart": {
17
+ "desc": "recalls published or updated on or after this date, YYYY-MM-DD"
18
+ },
19
+ "RecallNumber": {
20
+ "desc": "CPSC recall number, e.g. 27066"
21
+ },
22
+ "ProductName": {
23
+ "desc": "part of a product name (crib, dresser)"
24
+ },
25
+ "ProductType": {
26
+ "desc": "part of CPSC's product type (Slides, Cribs, Dressers)"
27
+ },
28
+ "RecallTitle": {
29
+ "desc": "part of the recall title"
30
+ },
31
+ "RecallDescription": {
32
+ "desc": "part of the recall description"
33
+ },
34
+ "Manufacturer": {
35
+ "desc": "part of a manufacturer name"
36
+ },
37
+ "Retailer": {
38
+ "desc": "part of the retailer text (Amazon, Walmart)"
39
+ },
40
+ "Importer": {
41
+ "desc": "part of an importer name"
42
+ },
43
+ "Injury": {
44
+ "desc": "part of the injuries text (death, burn)"
45
+ },
46
+ "Remedy": {
47
+ "desc": "part of the remedy text (Refund, Repair, Replace)"
48
+ },
49
+ "hazard": {
50
+ "in": "spec",
51
+ "desc": "keep recalls whose hazard text contains this (case-insensitive); filtered here, because the API's own Hazard filter matches nothing"
52
+ },
53
+ "country": {
54
+ "in": "spec",
55
+ "desc": "keep recalls with this manufacturer country (China, Vietnam); filtered here"
56
+ }
57
+ },
58
+ "defaults": {
59
+ "format": "json"
60
+ },
61
+ "records": "",
62
+ "paging": {
63
+ "type": "none"
64
+ },
65
+ "max_rows": 20000,
66
+ "flatten": false
67
+ },
68
+ "neiss": {
69
+ "path": "https://www.cpsc.gov/cgibin/NEISSQuery/home.aspx",
70
+ "method": "GET",
71
+ "description": "NEISS: one year of emergency-department injury cases involving consumer products (about 300,000 sampled cases a year, with national weights). Manual only: cpsc.gov refuses scripts; the user downloads the archived year as Tab Delimited Text (or Excel) and uploads it.",
72
+ "params": {
73
+ "year": {
74
+ "in": "spec",
75
+ "required": true,
76
+ "desc": "the data year, 2006 or later"
77
+ },
78
+ "product": {
79
+ "in": "spec",
80
+ "desc": "keep cases with this NEISS product code in Product_1, Product_2 or Product_3 (a code or a list; the codes are in the NEISS Coding Manual)"
81
+ }
82
+ },
83
+ "records": "$tsv",
84
+ "header_row": 1,
85
+ "paging": {
86
+ "type": "none"
87
+ },
88
+ "max_rows": 1000000
89
+ }
90
+ }
91
+ }
@@ -0,0 +1,91 @@
1
+ {
2
+ "id": "cpsc",
3
+ "title": "Consumer Product Safety Commission: product recalls and NEISS injury data",
4
+ "agency": "U.S. Consumer Product Safety Commission",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "consumer safety",
8
+ "product recalls",
9
+ "injuries",
10
+ "health"
11
+ ],
12
+ "keywords": [
13
+ "cpsc",
14
+ "consumer product safety commission",
15
+ "product recall",
16
+ "recalls",
17
+ "recalled product",
18
+ "saferproducts",
19
+ "crib recall",
20
+ "dresser tip-over",
21
+ "toy recall",
22
+ "neiss",
23
+ "national electronic injury surveillance system",
24
+ "emergency department injuries",
25
+ "product-related injuries",
26
+ "injury estimates",
27
+ "consumer product injuries"
28
+ ],
29
+ "questions": [
30
+ "consumer product recalls announced this year",
31
+ "recalls of cribs, dressers or strollers",
32
+ "recalled products made in China",
33
+ "recalls that mention deaths",
34
+ "emergency department visits for injuries involving a product (NEISS)",
35
+ "national estimate of injuries by product, age or sex"
36
+ ],
37
+ "homepage": "https://www.cpsc.gov/Recalls",
38
+ "terms_url": "https://www.saferproducts.gov/",
39
+ "auth": {
40
+ "env": null,
41
+ "required": "none",
42
+ "placement": "none",
43
+ "signup_url": null,
44
+ "echoes_key": false
45
+ },
46
+ "limits": {
47
+ "rps": 2,
48
+ "burst": 4,
49
+ "max_rows": 1000000
50
+ },
51
+ "routes": {
52
+ "api": {
53
+ "base": "https://www.saferproducts.gov/RestWebServices",
54
+ "probe": "https://www.saferproducts.gov/RestWebServices/Recall?format=json&RecallNumber=27066",
55
+ "hosts": [
56
+ "www.saferproducts.gov"
57
+ ]
58
+ },
59
+ "manual": {
60
+ "how": "Recalls: open the Recall API link (https://www.saferproducts.gov/RestWebServices/Recall?format=json&RecallDateStart=YYYY-MM-DD) in a browser and save the JSON. NEISS: open https://www.cpsc.gov/cgibin/NEISSQuery/home.aspx, pick a User Affiliation, choose the year under Archived Annual NEISS Data with File Format Tab Delimited Text, click Find Data, save the file. Upload with ingest."
61
+ }
62
+ },
63
+ "cache": {
64
+ "mode": "run_only",
65
+ "ttl_hours": null,
66
+ "why": "Recalls are added and updated daily; data/ is reused within a run."
67
+ },
68
+ "redistribution": {
69
+ "mode": "allowed",
70
+ "why": "CPSC recall notices and NEISS public-use data are U.S. government works in the public domain (17 U.S.C. 105)."
71
+ },
72
+ "notices": [],
73
+ "retention": null,
74
+ "citation": "U.S. Consumer Product Safety Commission, {dataset}, saferproducts.gov and cpsc.gov, retrieved {retrieved_long}.",
75
+ "audit_hooks": [],
76
+ "clarify": [
77
+ "Recalls or injuries? Recalls list products CPSC and firms recalled (counts of recalls and units); NEISS estimates emergency-department visits for injuries associated with products (national estimates from a sample, not counts of incidents). Default: recalls for 'recalled products', NEISS for 'how many injuries'.",
78
+ "Which period? Recalls by announcement date (RecallDateStart/End); NEISS by treatment year. Default: the last full calendar year."
79
+ ],
80
+ "formats": [
81
+ "json",
82
+ "tsv",
83
+ "xlsx"
84
+ ],
85
+ "aliases": [
86
+ "CPSC",
87
+ "Consumer Product Safety Commission",
88
+ "NEISS",
89
+ "SaferProducts.gov"
90
+ ]
91
+ }