data-analyst-mcp 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (292) hide show
  1. package/README.md +286 -262
  2. package/dist/adapters/bjs.d.ts +13 -0
  3. package/dist/adapters/bjs.js +28 -0
  4. package/dist/adapters/bjs.js.map +1 -0
  5. package/dist/adapters/cbp.d.ts +5 -0
  6. package/dist/adapters/cbp.js +64 -0
  7. package/dist/adapters/cbp.js.map +1 -0
  8. package/dist/adapters/cdc-wonder.d.ts +5 -0
  9. package/dist/adapters/cdc-wonder.js +68 -0
  10. package/dist/adapters/cdc-wonder.js.map +1 -0
  11. package/dist/adapters/cpsc.d.ts +4 -0
  12. package/dist/adapters/cpsc.js +125 -0
  13. package/dist/adapters/cpsc.js.map +1 -0
  14. package/dist/adapters/fhwa.d.ts +9 -0
  15. package/dist/adapters/fhwa.js +16 -0
  16. package/dist/adapters/fhwa.js.map +1 -0
  17. package/dist/adapters/foreign-assistance.d.ts +9 -0
  18. package/dist/adapters/foreign-assistance.js +42 -0
  19. package/dist/adapters/foreign-assistance.js.map +1 -0
  20. package/dist/adapters/hud-ahar.d.ts +3 -0
  21. package/dist/adapters/hud-ahar.js +80 -0
  22. package/dist/adapters/hud-ahar.js.map +1 -0
  23. package/dist/adapters/nces.d.ts +10 -0
  24. package/dist/adapters/nces.js +32 -0
  25. package/dist/adapters/nces.js.map +1 -0
  26. package/dist/adapters/nsf.d.ts +11 -0
  27. package/dist/adapters/nsf.js +25 -0
  28. package/dist/adapters/nsf.js.map +1 -0
  29. package/dist/adapters/onrr.d.ts +9 -0
  30. package/dist/adapters/onrr.js +35 -0
  31. package/dist/adapters/onrr.js.map +1 -0
  32. package/dist/adapters/sba.d.ts +10 -0
  33. package/dist/adapters/sba.js +134 -0
  34. package/dist/adapters/sba.js.map +1 -0
  35. package/dist/adapters/treasury-tic.d.ts +30 -0
  36. package/dist/adapters/treasury-tic.js +149 -0
  37. package/dist/adapters/treasury-tic.js.map +1 -0
  38. package/dist/adapters/uscourts.d.ts +8 -0
  39. package/dist/adapters/uscourts.js +8 -0
  40. package/dist/adapters/uscourts.js.map +1 -0
  41. package/dist/analysis/cite.d.ts +9 -1
  42. package/dist/analysis/cite.js +9 -2
  43. package/dist/analysis/cite.js.map +1 -1
  44. package/dist/bin/cli.d.ts +2 -0
  45. package/dist/bin/cli.js +17 -0
  46. package/dist/bin/cli.js.map +1 -1
  47. package/dist/core/generic-rest.js +42 -17
  48. package/dist/core/generic-rest.js.map +1 -1
  49. package/dist/core/guard.d.ts +32 -0
  50. package/dist/core/guard.js +176 -0
  51. package/dist/core/guard.js.map +1 -0
  52. package/dist/core/http.d.ts +10 -0
  53. package/dist/core/http.js +36 -7
  54. package/dist/core/http.js.map +1 -1
  55. package/dist/core/keys.d.ts +13 -3
  56. package/dist/core/keys.js +67 -7
  57. package/dist/core/keys.js.map +1 -1
  58. package/dist/core/package.d.ts +12 -3
  59. package/dist/core/package.js +56 -12
  60. package/dist/core/package.js.map +1 -1
  61. package/dist/core/secret_patterns.d.ts +6 -0
  62. package/dist/core/secret_patterns.js +20 -0
  63. package/dist/core/secret_patterns.js.map +1 -0
  64. package/dist/core/tabular.d.ts +43 -1
  65. package/dist/core/tabular.js +149 -6
  66. package/dist/core/tabular.js.map +1 -1
  67. package/dist/envelope.js +3 -1
  68. package/dist/envelope.js.map +1 -1
  69. package/dist/index.d.ts +29 -22
  70. package/dist/index.js +45 -46
  71. package/dist/index.js.map +1 -1
  72. package/dist/remote/app.js +87 -31
  73. package/dist/remote/app.js.map +1 -1
  74. package/dist/remote/oauth.js +8 -1
  75. package/dist/remote/oauth.js.map +1 -1
  76. package/dist/remote/pages.d.ts +3 -7
  77. package/dist/remote/pages.js +49 -19
  78. package/dist/remote/pages.js.map +1 -1
  79. package/dist/remote/sign.d.ts +7 -1
  80. package/dist/remote/sign.js +33 -7
  81. package/dist/remote/sign.js.map +1 -1
  82. package/dist/remote/store.js +14 -5
  83. package/dist/remote/store.js.map +1 -1
  84. package/dist/report/audit_report.js +29 -7
  85. package/dist/report/audit_report.js.map +1 -1
  86. package/dist/report/build_report.js +34 -4
  87. package/dist/report/build_report.js.map +1 -1
  88. package/dist/report/method_audit.js +10 -1
  89. package/dist/report/method_audit.js.map +1 -1
  90. package/dist/report/scan_secrets.d.ts +4 -1
  91. package/dist/report/scan_secrets.js +38 -18
  92. package/dist/report/scan_secrets.js.map +1 -1
  93. package/dist/server.d.ts +3 -1
  94. package/dist/server.js +46 -13
  95. package/dist/server.js.map +1 -1
  96. package/dist/tools.js +27 -6
  97. package/dist/tools.js.map +1 -1
  98. package/guide/WORKFLOW.md +134 -134
  99. package/guide/assets/disclaimer.txt +1 -1
  100. package/guide/assets/download_data.ps1 +164 -162
  101. package/guide/assets/download_data.sh +8 -2
  102. package/guide/assets/report_template.html +597 -597
  103. package/guide/references/routing.md +198 -171
  104. package/guide/references/style_guide.md +398 -398
  105. package/package.json +56 -56
  106. package/packs/acf/endpoints.json +114 -114
  107. package/packs/acf/policy.json +104 -104
  108. package/packs/actuaries/endpoints.json +82 -82
  109. package/packs/actuaries/policy.json +109 -109
  110. package/packs/bea/assets/download_bea.ps1 +182 -182
  111. package/packs/bea/docs/bea_api.md +461 -461
  112. package/packs/bea/docs/datasets.md +157 -157
  113. package/packs/bea/docs/report_patterns.md +112 -112
  114. package/packs/bea/policy.json +170 -170
  115. package/packs/bea/source.md +152 -152
  116. package/packs/bjs/endpoints.json +150 -0
  117. package/packs/bjs/policy.json +114 -0
  118. package/packs/bjs/source.md +98 -0
  119. package/packs/bls/assets/download_bls.ps1 +152 -152
  120. package/packs/bls/docs/analysis_patterns.md +108 -108
  121. package/packs/bls/docs/bls_api.md +73 -73
  122. package/packs/bls/docs/series_ids.md +149 -149
  123. package/packs/bls/policy.json +238 -238
  124. package/packs/bts/endpoints.json +346 -346
  125. package/packs/bts/policy.json +94 -94
  126. package/packs/cbp/endpoints.json +31 -0
  127. package/packs/cbp/policy.json +85 -0
  128. package/packs/cbp/source.md +57 -0
  129. package/packs/cdc/endpoints.json +197 -197
  130. package/packs/cdc/policy.json +83 -83
  131. package/packs/cdc/source.md +1 -1
  132. package/packs/cdc-wonder/endpoints.json +26 -0
  133. package/packs/cdc-wonder/policy.json +84 -0
  134. package/packs/cdc-wonder/source.md +53 -0
  135. package/packs/census/endpoints.json +99 -99
  136. package/packs/cfpb/endpoints.json +326 -326
  137. package/packs/cfpb/policy.json +119 -119
  138. package/packs/clinical-trials/endpoints.json +166 -166
  139. package/packs/clinical-trials/policy.json +84 -84
  140. package/packs/cms/endpoints.json +69 -69
  141. package/packs/cms/policy.json +96 -96
  142. package/packs/college-scorecard/endpoints.json +252 -252
  143. package/packs/college-scorecard/policy.json +106 -106
  144. package/packs/college-scorecard/source.md +1 -1
  145. package/packs/congress/endpoints.json +1701 -1701
  146. package/packs/congress/policy.json +117 -117
  147. package/packs/cpsc/endpoints.json +91 -0
  148. package/packs/cpsc/policy.json +91 -0
  149. package/packs/cpsc/source.md +76 -0
  150. package/packs/dhs-ohss/endpoints.json +66 -66
  151. package/packs/dhs-ohss/policy.json +107 -107
  152. package/packs/dhs-ohss/source.md +1 -1
  153. package/packs/doj-news/endpoints.json +59 -59
  154. package/packs/doj-news/policy.json +96 -96
  155. package/packs/doj-news/source.md +1 -1
  156. package/packs/dol/endpoints.json +817 -817
  157. package/packs/dol/policy.json +135 -135
  158. package/packs/eia/endpoints.json +717 -717
  159. package/packs/eia/policy.json +112 -112
  160. package/packs/epa/endpoints.json +291 -291
  161. package/packs/epa/policy.json +99 -99
  162. package/packs/epa-aqs/endpoints.json +173 -173
  163. package/packs/epa-aqs/policy.json +96 -96
  164. package/packs/fbi/endpoints.json +1841 -1841
  165. package/packs/fbi/policy.json +118 -118
  166. package/packs/fbi/source.md +3 -3
  167. package/packs/fda/endpoints.json +237 -237
  168. package/packs/fda/policy.json +121 -121
  169. package/packs/fdic/endpoints.json +416 -416
  170. package/packs/fdic/policy.json +83 -83
  171. package/packs/fec/endpoints.json +707 -707
  172. package/packs/fec/policy.json +93 -93
  173. package/packs/fed-board/endpoints.json +89 -89
  174. package/packs/fed-board/policy.json +136 -136
  175. package/packs/federal-register/endpoints.json +143 -143
  176. package/packs/federal-register/policy.json +86 -86
  177. package/packs/fema/endpoints.json +645 -645
  178. package/packs/fema/policy.json +103 -103
  179. package/packs/fhfa/endpoints.json +105 -105
  180. package/packs/fhfa/policy.json +109 -109
  181. package/packs/fhwa/endpoints.json +48 -0
  182. package/packs/fhwa/policy.json +91 -0
  183. package/packs/fhwa/source.md +59 -0
  184. package/packs/foreign-assistance/endpoints.json +166 -0
  185. package/packs/foreign-assistance/policy.json +80 -0
  186. package/packs/foreign-assistance/source.md +76 -0
  187. package/packs/fred/assets/download_fred.ps1 +122 -122
  188. package/packs/fred/docs/analysis_patterns.md +124 -124
  189. package/packs/fred/docs/fred_api.md +94 -94
  190. package/packs/fred/docs/series.md +188 -188
  191. package/packs/fred/policy.json +253 -253
  192. package/packs/fsa/endpoints.json +150 -150
  193. package/packs/fsa/policy.json +99 -99
  194. package/packs/fsa/source.md +1 -1
  195. package/packs/govinfo/endpoints.json +519 -519
  196. package/packs/gsa-calc/endpoints.json +93 -93
  197. package/packs/gsa-calc/policy.json +97 -97
  198. package/packs/hud/endpoints.json +146 -146
  199. package/packs/hud/policy.json +84 -84
  200. package/packs/hud/source.md +1 -1
  201. package/packs/hud-ahar/endpoints.json +92 -0
  202. package/packs/hud-ahar/policy.json +88 -0
  203. package/packs/hud-ahar/source.md +72 -0
  204. package/packs/ipums/docs/report_patterns.md +117 -117
  205. package/packs/irs-soi/endpoints.json +114 -114
  206. package/packs/irs-soi/policy.json +107 -107
  207. package/packs/medicaid/endpoints.json +412 -412
  208. package/packs/medicaid/policy.json +108 -108
  209. package/packs/naep/endpoints.json +217 -217
  210. package/packs/naep/policy.json +100 -100
  211. package/packs/naep/source.md +1 -1
  212. package/packs/nces/endpoints.json +101 -0
  213. package/packs/nces/policy.json +99 -0
  214. package/packs/nces/source.md +75 -0
  215. package/packs/nhtsa/endpoints.json +132 -132
  216. package/packs/nhtsa/policy.json +93 -93
  217. package/packs/nhtsa/source.md +1 -1
  218. package/packs/nih/endpoints.json +452 -452
  219. package/packs/nih/policy.json +86 -86
  220. package/packs/noaa/endpoints.json +431 -431
  221. package/packs/noaa/policy.json +95 -95
  222. package/packs/nrel/endpoints.json +327 -327
  223. package/packs/nrel/policy.json +96 -96
  224. package/packs/nsf/endpoints.json +33 -0
  225. package/packs/nsf/policy.json +83 -0
  226. package/packs/nsf/source.md +59 -0
  227. package/packs/nws/endpoints.json +271 -271
  228. package/packs/nws/policy.json +82 -82
  229. package/packs/nyfed/endpoints.json +80 -80
  230. package/packs/nyfed/policy.json +92 -92
  231. package/packs/omb/endpoints.json +460 -460
  232. package/packs/omb/policy.json +106 -106
  233. package/packs/onrr/endpoints.json +198 -0
  234. package/packs/onrr/policy.json +91 -0
  235. package/packs/onrr/source.md +55 -0
  236. package/packs/open-payments/endpoints.json +297 -297
  237. package/packs/open-payments/policy.json +101 -101
  238. package/packs/opm/assets/download_opm.ps1 +125 -125
  239. package/packs/opm/docs/fields.md +184 -184
  240. package/packs/opm/docs/opm_api.md +71 -71
  241. package/packs/opm/docs/report_patterns.md +83 -83
  242. package/packs/opm/lib/data_dictionary.csv +73 -73
  243. package/packs/pbgc/endpoints.json +483 -483
  244. package/packs/pbgc/policy.json +94 -94
  245. package/packs/philfed/endpoints.json +85 -85
  246. package/packs/philfed/policy.json +94 -94
  247. package/packs/port.json +65 -0
  248. package/packs/regulations/endpoints.json +287 -287
  249. package/packs/regulations/policy.json +86 -86
  250. package/packs/sba/endpoints.json +34 -0
  251. package/packs/sba/policy.json +86 -0
  252. package/packs/sba/source.md +60 -0
  253. package/packs/sec/endpoints.json +236 -236
  254. package/packs/sec/policy.json +86 -86
  255. package/packs/senate-lobbying/endpoints.json +553 -553
  256. package/packs/senate-lobbying/policy.json +99 -99
  257. package/packs/snapqc/endpoints.json +1426 -1426
  258. package/packs/snapqc/policy.json +92 -92
  259. package/packs/treasury/catalog.json +2267 -2267
  260. package/packs/treasury/docs/api_guide.md +245 -245
  261. package/packs/treasury/docs/datasets.md +56 -56
  262. package/packs/treasury/docs/report_patterns.md +126 -126
  263. package/packs/treasury/policy.json +159 -159
  264. package/packs/treasury/source.md +170 -165
  265. package/packs/treasury-tic/endpoints.json +77 -0
  266. package/packs/treasury-tic/policy.json +89 -0
  267. package/packs/treasury-tic/source.md +80 -0
  268. package/packs/treasury-yields/endpoints.json +226 -226
  269. package/packs/treasury-yields/policy.json +86 -86
  270. package/packs/usaspending/endpoints.json +1315 -1315
  271. package/packs/usaspending/policy.json +109 -109
  272. package/packs/uscourts/endpoints.json +22 -0
  273. package/packs/uscourts/policy.json +87 -0
  274. package/packs/uscourts/source.md +56 -0
  275. package/packs/usda-fna/policy.json +133 -133
  276. package/packs/usda-fooddata/endpoints.json +180 -180
  277. package/packs/usda-fooddata/policy.json +94 -94
  278. package/packs/usda-nass/endpoints.json +323 -323
  279. package/packs/usda-nass/policy.json +95 -95
  280. package/packs/usda-outlook/endpoints.json +86 -86
  281. package/packs/usda-outlook/policy.json +109 -109
  282. package/packs/usgs/endpoints.json +313 -313
  283. package/packs/usgs/policy.json +95 -95
  284. package/packs/usitc/endpoints.json +74 -74
  285. package/packs/usitc/policy.json +111 -111
  286. package/packs/uspto/endpoints.json +141 -141
  287. package/packs/uspto/policy.json +95 -95
  288. package/packs/uspto/source.md +1 -1
  289. package/packs/va/endpoints.json +414 -414
  290. package/packs/va/policy.json +101 -101
  291. package/packs/world-bank/endpoints.json +122 -122
  292. package/packs/world-bank/policy.json +101 -101
@@ -1,107 +1,107 @@
1
- {
2
- "id": "dhs-ohss",
3
- "title": "DHS Office of Homeland Security Statistics: Yearbook of Immigration Statistics",
4
- "agency": "U.S. Department of Homeland Security, Office of Homeland Security Statistics",
5
- "depth": "standard",
6
- "domains": [
7
- "immigration",
8
- "demographics"
9
- ],
10
- "keywords": [
11
- "immigration statistics",
12
- "green card",
13
- "green cards",
14
- "lawful permanent resident",
15
- "lawful permanent residents",
16
- "permanent residency",
17
- "immigrants",
18
- "adjustment of status",
19
- "naturalization",
20
- "naturalizations",
21
- "new citizens",
22
- "refugees",
23
- "refugee arrivals",
24
- "asylum",
25
- "asylees",
26
- "nonimmigrant admissions",
27
- "temporary visitors",
28
- "i-94",
29
- "h-1b",
30
- "student visas",
31
- "deportations",
32
- "removals",
33
- "apprehensions",
34
- "immigration enforcement",
35
- "yearbook of immigration statistics",
36
- "dhs",
37
- "uscis"
38
- ],
39
- "questions": [
40
- "how many green cards were issued in fiscal year 2024",
41
- "lawful permanent residents by country of birth over the last decade",
42
- "family-sponsored vs employment-based green cards",
43
- "naturalizations by state",
44
- "refugee arrivals by country of nationality",
45
- "asylum grants by country",
46
- "H-1B and student nonimmigrant admissions",
47
- "removals and returns by country of nationality",
48
- "green cards so far this fiscal year by quarter"
49
- ],
50
- "homepage": "https://ohss.dhs.gov/topics/immigration/yearbook",
51
- "terms_url": "https://ohss.dhs.gov/about-our-data",
52
- "auth": {
53
- "env": null,
54
- "required": "none",
55
- "placement": "none",
56
- "param": null,
57
- "prefix": "",
58
- "home": null,
59
- "signup_url": null,
60
- "echoes_key": false,
61
- "key_only": [],
62
- "extra": {},
63
- "demo_key": null
64
- },
65
- "limits": {
66
- "rps": 1,
67
- "burst": 3,
68
- "daily_quota": null,
69
- "errors_per_min": null,
70
- "bytes_per_min": null,
71
- "max_rows": 50000
72
- },
73
- "routes": {
74
- "api": {
75
- "base": "https://ohss.dhs.gov",
76
- "probe": "https://ohss.dhs.gov/topics/immigration/yearbook",
77
- "hosts": [
78
- "ohss.dhs.gov"
79
- ]
80
- },
81
- "manual": {
82
- "how": "Open https://ohss.dhs.gov/topics/immigration/yearbook, choose the fiscal year, and download the topic's data tables workbook (e.g. \"Lawful Permanent Residents 2024 Data Tables (xlsx)\"; FY2015-2021 are .zip files); for the quarterly report open https://ohss.dhs.gov/topics/immigration/legal-immigration-and-adjustment-status-report and download the fiscal year's .xlsx. Upload the file unchanged under the name shown in INSTRUCTIONS.md: the pack reads the requested sheet from it."
83
- }
84
- },
85
- "cache": {
86
- "mode": "run_only",
87
- "ttl_hours": null,
88
- "why": "Small published workbooks that are revised and re-posted under new names; data/ is reused only within a run."
89
- },
90
- "redistribution": {
91
- "mode": "allowed",
92
- "why": "Statistics published by a U.S. federal agency are U.S. government works in the public domain (17 U.S.C. 105)."
93
- },
94
- "notices": [],
95
- "retention": null,
96
- "citation": "U.S. Department of Homeland Security, Office of Homeland Security Statistics, {dataset}, ohss.dhs.gov, retrieved {retrieved_long}.",
97
- "audit_hooks": [],
98
- "clarify": [
99
- "Which fiscal year? DHS counts by federal fiscal year (October to September), not calendar year; the newest yearbook is the default, and the quarterly report covers the current year with preliminary figures.",
100
- "All persons obtaining LPR status, or only new arrivals from abroad (or only adjustments of status inside the U.S.)? The yearbook total counts both.",
101
- "By country of birth, country of last residence, or (for refugees, asylees and the quarterly report) country of nationality? They are different tables with different counts."
102
- ],
103
- "formats": [
104
- "xlsx",
105
- "zip"
106
- ]
107
- }
1
+ {
2
+ "id": "dhs-ohss",
3
+ "title": "DHS Office of Homeland Security Statistics: Yearbook of Immigration Statistics",
4
+ "agency": "U.S. Department of Homeland Security, Office of Homeland Security Statistics",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "immigration",
8
+ "demographics"
9
+ ],
10
+ "keywords": [
11
+ "immigration statistics",
12
+ "green card",
13
+ "green cards",
14
+ "lawful permanent resident",
15
+ "lawful permanent residents",
16
+ "permanent residency",
17
+ "immigrants",
18
+ "adjustment of status",
19
+ "naturalization",
20
+ "naturalizations",
21
+ "new citizens",
22
+ "refugees",
23
+ "refugee arrivals",
24
+ "asylum",
25
+ "asylees",
26
+ "nonimmigrant admissions",
27
+ "temporary visitors",
28
+ "i-94",
29
+ "h-1b",
30
+ "student visas",
31
+ "deportations",
32
+ "removals",
33
+ "apprehensions",
34
+ "immigration enforcement",
35
+ "yearbook of immigration statistics",
36
+ "dhs",
37
+ "uscis"
38
+ ],
39
+ "questions": [
40
+ "how many green cards were issued in fiscal year 2024",
41
+ "lawful permanent residents by country of birth over the last decade",
42
+ "family-sponsored vs employment-based green cards",
43
+ "naturalizations by state",
44
+ "refugee arrivals by country of nationality",
45
+ "asylum grants by country",
46
+ "H-1B and student nonimmigrant admissions",
47
+ "removals and returns by country of nationality",
48
+ "green cards so far this fiscal year by quarter"
49
+ ],
50
+ "homepage": "https://ohss.dhs.gov/topics/immigration/yearbook",
51
+ "terms_url": "https://ohss.dhs.gov/about-our-data",
52
+ "auth": {
53
+ "env": null,
54
+ "required": "none",
55
+ "placement": "none",
56
+ "param": null,
57
+ "prefix": "",
58
+ "home": null,
59
+ "signup_url": null,
60
+ "echoes_key": false,
61
+ "key_only": [],
62
+ "extra": {},
63
+ "demo_key": null
64
+ },
65
+ "limits": {
66
+ "rps": 1,
67
+ "burst": 3,
68
+ "daily_quota": null,
69
+ "errors_per_min": null,
70
+ "bytes_per_min": null,
71
+ "max_rows": 50000
72
+ },
73
+ "routes": {
74
+ "api": {
75
+ "base": "https://ohss.dhs.gov",
76
+ "probe": "https://ohss.dhs.gov/topics/immigration/yearbook",
77
+ "hosts": [
78
+ "ohss.dhs.gov"
79
+ ]
80
+ },
81
+ "manual": {
82
+ "how": "Open https://ohss.dhs.gov/topics/immigration/yearbook, choose the fiscal year, and download the topic's data tables workbook (e.g. \"Lawful Permanent Residents 2024 Data Tables (xlsx)\"; FY2015-2021 are .zip files); for the quarterly report open https://ohss.dhs.gov/topics/immigration/legal-immigration-and-adjustment-status-report and download the fiscal year's .xlsx. Upload the file unchanged under the name shown in INSTRUCTIONS.md: the pack reads the requested sheet from it."
83
+ }
84
+ },
85
+ "cache": {
86
+ "mode": "run_only",
87
+ "ttl_hours": null,
88
+ "why": "Small published workbooks that are revised and re-posted under new names; data/ is reused only within a run."
89
+ },
90
+ "redistribution": {
91
+ "mode": "allowed",
92
+ "why": "Statistics published by a U.S. federal agency are U.S. government works in the public domain (17 U.S.C. 105)."
93
+ },
94
+ "notices": [],
95
+ "retention": null,
96
+ "citation": "U.S. Department of Homeland Security, Office of Homeland Security Statistics, {dataset}, ohss.dhs.gov, retrieved {retrieved_long}.",
97
+ "audit_hooks": [],
98
+ "clarify": [
99
+ "Which fiscal year? DHS counts by federal fiscal year (October to September), not calendar year; the newest yearbook is the default, and the quarterly report covers the current year with preliminary figures.",
100
+ "All persons obtaining LPR status, or only new arrivals from abroad (or only adjustments of status inside the U.S.)? The yearbook total counts both.",
101
+ "By country of birth, country of last residence, or (for refugees, asylees and the quarterly report) country of nationality? They are different tables with different counts."
102
+ ],
103
+ "formats": [
104
+ "xlsx",
105
+ "zip"
106
+ ]
107
+ }
@@ -13,7 +13,7 @@ from the workbooks posted on ohss.dhs.gov, plus the quarterly Legal Immigration
13
13
  - Not here: the foreign-born population or immigrants' characteristics from surveys (census ACS); visa issuance
14
14
  (State Department, not covered); immigration court cases (EOIR, not covered); monthly enforcement statistics: the
15
15
  OHSS "Immigration Enforcement and Legal Processes Monthly Tables" stop at November 2024 and the page says the
16
- report "is delayed while it is under review" (verified 2026-10-30); this pack does not read them.
16
+ report "is delayed while it is under review" (verified 2026-10-30); this pack does not read them. Monthly border encounters are in cbp.
17
17
 
18
18
  ## Endpoints and datasets
19
19
  | Endpoint | Returns | Key parameters |
@@ -1,59 +1,59 @@
1
- {
2
- "base_url": "https://www.justice.gov/api/v1",
3
- "endpoints": {
4
- "press_releases": {
5
- "path": "/press_releases.json",
6
- "description": "DOJ press releases (Office of Public Affairs and all 94 U.S. Attorney's Offices, FBI, DEA, ATF, ...): title, date, component, topic, teaser, HTML body, URL.",
7
- "params": {
8
- "parameters[title]": {"desc": "title words: \"fentanyl trafficking\" in double quotes = phrase; without quotes = all words; case-insensitive"},
9
- "parameters[date]": {"desc": "one day: YYYY-MM-DD or a Unix timestamp (no ranges; use since/until)"},
10
- "since": {"desc": "pack option, not sent: keep releases dated on or after YYYY-MM-DD (pages newest first and stops early)"},
11
- "until": {"desc": "pack option, not sent: keep releases dated on or before YYYY-MM-DD"},
12
- "fields": {"desc": "comma-separated: uuid,title,date,created,changed,url,component,topic,teaser,body,number,attachment,image"},
13
- "sort": {"enum": ["date", "created", "changed"], "desc": "sort field (default order is not by date: set it)"},
14
- "direction": {"enum": ["ASC", "DESC"], "desc": "DESC = newest first"}
15
- },
16
- "defaults": {"sort": "date", "direction": "DESC", "fields": "uuid,title,date,component,topic,url,number,teaser"},
17
- "records": "results", "total": "metadata.resultset.count",
18
- "paging": {"type": "page", "page_param": "page", "size_param": "pagesize", "size": 50, "start": 0},
19
- "max_rows": 20000
20
- },
21
- "press_release": {
22
- "path": "/press_releases/{uuid}.json",
23
- "description": "One press release with its full HTML body.",
24
- "params": {"uuid": {"in": "path", "required": true, "desc": "uuid from press_releases"},
25
- "fields": {"desc": "fields to return (default all)"}},
26
- "records": "results", "paging": {"type": "none"}
27
- },
28
- "blog_entries": {
29
- "path": "/blog_entries.json",
30
- "description": "DOJ blog entries (Justice Blog, about 3,300 since 2009). No title or date filter upstream; since/until filter locally.",
31
- "params": {
32
- "since": {"desc": "pack option, not sent: keep entries dated on or after YYYY-MM-DD"},
33
- "until": {"desc": "pack option, not sent: keep entries dated on or before YYYY-MM-DD"},
34
- "fields": {"desc": "uuid,title,date,created,changed,url,component,topic,teaser,body,image,attachments"},
35
- "sort": {"enum": ["date", "created", "changed"], "desc": "sort field"},
36
- "direction": {"enum": ["ASC", "DESC"], "desc": "DESC = newest first"}
37
- },
38
- "defaults": {"sort": "date", "direction": "DESC", "fields": "uuid,title,date,component,topic,url,teaser"},
39
- "records": "results", "total": "metadata.resultset.count",
40
- "paging": {"type": "page", "page_param": "page", "size_param": "pagesize", "size": 50, "start": 0},
41
- "max_rows": 5000
42
- },
43
- "blog_entry": {
44
- "path": "/blog_entries/{uuid}.json",
45
- "description": "One blog entry with its full body.",
46
- "params": {"uuid": {"in": "path", "required": true, "desc": "uuid from blog_entries"},
47
- "fields": {"desc": "fields to return (default all)"}},
48
- "records": "results", "paging": {"type": "none"}
49
- }
50
- },
51
- "catalog": {
52
- "components": {"id": "component", "name": "DOJ components",
53
- "description": "Office of the Attorney General, Office of Public Affairs, Civil Rights Division, Civil Division, Criminal Division, National Security Division, Antitrust Division, Environment and Natural Resources Division, Tax Division, DEA, FBI, ATF, U.S. Marshals Service, Bureau of Prisons, Office of Justice Programs, U.S. Attorneys (USAO - <state, district>)",
54
- "endpoint": "press_releases"},
55
- "topics": {"id": "topic", "name": "Press release topics",
56
- "description": "Antitrust, Civil Rights, Cybercrime, Disability Rights, Drug Trafficking, Elder Justice, Environment, Firearms Offenses, Financial Fraud, Foreign Agents Registration Act, Health Care Fraud, Human Trafficking, Identity Theft, Immigration, National Security, Opioids, Project Safe Childhood, Public Corruption, Tax, Terrorism, Violent Crime, Voting, White Collar Crime",
57
- "endpoint": "press_releases"}
58
- }
59
- }
1
+ {
2
+ "base_url": "https://www.justice.gov/api/v1",
3
+ "endpoints": {
4
+ "press_releases": {
5
+ "path": "/press_releases.json",
6
+ "description": "DOJ press releases (Office of Public Affairs and all 94 U.S. Attorney's Offices, FBI, DEA, ATF, ...): title, date, component, topic, teaser, HTML body, URL.",
7
+ "params": {
8
+ "parameters[title]": {"desc": "title words: \"fentanyl trafficking\" in double quotes = phrase; without quotes = all words; case-insensitive"},
9
+ "parameters[date]": {"desc": "one day: YYYY-MM-DD or a Unix timestamp (no ranges; use since/until)"},
10
+ "since": {"desc": "pack option, not sent: keep releases dated on or after YYYY-MM-DD (pages newest first and stops early)"},
11
+ "until": {"desc": "pack option, not sent: keep releases dated on or before YYYY-MM-DD"},
12
+ "fields": {"desc": "comma-separated: uuid,title,date,created,changed,url,component,topic,teaser,body,number,attachment,image"},
13
+ "sort": {"enum": ["date", "created", "changed"], "desc": "sort field (default order is not by date: set it)"},
14
+ "direction": {"enum": ["ASC", "DESC"], "desc": "DESC = newest first"}
15
+ },
16
+ "defaults": {"sort": "date", "direction": "DESC", "fields": "uuid,title,date,component,topic,url,number,teaser"},
17
+ "records": "results", "total": "metadata.resultset.count",
18
+ "paging": {"type": "page", "page_param": "page", "size_param": "pagesize", "size": 50, "start": 0},
19
+ "max_rows": 20000
20
+ },
21
+ "press_release": {
22
+ "path": "/press_releases/{uuid}.json",
23
+ "description": "One press release with its full HTML body.",
24
+ "params": {"uuid": {"in": "path", "required": true, "desc": "uuid from press_releases"},
25
+ "fields": {"desc": "fields to return (default all)"}},
26
+ "records": "results", "paging": {"type": "none"}
27
+ },
28
+ "blog_entries": {
29
+ "path": "/blog_entries.json",
30
+ "description": "DOJ blog entries (Justice Blog, about 3,300 since 2009). No title or date filter upstream; since/until filter locally.",
31
+ "params": {
32
+ "since": {"desc": "pack option, not sent: keep entries dated on or after YYYY-MM-DD"},
33
+ "until": {"desc": "pack option, not sent: keep entries dated on or before YYYY-MM-DD"},
34
+ "fields": {"desc": "uuid,title,date,created,changed,url,component,topic,teaser,body,image,attachments"},
35
+ "sort": {"enum": ["date", "created", "changed"], "desc": "sort field"},
36
+ "direction": {"enum": ["ASC", "DESC"], "desc": "DESC = newest first"}
37
+ },
38
+ "defaults": {"sort": "date", "direction": "DESC", "fields": "uuid,title,date,component,topic,url,teaser"},
39
+ "records": "results", "total": "metadata.resultset.count",
40
+ "paging": {"type": "page", "page_param": "page", "size_param": "pagesize", "size": 50, "start": 0},
41
+ "max_rows": 5000
42
+ },
43
+ "blog_entry": {
44
+ "path": "/blog_entries/{uuid}.json",
45
+ "description": "One blog entry with its full body.",
46
+ "params": {"uuid": {"in": "path", "required": true, "desc": "uuid from blog_entries"},
47
+ "fields": {"desc": "fields to return (default all)"}},
48
+ "records": "results", "paging": {"type": "none"}
49
+ }
50
+ },
51
+ "catalog": {
52
+ "components": {"id": "component", "name": "DOJ components",
53
+ "description": "Office of the Attorney General, Office of Public Affairs, Civil Rights Division, Civil Division, Criminal Division, National Security Division, Antitrust Division, Environment and Natural Resources Division, Tax Division, DEA, FBI, ATF, U.S. Marshals Service, Bureau of Prisons, Office of Justice Programs, U.S. Attorneys (USAO - <state, district>)",
54
+ "endpoint": "press_releases"},
55
+ "topics": {"id": "topic", "name": "Press release topics",
56
+ "description": "Antitrust, Civil Rights, Cybercrime, Disability Rights, Drug Trafficking, Elder Justice, Environment, Firearms Offenses, Financial Fraud, Foreign Agents Registration Act, Health Care Fraud, Human Trafficking, Identity Theft, Immigration, National Security, Opioids, Project Safe Childhood, Public Corruption, Tax, Terrorism, Violent Crime, Voting, White Collar Crime",
57
+ "endpoint": "press_releases"}
58
+ }
59
+ }
@@ -1,96 +1,96 @@
1
- {
2
- "id": "doj-news",
3
- "title": "Department of Justice News (justice.gov press releases and blog)",
4
- "agency": "U.S. Department of Justice",
5
- "depth": "standard",
6
- "domains": [
7
- "justice",
8
- "law enforcement",
9
- "news releases"
10
- ],
11
- "keywords": [
12
- "doj",
13
- "justice department",
14
- "press release",
15
- "press releases",
16
- "indictment",
17
- "indicted",
18
- "sentenced",
19
- "guilty plea",
20
- "settlement",
21
- "prosecution",
22
- "enforcement action",
23
- "u.s. attorney",
24
- "usao",
25
- "antitrust",
26
- "fraud",
27
- "health care fraud",
28
- "fentanyl",
29
- "civil rights division",
30
- "doj blog"
31
- ],
32
- "questions": [
33
- "DOJ press releases about fentanyl trafficking in 2025",
34
- "recent antitrust enforcement announcements",
35
- "how often U.S. Attorney offices announced health care fraud cases",
36
- "full text of a DOJ press release"
37
- ],
38
- "homepage": "https://www.justice.gov/news",
39
- "terms_url": "https://www.justice.gov/legalpolicies",
40
- "auth": {
41
- "env": null,
42
- "required": "none",
43
- "placement": "none",
44
- "signup_url": null,
45
- "echoes_key": false
46
- },
47
- "limits": {
48
- "rps": 3,
49
- "burst": 4,
50
- "max_rows": 20000
51
- },
52
- "routes": {
53
- "api": {
54
- "base": "https://www.justice.gov/api/v1",
55
- "probe": "https://www.justice.gov/api/v1/blog_entries.json?pagesize=1&fields=uuid",
56
- "hosts": [
57
- "www.justice.gov"
58
- ]
59
- },
60
- "manual": {
61
- "how": "Open each link in manual_download/INSTRUCTIONS.md in a browser (JSON, 50 records per page; save under the name shown) and upload the files. Without the links: search https://www.justice.gov/news (filters for type, component, topic and date), then copy the titles, dates and URLs into a CSV with columns title,date_iso,component_names,url and upload it."
62
- }
63
- },
64
- "cache": {
65
- "mode": "run_only",
66
- "ttl_hours": null,
67
- "why": "New releases appear daily; data/ is reused only within a run."
68
- },
69
- "redistribution": {
70
- "mode": "allowed",
71
- "why": "Press releases are U.S. government works in the public domain (17 U.S.C. 105; justice.gov legal policies, https://www.justice.gov/legalpolicies); photos or quoted third-party material in a release may not be."
72
- },
73
- "notices": [],
74
- "retention": null,
75
- "citation": "U.S. Department of Justice, {dataset}, justice.gov News API (api/v1), retrieved {retrieved_long}.",
76
- "audit_hooks": [
77
- {
78
- "id": "doj_text_not_statistics",
79
- "kind": "require_text",
80
- "any": [
81
- "press release",
82
- "announcement",
83
- "not official statistics",
84
- "not statistics"
85
- ],
86
- "why": "DOJ News is a set of announcements, not a statistical series; a report that counts releases must say it counts announcements."
87
- }
88
- ],
89
- "clarify": [
90
- "Which period and which title words? The API filters by title words and by single day only; a range is filtered after download.",
91
- "Press releases or blog entries? Releases cover enforcement actions; the blog covers policy and programs."
92
- ],
93
- "formats": [
94
- "json"
95
- ]
96
- }
1
+ {
2
+ "id": "doj-news",
3
+ "title": "Department of Justice News (justice.gov press releases and blog)",
4
+ "agency": "U.S. Department of Justice",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "justice",
8
+ "law enforcement",
9
+ "news releases"
10
+ ],
11
+ "keywords": [
12
+ "doj",
13
+ "justice department",
14
+ "press release",
15
+ "press releases",
16
+ "indictment",
17
+ "indicted",
18
+ "sentenced",
19
+ "guilty plea",
20
+ "settlement",
21
+ "prosecution",
22
+ "enforcement action",
23
+ "u.s. attorney",
24
+ "usao",
25
+ "antitrust",
26
+ "fraud",
27
+ "health care fraud",
28
+ "fentanyl",
29
+ "civil rights division",
30
+ "doj blog"
31
+ ],
32
+ "questions": [
33
+ "DOJ press releases about fentanyl trafficking in 2025",
34
+ "recent antitrust enforcement announcements",
35
+ "how often U.S. Attorney offices announced health care fraud cases",
36
+ "full text of a DOJ press release"
37
+ ],
38
+ "homepage": "https://www.justice.gov/news",
39
+ "terms_url": "https://www.justice.gov/legalpolicies",
40
+ "auth": {
41
+ "env": null,
42
+ "required": "none",
43
+ "placement": "none",
44
+ "signup_url": null,
45
+ "echoes_key": false
46
+ },
47
+ "limits": {
48
+ "rps": 3,
49
+ "burst": 4,
50
+ "max_rows": 20000
51
+ },
52
+ "routes": {
53
+ "api": {
54
+ "base": "https://www.justice.gov/api/v1",
55
+ "probe": "https://www.justice.gov/api/v1/blog_entries.json?pagesize=1&fields=uuid",
56
+ "hosts": [
57
+ "www.justice.gov"
58
+ ]
59
+ },
60
+ "manual": {
61
+ "how": "Open each link in manual_download/INSTRUCTIONS.md in a browser (JSON, 50 records per page; save under the name shown) and upload the files. Without the links: search https://www.justice.gov/news (filters for type, component, topic and date), then copy the titles, dates and URLs into a CSV with columns title,date_iso,component_names,url and upload it."
62
+ }
63
+ },
64
+ "cache": {
65
+ "mode": "run_only",
66
+ "ttl_hours": null,
67
+ "why": "New releases appear daily; data/ is reused only within a run."
68
+ },
69
+ "redistribution": {
70
+ "mode": "allowed",
71
+ "why": "Press releases are U.S. government works in the public domain (17 U.S.C. 105; justice.gov legal policies, https://www.justice.gov/legalpolicies); photos or quoted third-party material in a release may not be."
72
+ },
73
+ "notices": [],
74
+ "retention": null,
75
+ "citation": "U.S. Department of Justice, {dataset}, justice.gov News API (api/v1), retrieved {retrieved_long}.",
76
+ "audit_hooks": [
77
+ {
78
+ "id": "doj_text_not_statistics",
79
+ "kind": "require_text",
80
+ "any": [
81
+ "press release",
82
+ "announcement",
83
+ "not official statistics",
84
+ "not statistics"
85
+ ],
86
+ "why": "DOJ News is a set of announcements, not a statistical series; a report that counts releases must say it counts announcements."
87
+ }
88
+ ],
89
+ "clarify": [
90
+ "Which period and which title words? The API filters by title words and by single day only; a range is filtered after download.",
91
+ "Press releases or blog entries? Releases cover enforcement actions; the blog covers policy and programs."
92
+ ],
93
+ "formats": [
94
+ "json"
95
+ ]
96
+ }
@@ -9,7 +9,7 @@ the 94 U.S. Attorney's Offices, FBI, DEA, ATF and other components) and Justice
9
9
  - **This is text, not statistics.** A count of press releases is a count of announcements, which depend
10
10
  on press office practice, not a measure of prosecutions, convictions or crime. Say so whenever releases
11
11
  are counted. For case statistics use the U.S. Sentencing Commission or EOUSA caseload reports (not
12
- covered here); for crime counts use the `fbi` pack; for court filings, `courtlistener` if present.
12
+ covered here); for crime counts use the `fbi` pack; for federal court caseload statistics, `uscourts`; for victimization and prisoners, `bjs`.
13
13
  - Not here: speeches and congressional testimony (no API endpoint; `/api/v1/speeches.json` answers 404).
14
14
 
15
15
  ## Endpoints and datasets