data-analyst-mcp 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/README.md +28 -4
  2. package/dist/adapters/bjs.d.ts +13 -0
  3. package/dist/adapters/bjs.js +28 -0
  4. package/dist/adapters/bjs.js.map +1 -0
  5. package/dist/adapters/cbp.d.ts +5 -0
  6. package/dist/adapters/cbp.js +64 -0
  7. package/dist/adapters/cbp.js.map +1 -0
  8. package/dist/adapters/cdc-wonder.d.ts +5 -0
  9. package/dist/adapters/cdc-wonder.js +68 -0
  10. package/dist/adapters/cdc-wonder.js.map +1 -0
  11. package/dist/adapters/cpsc.d.ts +4 -0
  12. package/dist/adapters/cpsc.js +125 -0
  13. package/dist/adapters/cpsc.js.map +1 -0
  14. package/dist/adapters/fhwa.d.ts +9 -0
  15. package/dist/adapters/fhwa.js +16 -0
  16. package/dist/adapters/fhwa.js.map +1 -0
  17. package/dist/adapters/foreign-assistance.d.ts +9 -0
  18. package/dist/adapters/foreign-assistance.js +42 -0
  19. package/dist/adapters/foreign-assistance.js.map +1 -0
  20. package/dist/adapters/hud-ahar.d.ts +3 -0
  21. package/dist/adapters/hud-ahar.js +80 -0
  22. package/dist/adapters/hud-ahar.js.map +1 -0
  23. package/dist/adapters/nces.d.ts +10 -0
  24. package/dist/adapters/nces.js +32 -0
  25. package/dist/adapters/nces.js.map +1 -0
  26. package/dist/adapters/nsf.d.ts +11 -0
  27. package/dist/adapters/nsf.js +25 -0
  28. package/dist/adapters/nsf.js.map +1 -0
  29. package/dist/adapters/onrr.d.ts +9 -0
  30. package/dist/adapters/onrr.js +35 -0
  31. package/dist/adapters/onrr.js.map +1 -0
  32. package/dist/adapters/sba.d.ts +10 -0
  33. package/dist/adapters/sba.js +134 -0
  34. package/dist/adapters/sba.js.map +1 -0
  35. package/dist/adapters/treasury-tic.d.ts +30 -0
  36. package/dist/adapters/treasury-tic.js +149 -0
  37. package/dist/adapters/treasury-tic.js.map +1 -0
  38. package/dist/adapters/uscourts.d.ts +8 -0
  39. package/dist/adapters/uscourts.js +8 -0
  40. package/dist/adapters/uscourts.js.map +1 -0
  41. package/dist/analysis/cite.d.ts +9 -1
  42. package/dist/analysis/cite.js +9 -2
  43. package/dist/analysis/cite.js.map +1 -1
  44. package/dist/bin/cli.d.ts +2 -0
  45. package/dist/bin/cli.js +17 -0
  46. package/dist/bin/cli.js.map +1 -1
  47. package/dist/core/generic-rest.js +42 -17
  48. package/dist/core/generic-rest.js.map +1 -1
  49. package/dist/core/guard.d.ts +32 -0
  50. package/dist/core/guard.js +176 -0
  51. package/dist/core/guard.js.map +1 -0
  52. package/dist/core/http.d.ts +10 -0
  53. package/dist/core/http.js +36 -7
  54. package/dist/core/http.js.map +1 -1
  55. package/dist/core/keys.d.ts +13 -3
  56. package/dist/core/keys.js +67 -7
  57. package/dist/core/keys.js.map +1 -1
  58. package/dist/core/package.d.ts +12 -3
  59. package/dist/core/package.js +56 -12
  60. package/dist/core/package.js.map +1 -1
  61. package/dist/core/secret_patterns.d.ts +6 -0
  62. package/dist/core/secret_patterns.js +20 -0
  63. package/dist/core/secret_patterns.js.map +1 -0
  64. package/dist/core/tabular.d.ts +43 -1
  65. package/dist/core/tabular.js +149 -6
  66. package/dist/core/tabular.js.map +1 -1
  67. package/dist/envelope.js +3 -1
  68. package/dist/envelope.js.map +1 -1
  69. package/dist/index.d.ts +29 -22
  70. package/dist/index.js +45 -46
  71. package/dist/index.js.map +1 -1
  72. package/dist/remote/app.js +87 -31
  73. package/dist/remote/app.js.map +1 -1
  74. package/dist/remote/oauth.js +8 -1
  75. package/dist/remote/oauth.js.map +1 -1
  76. package/dist/remote/pages.d.ts +3 -7
  77. package/dist/remote/pages.js +49 -19
  78. package/dist/remote/pages.js.map +1 -1
  79. package/dist/remote/sign.d.ts +7 -1
  80. package/dist/remote/sign.js +33 -7
  81. package/dist/remote/sign.js.map +1 -1
  82. package/dist/remote/store.js +14 -5
  83. package/dist/remote/store.js.map +1 -1
  84. package/dist/report/audit_report.js +29 -7
  85. package/dist/report/audit_report.js.map +1 -1
  86. package/dist/report/build_report.js +34 -4
  87. package/dist/report/build_report.js.map +1 -1
  88. package/dist/report/method_audit.js +3 -0
  89. package/dist/report/method_audit.js.map +1 -1
  90. package/dist/report/scan_secrets.js +2 -13
  91. package/dist/report/scan_secrets.js.map +1 -1
  92. package/dist/server.d.ts +3 -1
  93. package/dist/server.js +44 -12
  94. package/dist/server.js.map +1 -1
  95. package/dist/tools.js +27 -6
  96. package/dist/tools.js.map +1 -1
  97. package/guide/assets/download_data.ps1 +2 -0
  98. package/guide/assets/download_data.sh +8 -2
  99. package/guide/references/routing.md +30 -3
  100. package/package.json +1 -1
  101. package/packs/bjs/endpoints.json +150 -0
  102. package/packs/bjs/policy.json +114 -0
  103. package/packs/bjs/source.md +98 -0
  104. package/packs/cbp/endpoints.json +31 -0
  105. package/packs/cbp/policy.json +85 -0
  106. package/packs/cbp/source.md +57 -0
  107. package/packs/cdc/source.md +1 -1
  108. package/packs/cdc-wonder/endpoints.json +26 -0
  109. package/packs/cdc-wonder/policy.json +84 -0
  110. package/packs/cdc-wonder/source.md +53 -0
  111. package/packs/college-scorecard/source.md +1 -1
  112. package/packs/cpsc/endpoints.json +91 -0
  113. package/packs/cpsc/policy.json +91 -0
  114. package/packs/cpsc/source.md +76 -0
  115. package/packs/dhs-ohss/source.md +1 -1
  116. package/packs/doj-news/source.md +1 -1
  117. package/packs/fbi/source.md +3 -3
  118. package/packs/fhwa/endpoints.json +48 -0
  119. package/packs/fhwa/policy.json +91 -0
  120. package/packs/fhwa/source.md +59 -0
  121. package/packs/foreign-assistance/endpoints.json +166 -0
  122. package/packs/foreign-assistance/policy.json +80 -0
  123. package/packs/foreign-assistance/source.md +76 -0
  124. package/packs/fsa/source.md +1 -1
  125. package/packs/hud/source.md +1 -1
  126. package/packs/hud-ahar/endpoints.json +92 -0
  127. package/packs/hud-ahar/policy.json +88 -0
  128. package/packs/hud-ahar/source.md +72 -0
  129. package/packs/naep/source.md +1 -1
  130. package/packs/nces/endpoints.json +101 -0
  131. package/packs/nces/policy.json +99 -0
  132. package/packs/nces/source.md +75 -0
  133. package/packs/nhtsa/source.md +1 -1
  134. package/packs/nsf/endpoints.json +33 -0
  135. package/packs/nsf/policy.json +83 -0
  136. package/packs/nsf/source.md +59 -0
  137. package/packs/onrr/endpoints.json +198 -0
  138. package/packs/onrr/policy.json +91 -0
  139. package/packs/onrr/source.md +55 -0
  140. package/packs/port.json +65 -0
  141. package/packs/sba/endpoints.json +34 -0
  142. package/packs/sba/policy.json +86 -0
  143. package/packs/sba/source.md +60 -0
  144. package/packs/treasury/source.md +1 -1
  145. package/packs/treasury-tic/endpoints.json +77 -0
  146. package/packs/treasury-tic/policy.json +89 -0
  147. package/packs/treasury-tic/source.md +80 -0
  148. package/packs/uscourts/endpoints.json +22 -0
  149. package/packs/uscourts/policy.json +87 -0
  150. package/packs/uscourts/source.md +56 -0
  151. package/packs/uspto/source.md +1 -1
@@ -0,0 +1,76 @@
1
+ # Consumer Product Safety Commission: product recalls and NEISS injury data
2
+ Depth: standard. Key: none.
3
+ - Recalls: from the SaferProducts.gov Recall API, one row per recall with products, units, hazards, injuries,
4
+ remedies, firms and countries.
5
+ - NEISS: the National Electronic Injury Surveillance System, a probability sample of hospital emergency
6
+ departments with weights for national estimates of product-related injuries. NEISS is manual only: the user
7
+ downloads the yearly file and uploads it.
8
+
9
+ ## When to use
10
+ - Which consumer products were recalled, when, why (hazard), how many units, the remedy, who made, imported
11
+ or sold them, and where they were made; recalls that mention injuries or deaths.
12
+ - Emergency-department-treated injuries associated with consumer products: national estimates by product,
13
+ age, sex, body part, diagnosis, disposition, location.
14
+ - Not here: vehicle, tire and car-seat recalls (nhtsa); food, drug, device and cosmetic recalls (fda);
15
+ injury deaths from death certificates (cdc).
16
+
17
+ ## Endpoints
18
+ | Endpoint | Returns | Parameters |
19
+ |---|---|---|
20
+ | `recalls` | one row per recall: `recall_number`, `recall_date`, `title`, `products`, `product_types`, `number_of_units`, `hazards`, `injuries`, `remedies`, `remedy_options`, `manufacturers`, `importers`, `distributors`, `retailers`, `manufacturer_countries`, `upcs`, `description`, `url`, `last_publish_date`, `recall_id` | API filters (at least one): `RecallDateStart`, `RecallDateEnd`, `LastPublishDateStart` (YYYY-MM-DD), `RecallNumber`, `ProductName`, `ProductType`, `RecallTitle`, `RecallDescription`, `Manufacturer`, `Retailer`, `Importer`, `Injury`, `Remedy`; filtered here: `hazard`, `country` |
21
+ | `neiss` | the uploaded NEISS year as published: `CPSC_Case_Number`, `Treatment_Date`, `Age`, `Sex`, `Race`, `Other_Race`, `Hispanic`, `Body_Part`, `Body_Part_2`, `Diagnosis`, `Diagnosis_2`, `Other_Diagnosis`, `Other_Diagnosis_2`, `Disposition`, `Location`, `Fire_Involvement`, `Alcohol`, `Drug`, `Product_1`, `Product_2`, `Product_3`, `Narrative`, `Stratum`, `PSU`, `Weight` (2019 on; earlier years lack the `_2`, `Hispanic`, `Alcohol`, `Drug`, `Product_3` columns) | `year` (required), `product` (codes kept in any product column) |
22
+
23
+ ## Spec examples
24
+ ```json
25
+ {"description": "Consumer product recalls announced in 2025 with fire hazards",
26
+ "requests": [{"name": "fire_2025", "endpoint": "recalls",
27
+ "params": {"RecallDateStart": "2025-01-01", "RecallDateEnd": "2025-12-31", "hazard": "fire"}}]}
28
+ ```
29
+ ```json
30
+ {"description": "Crib recalls since 2020",
31
+ "requests": [{"name": "cribs", "endpoint": "recalls", "params": {"ProductName": "crib", "RecallDateStart": "2020-01-01"}}]}
32
+ ```
33
+ ```json
34
+ {"description": "NEISS 2024 cases (manual: manual_bundle, then ingest)",
35
+ "requests": [{"name": "neiss_2024", "endpoint": "neiss", "params": {"year": "2024"}}]}
36
+ ```
37
+
38
+ ## Data format and traps
39
+ - Recall filters (verified 2026-11-07):
40
+ - Text filters match part of the field. With no filter the API returns nothing, so the pack asks for one.
41
+ - An unknown parameter is ignored silently. Only the names above are sent; check the rows match what was
42
+ asked.
43
+ - The API's `Hazard`, `ManufacturerCountry` and `ProductModel` filters match nothing, so use `hazard` and
44
+ `country` here (they filter the returned rows). Narrow by date first.
45
+ - One recall can cover several products. The list fields are joined with ` | `. `number_of_units` is free text
46
+ as published ("About 10,318", "5.9 million", "About 338,800 (In addition, about 36,300 were sold in Canada)"):
47
+ parse it with care, never sum it blindly, and say when Canadian or Mexican units are included.
48
+ - `retailers` is free text ("Sold At: ... from May 2025 through September 2026 for about $20") and sometimes also
49
+ names the seller or importer. `recall_date` is the announcement date. A recall number's first two digits are
50
+ CPSC's fiscal year (27xxx = FY2027, from October 2026).
51
+ - Recalls are announcements, not incidents: a count of recalls measures CPSC and firm actions, not product
52
+ danger.
53
+ - NEISS is a sample: each case has a `Weight`. National estimates are the sum of weights; the number of cases is
54
+ not an estimate. Report the number of cases behind every estimate, and say when an estimate rests on few
55
+ cases. CPSC's NEISS Estimates Query Builder gives the case count, coefficient of variation and 95% confidence
56
+ interval for its own estimates: prefer those for a published-quality figure. Standard errors need the
57
+ `Stratum` and `PSU` design and are not computed here.
58
+ - NEISS counts injuries *associated with* a product, not caused by it: a fall in a bathtub involves the
59
+ bathtub. Products are codes (`Product_1` ... `Product_3`); a case can involve up to three. The codes and the
60
+ values of `Body_Part`, `Diagnosis`, `Disposition` and `Location` are in the NEISS Coding Manual and the data
61
+ dictionary (https://www.cpsc.gov/s3fs-public/DataDictionary042022.xlsx). `Age` 201-223 means 1-23 months. Treatment
62
+ year, not incident year.
63
+ - The `Narrative` is free text from the hospital record. Quote it only as examples, never as counts.
64
+
65
+ ## Access
66
+ No key. `fetch` works for recalls (www.saferproducts.gov). NEISS: www.cpsc.gov answers scripts with HTTP 403, and
67
+ the archived yearly files sit behind a form, so `fetch` refuses `neiss` and points to `manual_bundle`. The user
68
+ opens https://www.cpsc.gov/cgibin/NEISSQuery/home.aspx in a browser, picks a User Affiliation, chooses the year
69
+ under "Archived Annual NEISS Data" with File Format "Tab Delimited Text" (Excel also works), clicks Find Data,
70
+ saves the file and uploads it; then `ingest`.
71
+
72
+ ## Citation and terms
73
+ "U.S. Consumer Product Safety Commission, Recalls (SaferProducts.gov Recall API), retrieved <date>" or "U.S.
74
+ Consumer Product Safety Commission, National Electronic Injury Surveillance System (NEISS), <year> data,
75
+ downloaded <date>." Public domain (17 U.S.C. 105). Estimates computed from NEISS cases are the analyst's own,
76
+ not CPSC's: say so.
@@ -13,7 +13,7 @@ from the workbooks posted on ohss.dhs.gov, plus the quarterly Legal Immigration
13
13
  - Not here: the foreign-born population or immigrants' characteristics from surveys (census ACS); visa issuance
14
14
  (State Department, not covered); immigration court cases (EOIR, not covered); monthly enforcement statistics: the
15
15
  OHSS "Immigration Enforcement and Legal Processes Monthly Tables" stop at November 2024 and the page says the
16
- report "is delayed while it is under review" (verified 2026-10-30); this pack does not read them.
16
+ report "is delayed while it is under review" (verified 2026-10-30); this pack does not read them. Monthly border encounters are in cbp.
17
17
 
18
18
  ## Endpoints and datasets
19
19
  | Endpoint | Returns | Key parameters |
@@ -9,7 +9,7 @@ the 94 U.S. Attorney's Offices, FBI, DEA, ATF and other components) and Justice
9
9
  - **This is text, not statistics.** A count of press releases is a count of announcements, which depend
10
10
  on press office practice, not a measure of prosecutions, convictions or crime. Say so whenever releases
11
11
  are counted. For case statistics use the U.S. Sentencing Commission or EOUSA caseload reports (not
12
- covered here); for crime counts use the `fbi` pack; for court filings, `courtlistener` if present.
12
+ covered here); for crime counts use the `fbi` pack; for federal court caseload statistics, `uscourts`; for victimization and prisoners, `bjs`.
13
13
  - Not here: speeches and congressional testimony (no API endpoint; `/api/v1/speeches.json` answers 404).
14
14
 
15
15
  ## Endpoints and datasets
@@ -6,9 +6,9 @@ data from the FBI Uniform Crime Reporting (UCR) Program, served by the Crime Dat
6
6
  - Offenses and clearances by month (violent, property, homicide, robbery ...), national, state or agency;
7
7
  arrests; NIBRS incident breakdowns (victim, offender, location, weapon); Supplementary Homicide Reports;
8
8
  hate crimes by bias; law enforcement employees; law enforcement suicide data (LESDC).
9
- - Not here: victimization surveys including crimes not reported to police (BJS NCVS, not covered), prisons
10
- and sentencing (BJS / U.S. Sentencing Commission), federal prosecutions and DOJ announcements (doj-news,
11
- which is text), the FBI's annual "Crime in the Nation" estimate tables (published reports).
9
+ - Not here: victimization surveys including crimes not reported to police (bjs: NCVS), prisons,
10
+ jails, probation and parole (bjs), federal court caseloads (uscourts), sentencing (U.S. Sentencing
11
+ Commission, not covered), federal prosecutions and DOJ announcements (doj-news, which is text), the FBI's annual "Crime in the Nation" estimate tables (published reports).
12
12
  - Not wrapped by this pack, though the CDE API has them (verified 2026-10): NIBRS estimates with lower and
13
13
  upper bounds (`/nibrs-estimation/national|region|state/<numeric offense ID>?year=`, years 2021 and 2022
14
14
  only, other years HTTP 400; the offense and state IDs are numeric codes not documented here) and use of
@@ -0,0 +1,48 @@
1
+ {
2
+ "base_url": "https://www.fhwa.dot.gov/policyinformation/statistics",
3
+ "timeout": 90,
4
+ "endpoints": {
5
+ "table": {
6
+ "path": "/{year}/xls/{table}.xlsx",
7
+ "method": "GET",
8
+ "description": "One Highway Statistics table as FHWA publishes it (Excel): travel (VM-2 vehicle-miles by state and functional system), drivers (DL-22 by age and sex), vehicles (MV-1 registrations), roads (HM-20 public road length), motor fuel (MF-1 state fuel taxes) and Highway Trust Fund (FE-9) ... Default: every line numbered (row, col1...) to find the header lines; then header_lines names the columns.",
9
+ "params": {
10
+ "year": {
11
+ "in": "path",
12
+ "required": true,
13
+ "desc": "the Highway Statistics edition year, e.g. 2023 (the data year for most tables)"
14
+ },
15
+ "table": {
16
+ "in": "path",
17
+ "required": true,
18
+ "desc": "table id in lower case without the hyphen: vm2, dl22, mv1, hm20, mf1, fe9 ..."
19
+ },
20
+ "sheet": {
21
+ "in": "records",
22
+ "desc": "sheet name or number (default 1, not counting the hidden CRYSTAL_PERSIST sheet); DL-22 has MALES, FEMALES, TOTAL"
23
+ },
24
+ "header_lines": {
25
+ "in": "records",
26
+ "desc": "the header line numbers, e.g. '5-8' for VM-2"
27
+ },
28
+ "header_row": {
29
+ "in": "records",
30
+ "desc": "one header line instead; 0 (default) returns every line numbered"
31
+ },
32
+ "carry": {
33
+ "in": "records",
34
+ "desc": "which header lines span to the right over blank cells: top (default), all, none; check the names against the numbered lines"
35
+ }
36
+ },
37
+ "defaults": {
38
+ "header_row": 0
39
+ },
40
+ "records": "$xlsx",
41
+ "skip_sheets": "^CRYSTAL_PERSIST$",
42
+ "paging": {
43
+ "type": "none"
44
+ },
45
+ "max_rows": 20000
46
+ }
47
+ }
48
+ }
@@ -0,0 +1,91 @@
1
+ {
2
+ "id": "fhwa",
3
+ "title": "FHWA Highway Statistics: travel, drivers, vehicles, roads, motor fuel and highway finance by state",
4
+ "agency": "U.S. Department of Transportation, Federal Highway Administration",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "transportation",
8
+ "highways",
9
+ "energy"
10
+ ],
11
+ "keywords": [
12
+ "fhwa",
13
+ "federal highway administration",
14
+ "highway statistics",
15
+ "vehicle miles traveled",
16
+ "vmt",
17
+ "miles driven",
18
+ "licensed drivers",
19
+ "drivers by age",
20
+ "registered vehicles",
21
+ "vehicle registrations",
22
+ "public road mileage",
23
+ "road length",
24
+ "lane miles",
25
+ "functional system",
26
+ "gas tax",
27
+ "motor fuel tax",
28
+ "highway trust fund",
29
+ "highway finance"
30
+ ],
31
+ "questions": [
32
+ "vehicle miles traveled by state",
33
+ "miles driven per capita by state",
34
+ "licensed drivers by age and sex",
35
+ "registered vehicles by state",
36
+ "public road miles by state and functional system",
37
+ "state motor fuel tax receipts",
38
+ "Highway Trust Fund receipts attributable to each state"
39
+ ],
40
+ "homepage": "https://www.fhwa.dot.gov/policyinformation/statistics.cfm",
41
+ "terms_url": "https://www.fhwa.dot.gov/policyinformation/statistics.cfm",
42
+ "auth": {
43
+ "env": null,
44
+ "required": "none",
45
+ "placement": "none",
46
+ "signup_url": null,
47
+ "echoes_key": false
48
+ },
49
+ "limits": {
50
+ "rps": 1,
51
+ "burst": 3,
52
+ "max_rows": 20000
53
+ },
54
+ "routes": {
55
+ "api": {
56
+ "base": "https://www.fhwa.dot.gov/policyinformation/statistics",
57
+ "probe": "https://www.fhwa.dot.gov/policyinformation/statistics/2023/xls/vm2.xlsx",
58
+ "hosts": [
59
+ "www.fhwa.dot.gov"
60
+ ]
61
+ },
62
+ "manual": {
63
+ "how": "Open https://www.fhwa.dot.gov/policyinformation/statistics/<year>/<table>.cfm (e.g. 2023/vm2.cfm), click the XLSX link, save the workbook and upload it with ingest."
64
+ }
65
+ },
66
+ "cache": {
67
+ "mode": "run_only",
68
+ "ttl_hours": null,
69
+ "why": "Tables are revised in place during the year; data/ is reused within a run."
70
+ },
71
+ "redistribution": {
72
+ "mode": "allowed",
73
+ "why": "FHWA statistics are U.S. government works in the public domain (17 U.S.C. 105)."
74
+ },
75
+ "notices": [],
76
+ "retention": null,
77
+ "citation": "U.S. Department of Transportation, Federal Highway Administration, Highway Statistics {dataset}, retrieved {retrieved_long}.",
78
+ "audit_hooks": [],
79
+ "clarify": [
80
+ "Which edition and year? Highway Statistics <year> tables mostly hold that year's data (VM-2 2023 = travel in 2023); tables are revised in later editions. Default: the latest edition that has the table.",
81
+ "Which table? Travel VM-2 (by functional system), drivers DL-22, vehicles MV-1, road length HM-20, motor fuel MF-1, trust fund FE-9. Default: the table whose title matches the question, named in the answer."
82
+ ],
83
+ "formats": [
84
+ "xlsx"
85
+ ],
86
+ "aliases": [
87
+ "FHWA",
88
+ "Highway Statistics",
89
+ "Federal Highway Administration"
90
+ ]
91
+ }
@@ -0,0 +1,59 @@
1
+ # FHWA Highway Statistics: travel, drivers, vehicles, roads, motor fuel and highway finance by state
2
+ Depth: standard. Key: none. The tables of FHWA's annual Highway Statistics series, as the Excel files FHWA
3
+ publishes, read line by line or with a named stacked header. Examples: VM-2 (vehicle-miles of travel by state
4
+ and functional system), DL-22 (licensed drivers by age and sex), MV-1 (registered vehicles), HM-20 (public road
5
+ length), MF-1 (state motor-fuel tax receipts), FE-9 (Highway Trust Fund receipts attributable to each state).
6
+
7
+ ## When to use
8
+ - Driving and roads by state: vehicle-miles traveled, per capita (with census population), licensed drivers,
9
+ registered vehicles, road miles, fuel tax receipts, Highway Trust Fund flows.
10
+ - Not here: national monthly travel and transportation indicators (bts); fuel prices and consumption totals
11
+ (eia); traffic deaths and recalls (nhtsa); federal highway grants by recipient (usaspending).
12
+
13
+ ## Endpoints
14
+ | Endpoint | Returns | Parameters |
15
+ |---|---|---|
16
+ | `table` | one Highway Statistics table: without `header_lines`, every line numbered (`row`, `col1`...); with it, named columns | `year` (edition, e.g. `2023`), `table` (`vm2`, `dl22`, `mv1`, `hm20`, `mf1`, `fe9` ...), `sheet`, `header_lines`, `header_row`, `carry` |
17
+
18
+ The table list is at https://www.fhwa.dot.gov/policyinformation/statistics/2023/ (each table page has an XLSX
19
+ link). Verified 2026-11-07: vm2 (2023, 2024), dl22, mv1, hm20, mf1, fe9 (2023). `mf2` is not a 2023 file.
20
+
21
+ ## Spec examples
22
+ ```json
23
+ {"description": "Vehicle-miles traveled by state, 2023 (VM-2)",
24
+ "requests": [{"name": "vm2_2023", "endpoint": "table", "params": {"year": "2023", "table": "vm2", "header_lines": "5-8"}}]}
25
+ ```
26
+ ```json
27
+ {"description": "Licensed drivers by age, 2023 (DL-22, all drivers): first look at the lines",
28
+ "requests": [{"name": "dl22_total", "endpoint": "table", "params": {"year": "2023", "table": "dl22", "sheet": "TOTAL"}}]}
29
+ ```
30
+
31
+ ## Data format and traps
32
+ - Read the raw lines first. Each table has its own layout:
33
+ - VM-2 and HM-20: title lines, the date line, then a four-line header (lines 5-8).
34
+ - MV-1 and MF-1: the header starts on line 4 or 5.
35
+ - DL-22: one sheet per sex (MALES, FEMALES, TOTAL).
36
+ - FE-9: two pages.
37
+ Some workbooks have a hidden CRYSTAL_PERSIST sheet first; it is left out of the sheet numbering.
38
+ - Stacked headers: only the top header line spans right by default (`carry: "top"`), because FHWA centers labels
39
+ over columns without merging them. In VM-2 2023:
40
+ - the RURAL and URBAN spans come out right;
41
+ - column 18, the grand TOTAL of rural and urban, is named `URBAN | TOTAL_col18`; rename it;
42
+ - the columns after it (`URBAN | State Cd`, `URBAN | RINT` ...) are a hidden copy with database codes; drop
43
+ them.
44
+ Always compare names with the numbered lines.
45
+ - Units are in each table's title or date line: VM-2 is in millions of vehicle-miles, MF-1 in thousands of dollars.
46
+ Values are stored numbers as text, often with float noise (`5798.6313300000002`): round to the published
47
+ precision.
48
+ - Rows include state rows, a Total row, and footnote or blank rows below; Puerto Rico and DC appear in some tables.
49
+ Keep state rows by name before ranking. Footnotes are on a separate `footnotes` sheet or below the table.
50
+ - Editions: "Highway Statistics 2023" holds mostly 2023 data; tables are revised in later editions and some are
51
+ published later than others. State the edition and table.
52
+
53
+ ## Access
54
+ No key. `fetch` reads https://www.fhwa.dot.gov/policyinformation/statistics/<year>/xls/<table>.xlsx (verified
55
+ 2026-11-07). Manual: open the table page and click XLSX, then `ingest`.
56
+
57
+ ## Citation and terms
58
+ "U.S. Department of Transportation, Federal Highway Administration, Highway Statistics 2023, Table VM-2,
59
+ retrieved <date>." Public domain (17 U.S.C. 105).
@@ -0,0 +1,166 @@
1
+ {
2
+ "base_url": "https://foreignassistance.gov",
3
+ "timeout": 180,
4
+ "endpoints": {
5
+ "by_country": {
6
+ "path": "/static/us_foreign_aid_country.csv",
7
+ "method": "GET",
8
+ "description": "U.S. foreign assistance by recipient country (and regional or worldwide programs), transaction type and fiscal year, current and constant dollars (ForeignAssistance.gov Data page, 'By Country').",
9
+ "params": {
10
+ "country": {
11
+ "in": "spec",
12
+ "desc": "keep these countries: ISO3 code (UKR) or name as published (Ukraine); a list works"
13
+ },
14
+ "fiscal_year": {
15
+ "in": "spec",
16
+ "desc": "keep these fiscal years (a year or a list)"
17
+ },
18
+ "from": {
19
+ "in": "spec",
20
+ "desc": "first fiscal year to keep"
21
+ },
22
+ "to": {
23
+ "in": "spec",
24
+ "desc": "last fiscal year to keep"
25
+ },
26
+ "transaction_type": {
27
+ "in": "spec",
28
+ "desc": "keep these transaction types: Obligations, Disbursements, Appropriated and Planned, President's Budget Requests (or their ids 2, 3, 1, 18)"
29
+ }
30
+ },
31
+ "records": "$csv",
32
+ "paging": {
33
+ "type": "none"
34
+ },
35
+ "max_rows": 300000
36
+ },
37
+ "by_dac_sector": {
38
+ "path": "/static/us_foreign_aid_dac_sector.csv",
39
+ "method": "GET",
40
+ "description": "U.S. foreign assistance by OECD DAC (international) category and sector, transaction type and fiscal year, all countries together.",
41
+ "params": {
42
+ "country": {
43
+ "in": "spec",
44
+ "desc": "keep these countries: ISO3 code (UKR) or name as published (Ukraine); a list works"
45
+ },
46
+ "fiscal_year": {
47
+ "in": "spec",
48
+ "desc": "keep these fiscal years (a year or a list)"
49
+ },
50
+ "from": {
51
+ "in": "spec",
52
+ "desc": "first fiscal year to keep"
53
+ },
54
+ "to": {
55
+ "in": "spec",
56
+ "desc": "last fiscal year to keep"
57
+ },
58
+ "transaction_type": {
59
+ "in": "spec",
60
+ "desc": "keep these transaction types: Obligations, Disbursements, Appropriated and Planned, President's Budget Requests (or their ids 2, 3, 1, 18)"
61
+ }
62
+ },
63
+ "records": "$csv",
64
+ "paging": {
65
+ "type": "none"
66
+ },
67
+ "max_rows": 300000
68
+ },
69
+ "by_usg_sector": {
70
+ "path": "/static/us_foreign_aid_usg_sector.csv",
71
+ "method": "GET",
72
+ "description": "U.S. foreign assistance by U.S. government category and sector (health, peace and security, humanitarian, economic development ...) and country, by transaction type and fiscal year.",
73
+ "params": {
74
+ "country": {
75
+ "in": "spec",
76
+ "desc": "keep these countries: ISO3 code (UKR) or name as published (Ukraine); a list works"
77
+ },
78
+ "fiscal_year": {
79
+ "in": "spec",
80
+ "desc": "keep these fiscal years (a year or a list)"
81
+ },
82
+ "from": {
83
+ "in": "spec",
84
+ "desc": "first fiscal year to keep"
85
+ },
86
+ "to": {
87
+ "in": "spec",
88
+ "desc": "last fiscal year to keep"
89
+ },
90
+ "transaction_type": {
91
+ "in": "spec",
92
+ "desc": "keep these transaction types: Obligations, Disbursements, Appropriated and Planned, President's Budget Requests (or their ids 2, 3, 1, 18)"
93
+ }
94
+ },
95
+ "records": "$csv",
96
+ "paging": {
97
+ "type": "none"
98
+ },
99
+ "max_rows": 300000
100
+ },
101
+ "by_managing_agency": {
102
+ "path": "/static/us_foreign_aid_implementing.csv",
103
+ "method": "GET",
104
+ "description": "U.S. foreign assistance by managing agency (USAID, State, Defense, HHS ...) and country, by transaction type and fiscal year.",
105
+ "params": {
106
+ "country": {
107
+ "in": "spec",
108
+ "desc": "keep these countries: ISO3 code (UKR) or name as published (Ukraine); a list works"
109
+ },
110
+ "fiscal_year": {
111
+ "in": "spec",
112
+ "desc": "keep these fiscal years (a year or a list)"
113
+ },
114
+ "from": {
115
+ "in": "spec",
116
+ "desc": "first fiscal year to keep"
117
+ },
118
+ "to": {
119
+ "in": "spec",
120
+ "desc": "last fiscal year to keep"
121
+ },
122
+ "transaction_type": {
123
+ "in": "spec",
124
+ "desc": "keep these transaction types: Obligations, Disbursements, Appropriated and Planned, President's Budget Requests (or their ids 2, 3, 1, 18)"
125
+ }
126
+ },
127
+ "records": "$csv",
128
+ "paging": {
129
+ "type": "none"
130
+ },
131
+ "max_rows": 300000
132
+ },
133
+ "by_funding_agency": {
134
+ "path": "/static/us_foreign_aid_funding.csv",
135
+ "method": "GET",
136
+ "description": "U.S. foreign assistance by funding agency and funding account (appropriation account) and country, by transaction type and fiscal year.",
137
+ "params": {
138
+ "country": {
139
+ "in": "spec",
140
+ "desc": "keep these countries: ISO3 code (UKR) or name as published (Ukraine); a list works"
141
+ },
142
+ "fiscal_year": {
143
+ "in": "spec",
144
+ "desc": "keep these fiscal years (a year or a list)"
145
+ },
146
+ "from": {
147
+ "in": "spec",
148
+ "desc": "first fiscal year to keep"
149
+ },
150
+ "to": {
151
+ "in": "spec",
152
+ "desc": "last fiscal year to keep"
153
+ },
154
+ "transaction_type": {
155
+ "in": "spec",
156
+ "desc": "keep these transaction types: Obligations, Disbursements, Appropriated and Planned, President's Budget Requests (or their ids 2, 3, 1, 18)"
157
+ }
158
+ },
159
+ "records": "$csv",
160
+ "paging": {
161
+ "type": "none"
162
+ },
163
+ "max_rows": 300000
164
+ }
165
+ }
166
+ }
@@ -0,0 +1,80 @@
1
+ {
2
+ "id": "foreign-assistance",
3
+ "title": "ForeignAssistance.gov: U.S. foreign aid by country, sector and agency",
4
+ "agency": "U.S. Department of State, Office of Foreign Assistance",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "foreign aid",
8
+ "federal spending",
9
+ "international"
10
+ ],
11
+ "keywords": [
12
+ "foreign aid",
13
+ "foreign assistance",
14
+ "foreignassistance.gov",
15
+ "usaid",
16
+ "aid to ukraine",
17
+ "aid to israel",
18
+ "military aid",
19
+ "humanitarian assistance",
20
+ "development assistance",
21
+ "global health aid",
22
+ "aid by country",
23
+ "aid by sector",
24
+ "obligations and disbursements",
25
+ "oda",
26
+ "dac sector"
27
+ ],
28
+ "questions": [
29
+ "how much U.S. foreign aid a country received by year",
30
+ "largest recipients of U.S. foreign assistance",
31
+ "foreign aid by sector (health, security, humanitarian)",
32
+ "aid managed by USAID vs State vs Defense",
33
+ "foreign assistance budget requests by account"
34
+ ],
35
+ "homepage": "https://foreignassistance.gov/",
36
+ "terms_url": "https://foreignassistance.gov/data",
37
+ "auth": {
38
+ "env": null,
39
+ "required": "none",
40
+ "placement": "none",
41
+ "signup_url": null,
42
+ "echoes_key": false
43
+ },
44
+ "limits": {
45
+ "rps": 1,
46
+ "burst": 2,
47
+ "max_rows": 300000
48
+ },
49
+ "routes": {
50
+ "manual": {
51
+ "how": "Open https://foreignassistance.gov/data (Data Download), download the CSV (By Country, By DAC Sector, By USG Sector, By Managing Agency, By Funding Agency), and upload it with ingest."
52
+ }
53
+ },
54
+ "cache": {
55
+ "mode": "run_only",
56
+ "ttl_hours": null,
57
+ "why": "Files are refreshed with the site (data last updated date shown on every page)."
58
+ },
59
+ "redistribution": {
60
+ "mode": "allowed",
61
+ "why": "U.S. government data; the site states an Open Data License."
62
+ },
63
+ "notices": [],
64
+ "retention": null,
65
+ "citation": "U.S. Department of State and USAID, ForeignAssistance.gov, {dataset}, retrieved {retrieved_long}.",
66
+ "audit_hooks": [],
67
+ "clarify": [
68
+ "Which transaction type? Obligations (commitments), Disbursements (money spent), Appropriated and Planned, or President's Budget Requests: never add them together. Default: obligations for 'how much aid', disbursements for 'how much was spent'.",
69
+ "Current or constant dollars? current_amount is nominal; constant_amount is inflation-adjusted. Default: current for one year, constant for trends.",
70
+ "Which years? Fiscal years (October-September); the latest year is incomplete until reporting closes."
71
+ ],
72
+ "formats": [
73
+ "csv"
74
+ ],
75
+ "aliases": [
76
+ "ForeignAssistance.gov",
77
+ "FA.gov",
78
+ "U.S. foreign aid"
79
+ ]
80
+ }
@@ -0,0 +1,76 @@
1
+ # ForeignAssistance.gov: U.S. foreign aid by country, sector and agency
2
+ Depth: standard. Key: none. The aggregate data files of ForeignAssistance.gov, the U.S. government's foreign
3
+ assistance reporting site (State Department and USAID). They give foreign aid by:
4
+ - recipient country or region;
5
+ - international (OECD DAC) and U.S. government sector;
6
+ - managing agency;
7
+ - funding agency and account.
8
+ Each file has transaction type (obligations, disbursements, appropriated and planned, President's budget
9
+ requests) and fiscal year, in current and constant dollars.
10
+
11
+ ## When to use
12
+ - How much U.S. aid a country received; the largest recipients; aid by sector (health, peace and security,
13
+ humanitarian, economic development, democracy); which agencies manage it; trends since the 1940s (in constant
14
+ dollars).
15
+ - Not here: award-level federal grants and contracts with recipient names (usaspending); federal budget totals
16
+ by function (omb, treasury); other donors' aid (world-bank has some ODA indicators).
17
+
18
+ ## Endpoints
19
+ All take the same filters (applied to the published file): `country` (ISO3 code like `UKR`, or the name),
20
+ `fiscal_year`, `from`/`to`, `transaction_type` (name or id).
21
+
22
+ | Endpoint | File | Columns before the amounts |
23
+ |---|---|---|
24
+ | `by_country` | us_foreign_aid_country.csv | Country Code, Country Name, Region ID, Region Name, Income Group Acronym, Income Group Name |
25
+ | `by_dac_sector` | us_foreign_aid_dac_sector.csv | International Category ID/Name, International Sector Code/Name (no country) |
26
+ | `by_usg_sector` | us_foreign_aid_usg_sector.csv | US Category ID/Name, US Sector ID/Name, Country Code/Name |
27
+ | `by_managing_agency` | us_foreign_aid_implementing.csv | Managing Agency ID/Acronym/Name, Country Code/Name |
28
+ | `by_funding_agency` | us_foreign_aid_funding.csv | Funding Agency ID/Acronym/Name, Funding Account ID/Name, Country Code/Name |
29
+
30
+ Every file then has `Transaction Type ID`, `Transaction Type Name`, `Fiscal Year`, `current_amount` and
31
+ `constant_amount`.
32
+
33
+ ## Spec examples
34
+ ```json
35
+ {"description": "U.S. aid obligations to Ukraine, FY2020-2025",
36
+ "requests": [{"name": "ukraine", "endpoint": "by_country",
37
+ "params": {"country": "UKR", "from": "2020", "to": "2025", "transaction_type": "Obligations"}}]}
38
+ ```
39
+ ```json
40
+ {"description": "Aid to Ukraine by managing agency, FY2023",
41
+ "requests": [{"name": "ukraine_agency_2023", "endpoint": "by_managing_agency",
42
+ "params": {"country": "UKR", "fiscal_year": "2023", "transaction_type": "Obligations"}}]}
43
+ ```
44
+
45
+ ## Data format and traps
46
+ - Transaction types are different measures; never add them:
47
+ - 2, Obligations: commitments made, FY1946 on.
48
+ - 3, Disbursements: money paid out, FY2001 on.
49
+ - 1, Appropriated and Planned: FY2004-2023.
50
+ - 18, President's Budget Requests: FY2005-2024.
51
+ (Ranges in the by-country file of 2026-10-30.)
52
+ - Obligations and disbursements in one year are not the same money: disbursements lag obligations.
53
+ - Dollars are whole dollars as text. `constant_amount` is inflation-adjusted and equals `current_amount` in the
54
+ latest complete year (FY2025 in the files of 2026-10-30), so it is in that year's dollars. Name the base year.
55
+ - Fiscal years run October to September. The newest year (2026 in those files) holds partial reporting or
56
+ budget figures only: do not compare it with full years.
57
+ - The country file includes regions and worldwide programs as rows (`WLD` World, `AFR` Africa Region, `ASN` Asia
58
+ Region ...). Former countries have the code `NULL` ("Soviet Union (former)"). Kosovo is `CS-KM`. Some names
59
+ contain line breaks. Drop regional rows before ranking countries, and say that regional and worldwide programs
60
+ are not attributed to countries.
61
+ - Sector and agency files split the same money by country and category: totals from different files agree
62
+ only roughly (late or unattributed reporting). Do not mix files in one sum.
63
+ - Security and military assistance appears where agencies report it (e.g. State and Defense programs): check
64
+ the managing-agency file before calling a total "civilian" or "military".
65
+
66
+ ## Access
67
+ No key. Manual only: on 2026-11-07 foreignassistance.gov answered this server's requests with HTTP 403 (a plain
68
+ browser download works), and the site's Data API at /data-api/ answered 404, its documentation listing
69
+ authentication errors. So `fetch` refuses and `manual_bundle` gives the link. The user downloads the CSV
70
+ from https://foreignassistance.gov/data (Data Download; files up to 22 MB, e.g.
71
+ https://foreignassistance.gov/static/us_foreign_aid_country.csv) and uploads it, and `ingest` applies the
72
+ filters.
73
+
74
+ ## Citation and terms
75
+ "U.S. Department of State and USAID, ForeignAssistance.gov, <file> (data last updated <date shown on the site>),
76
+ retrieved <date>." Open Data License (as stated on the site).
@@ -11,7 +11,7 @@ school and award-year quarter (from COD). Published .xls workbooks, read into ti
11
11
  - Loans originated and disbursed, and Pell/TEACH grants disbursed, by school, state and program, per award year.
12
12
  - Not here: college costs, completion, earnings and debt at graduation by institution (college-scorecard);
13
13
  federal outlays and awards for student aid (treasury, usaspending); household debt from
14
- surveys (fred for the Fed's G.19 consumer credit series); enrollment counts (IPEDS, not a pack).
14
+ surveys (fred for the Fed's G.19 consumer credit series); enrollment counts (IPEDS: nces).
15
15
 
16
16
  ## Endpoints and datasets
17
17
  | Endpoint | Returns (one row per ...) | Params | Units | Coverage |
@@ -8,7 +8,7 @@ England town, metro FMR area and state, per federal fiscal year.
8
8
  income limits (30%, 50%, 80% of area median) by household size; area median family income as HUD
9
9
  sets it; MTSP (LIHTC) limits.
10
10
  - Not here: market rents and home values from surveys (census ACS median gross rent), house price
11
- indexes (fred, FHFA), homelessness counts and public housing (not covered), mortgage complaints
11
+ indexes (fred, FHFA), homelessness counts (hud-ahar), public housing (not covered), mortgage complaints
12
12
  (cfpb). FMRs are policy values (the 40th percentile gross rent standard), not observed averages.
13
13
 
14
14
  ## Endpoints and datasets