data-analyst-mcp 0.3.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/README.md +31 -6
  2. package/dist/adapters/_census_micro.js +2 -2
  3. package/dist/adapters/_census_micro.js.map +1 -1
  4. package/dist/adapters/bjs.d.ts +13 -0
  5. package/dist/adapters/bjs.js +28 -0
  6. package/dist/adapters/bjs.js.map +1 -0
  7. package/dist/adapters/cbp.d.ts +5 -0
  8. package/dist/adapters/cbp.js +64 -0
  9. package/dist/adapters/cbp.js.map +1 -0
  10. package/dist/adapters/cdc-wonder.d.ts +5 -0
  11. package/dist/adapters/cdc-wonder.js +68 -0
  12. package/dist/adapters/cdc-wonder.js.map +1 -0
  13. package/dist/adapters/cpsc.d.ts +4 -0
  14. package/dist/adapters/cpsc.js +125 -0
  15. package/dist/adapters/cpsc.js.map +1 -0
  16. package/dist/adapters/fhwa.d.ts +9 -0
  17. package/dist/adapters/fhwa.js +16 -0
  18. package/dist/adapters/fhwa.js.map +1 -0
  19. package/dist/adapters/foreign-assistance.d.ts +9 -0
  20. package/dist/adapters/foreign-assistance.js +42 -0
  21. package/dist/adapters/foreign-assistance.js.map +1 -0
  22. package/dist/adapters/hud-ahar.d.ts +3 -0
  23. package/dist/adapters/hud-ahar.js +80 -0
  24. package/dist/adapters/hud-ahar.js.map +1 -0
  25. package/dist/adapters/nces.d.ts +10 -0
  26. package/dist/adapters/nces.js +32 -0
  27. package/dist/adapters/nces.js.map +1 -0
  28. package/dist/adapters/nsf.d.ts +11 -0
  29. package/dist/adapters/nsf.js +25 -0
  30. package/dist/adapters/nsf.js.map +1 -0
  31. package/dist/adapters/onrr.d.ts +9 -0
  32. package/dist/adapters/onrr.js +35 -0
  33. package/dist/adapters/onrr.js.map +1 -0
  34. package/dist/adapters/sba.d.ts +10 -0
  35. package/dist/adapters/sba.js +134 -0
  36. package/dist/adapters/sba.js.map +1 -0
  37. package/dist/adapters/treasury-tic.d.ts +30 -0
  38. package/dist/adapters/treasury-tic.js +149 -0
  39. package/dist/adapters/treasury-tic.js.map +1 -0
  40. package/dist/adapters/uscourts.d.ts +8 -0
  41. package/dist/adapters/uscourts.js +8 -0
  42. package/dist/adapters/uscourts.js.map +1 -0
  43. package/dist/analysis/cite.d.ts +9 -1
  44. package/dist/analysis/cite.js +9 -2
  45. package/dist/analysis/cite.js.map +1 -1
  46. package/dist/analysis/compute.d.ts +22 -10
  47. package/dist/analysis/compute.js +75 -16
  48. package/dist/analysis/compute.js.map +1 -1
  49. package/dist/analysis/expr.d.ts +2 -0
  50. package/dist/analysis/expr.js +17 -0
  51. package/dist/analysis/expr.js.map +1 -1
  52. package/dist/analysis/replicate.d.ts +15 -0
  53. package/dist/analysis/replicate.js +228 -0
  54. package/dist/analysis/replicate.js.map +1 -0
  55. package/dist/analysis/rerun.js +2 -1
  56. package/dist/analysis/rerun.js.map +1 -1
  57. package/dist/analysis/table.d.ts +14 -0
  58. package/dist/analysis/table.js +79 -28
  59. package/dist/analysis/table.js.map +1 -1
  60. package/dist/analysis/tools.js +16 -4
  61. package/dist/analysis/tools.js.map +1 -1
  62. package/dist/bin/cli.d.ts +2 -0
  63. package/dist/bin/cli.js +17 -0
  64. package/dist/bin/cli.js.map +1 -1
  65. package/dist/core/generic-rest.js +42 -17
  66. package/dist/core/generic-rest.js.map +1 -1
  67. package/dist/core/guard.d.ts +32 -0
  68. package/dist/core/guard.js +176 -0
  69. package/dist/core/guard.js.map +1 -0
  70. package/dist/core/http.d.ts +10 -0
  71. package/dist/core/http.js +36 -7
  72. package/dist/core/http.js.map +1 -1
  73. package/dist/core/keys.d.ts +13 -3
  74. package/dist/core/keys.js +67 -7
  75. package/dist/core/keys.js.map +1 -1
  76. package/dist/core/package.d.ts +12 -3
  77. package/dist/core/package.js +56 -12
  78. package/dist/core/package.js.map +1 -1
  79. package/dist/core/packs.d.ts +14 -5
  80. package/dist/core/packs.js.map +1 -1
  81. package/dist/{deep/_opm_parquet.d.ts → core/parquet.d.ts} +2 -1
  82. package/dist/{deep/_opm_parquet.js → core/parquet.js} +7 -5
  83. package/dist/core/parquet.js.map +1 -0
  84. package/dist/core/secret_patterns.d.ts +6 -0
  85. package/dist/core/secret_patterns.js +20 -0
  86. package/dist/core/secret_patterns.js.map +1 -0
  87. package/dist/core/tabular.d.ts +43 -1
  88. package/dist/core/tabular.js +149 -6
  89. package/dist/core/tabular.js.map +1 -1
  90. package/dist/deep/_opm_io.js +2 -2
  91. package/dist/deep/_opm_io.js.map +1 -1
  92. package/dist/deep/bea.js +1 -0
  93. package/dist/deep/bea.js.map +1 -1
  94. package/dist/deep/fred.js +40 -9
  95. package/dist/deep/fred.js.map +1 -1
  96. package/dist/deep/ipums.js +1 -0
  97. package/dist/deep/ipums.js.map +1 -1
  98. package/dist/deep/opm.js +1 -1
  99. package/dist/deep/opm.js.map +1 -1
  100. package/dist/envelope.js +3 -1
  101. package/dist/envelope.js.map +1 -1
  102. package/dist/index.d.ts +29 -22
  103. package/dist/index.js +45 -46
  104. package/dist/index.js.map +1 -1
  105. package/dist/remote/app.js +87 -31
  106. package/dist/remote/app.js.map +1 -1
  107. package/dist/remote/oauth.js +8 -1
  108. package/dist/remote/oauth.js.map +1 -1
  109. package/dist/remote/pages.d.ts +3 -7
  110. package/dist/remote/pages.js +49 -19
  111. package/dist/remote/pages.js.map +1 -1
  112. package/dist/remote/sign.d.ts +7 -1
  113. package/dist/remote/sign.js +33 -7
  114. package/dist/remote/sign.js.map +1 -1
  115. package/dist/remote/store.js +14 -5
  116. package/dist/remote/store.js.map +1 -1
  117. package/dist/report/audit_report.js +29 -7
  118. package/dist/report/audit_report.js.map +1 -1
  119. package/dist/report/build_report.js +34 -4
  120. package/dist/report/build_report.js.map +1 -1
  121. package/dist/report/method_audit.js +3 -0
  122. package/dist/report/method_audit.js.map +1 -1
  123. package/dist/report/scan_secrets.js +2 -13
  124. package/dist/report/scan_secrets.js.map +1 -1
  125. package/dist/report/tools.js +2 -1
  126. package/dist/report/tools.js.map +1 -1
  127. package/dist/server.d.ts +3 -1
  128. package/dist/server.js +53 -17
  129. package/dist/server.js.map +1 -1
  130. package/dist/tools.d.ts +3 -1
  131. package/dist/tools.js +82 -9
  132. package/dist/tools.js.map +1 -1
  133. package/guide/WORKFLOW.md +6 -5
  134. package/guide/assets/download_data.ps1 +2 -0
  135. package/guide/assets/download_data.sh +8 -2
  136. package/guide/references/report_patterns.md +5 -2
  137. package/guide/references/report_schema.md +140 -0
  138. package/guide/references/routing.md +30 -3
  139. package/package.json +1 -1
  140. package/packs/bjs/endpoints.json +150 -0
  141. package/packs/bjs/policy.json +114 -0
  142. package/packs/bjs/source.md +98 -0
  143. package/packs/cbp/endpoints.json +31 -0
  144. package/packs/cbp/policy.json +85 -0
  145. package/packs/cbp/source.md +57 -0
  146. package/packs/cdc/source.md +1 -1
  147. package/packs/cdc-wonder/endpoints.json +26 -0
  148. package/packs/cdc-wonder/policy.json +84 -0
  149. package/packs/cdc-wonder/source.md +53 -0
  150. package/packs/census/source.md +8 -4
  151. package/packs/college-scorecard/source.md +1 -1
  152. package/packs/cpsc/endpoints.json +91 -0
  153. package/packs/cpsc/policy.json +91 -0
  154. package/packs/cpsc/source.md +76 -0
  155. package/packs/dhs-ohss/source.md +1 -1
  156. package/packs/doj-news/source.md +1 -1
  157. package/packs/fbi/source.md +3 -3
  158. package/packs/fhwa/endpoints.json +48 -0
  159. package/packs/fhwa/policy.json +91 -0
  160. package/packs/fhwa/source.md +59 -0
  161. package/packs/foreign-assistance/endpoints.json +166 -0
  162. package/packs/foreign-assistance/policy.json +80 -0
  163. package/packs/foreign-assistance/source.md +76 -0
  164. package/packs/fred/source.md +2 -1
  165. package/packs/fsa/source.md +1 -1
  166. package/packs/hud/source.md +1 -1
  167. package/packs/hud-ahar/endpoints.json +92 -0
  168. package/packs/hud-ahar/policy.json +88 -0
  169. package/packs/hud-ahar/source.md +72 -0
  170. package/packs/ipums/docs/report_patterns.md +1 -1
  171. package/packs/ipums/docs/variables.md +4 -3
  172. package/packs/ipums/source.md +10 -8
  173. package/packs/naep/source.md +1 -1
  174. package/packs/nces/endpoints.json +101 -0
  175. package/packs/nces/policy.json +99 -0
  176. package/packs/nces/source.md +75 -0
  177. package/packs/nhtsa/source.md +1 -1
  178. package/packs/nsf/endpoints.json +33 -0
  179. package/packs/nsf/policy.json +83 -0
  180. package/packs/nsf/source.md +59 -0
  181. package/packs/onrr/endpoints.json +198 -0
  182. package/packs/onrr/policy.json +91 -0
  183. package/packs/onrr/source.md +55 -0
  184. package/packs/opm/docs/opm_api.md +2 -1
  185. package/packs/opm/source.md +2 -1
  186. package/packs/port.json +65 -0
  187. package/packs/sba/endpoints.json +34 -0
  188. package/packs/sba/policy.json +86 -0
  189. package/packs/sba/source.md +60 -0
  190. package/packs/treasury/source.md +1 -1
  191. package/packs/treasury-tic/endpoints.json +77 -0
  192. package/packs/treasury-tic/policy.json +89 -0
  193. package/packs/treasury-tic/source.md +80 -0
  194. package/packs/uscourts/endpoints.json +22 -0
  195. package/packs/uscourts/policy.json +87 -0
  196. package/packs/uscourts/source.md +56 -0
  197. package/packs/uspto/source.md +1 -1
  198. package/dist/deep/_opm_parquet.js.map +0 -1
@@ -0,0 +1,198 @@
1
+ {
2
+ "base_url": "https://revenuedata.onrr.gov",
3
+ "timeout": 240,
4
+ "endpoints": {
5
+ "fiscal_year_revenue": {
6
+ "path": "/downloads/fiscal_year_revenue.csv",
7
+ "method": "GET",
8
+ "description": "Federal and Native American natural resource revenue (royalties, rents, bonuses, other) by fiscal year, land class and category, state, county, offshore region, revenue type, lease type, commodity and product, dollars (ONRR, FY2004 on).",
9
+ "params": {
10
+ "state": {
11
+ "in": "spec",
12
+ "desc": "keep these states (full name as published, e.g. New Mexico; a list works)"
13
+ },
14
+ "from": {
15
+ "in": "spec",
16
+ "desc": "first year to keep (fiscal or calendar, as the file)"
17
+ },
18
+ "to": {
19
+ "in": "spec",
20
+ "desc": "last year to keep"
21
+ },
22
+ "commodity": {
23
+ "in": "spec",
24
+ "desc": "keep these commodities (Oil, Gas, Coal ...; as published, case-insensitive)"
25
+ },
26
+ "revenue_type": {
27
+ "in": "spec",
28
+ "desc": "keep these revenue types: Royalties, Rents, Bonus, Other revenues, Civil penalties, Inspection fees"
29
+ },
30
+ "land_class": {
31
+ "in": "spec",
32
+ "desc": "Federal or Native American"
33
+ }
34
+ },
35
+ "records": "$csv",
36
+ "paging": {
37
+ "type": "none"
38
+ },
39
+ "max_rows": 600000
40
+ },
41
+ "calendar_year_revenue": {
42
+ "path": "/downloads/calendar_year_revenue.csv",
43
+ "method": "GET",
44
+ "description": "The same revenue by calendar year (CY2003 on).",
45
+ "params": {
46
+ "state": {
47
+ "in": "spec",
48
+ "desc": "keep these states (full name as published, e.g. New Mexico; a list works)"
49
+ },
50
+ "from": {
51
+ "in": "spec",
52
+ "desc": "first year to keep (fiscal or calendar, as the file)"
53
+ },
54
+ "to": {
55
+ "in": "spec",
56
+ "desc": "last year to keep"
57
+ },
58
+ "commodity": {
59
+ "in": "spec",
60
+ "desc": "keep these commodities (Oil, Gas, Coal ...; as published, case-insensitive)"
61
+ },
62
+ "revenue_type": {
63
+ "in": "spec",
64
+ "desc": "keep these revenue types: Royalties, Rents, Bonus, Other revenues, Civil penalties, Inspection fees"
65
+ },
66
+ "land_class": {
67
+ "in": "spec",
68
+ "desc": "Federal or Native American"
69
+ }
70
+ },
71
+ "records": "$csv",
72
+ "paging": {
73
+ "type": "none"
74
+ },
75
+ "max_rows": 600000
76
+ },
77
+ "monthly_revenue": {
78
+ "path": "/downloads/monthly_revenue.csv",
79
+ "method": "GET",
80
+ "description": "The same revenue by month (January 2003 on); a 42 MB file: filter by state and years.",
81
+ "params": {
82
+ "state": {
83
+ "in": "spec",
84
+ "desc": "keep these states (full name as published, e.g. New Mexico; a list works)"
85
+ },
86
+ "from": {
87
+ "in": "spec",
88
+ "desc": "first year to keep (fiscal or calendar, as the file)"
89
+ },
90
+ "to": {
91
+ "in": "spec",
92
+ "desc": "last year to keep"
93
+ },
94
+ "commodity": {
95
+ "in": "spec",
96
+ "desc": "keep these commodities (Oil, Gas, Coal ...; as published, case-insensitive)"
97
+ },
98
+ "revenue_type": {
99
+ "in": "spec",
100
+ "desc": "keep these revenue types: Royalties, Rents, Bonus, Other revenues, Civil penalties, Inspection fees"
101
+ },
102
+ "land_class": {
103
+ "in": "spec",
104
+ "desc": "Federal or Native American"
105
+ }
106
+ },
107
+ "records": "$csv",
108
+ "paging": {
109
+ "type": "none"
110
+ },
111
+ "max_rows": 2000000
112
+ },
113
+ "fiscal_year_disbursements": {
114
+ "path": "/downloads/fiscal_year_disbursements.csv",
115
+ "method": "GET",
116
+ "description": "Where the revenue went: disbursements by fiscal year to the U.S. Treasury, states and counties, the Reclamation Fund, Land and Water Conservation Fund, Historic Preservation Fund, Native American tribes and individuals, other funds (FY2003 on).",
117
+ "params": {
118
+ "state": {
119
+ "in": "spec",
120
+ "desc": "keep these states (full name as published, e.g. New Mexico; a list works)"
121
+ },
122
+ "from": {
123
+ "in": "spec",
124
+ "desc": "first year to keep (fiscal or calendar, as the file)"
125
+ },
126
+ "to": {
127
+ "in": "spec",
128
+ "desc": "last year to keep"
129
+ },
130
+ "fund_type": {
131
+ "in": "spec",
132
+ "desc": "keep these fund types as published (State and local governments, U.S. Treasury, Reclamation Fund ...)"
133
+ }
134
+ },
135
+ "records": "$csv",
136
+ "paging": {
137
+ "type": "none"
138
+ },
139
+ "max_rows": 600000
140
+ },
141
+ "fiscal_year_production": {
142
+ "path": "/downloads/fiscal_year_production.csv",
143
+ "method": "GET",
144
+ "description": "Production volumes on federal and Native American lands and offshore by fiscal year, state, county, offshore region and product (oil bbl, gas mcf, coal tons ...).",
145
+ "params": {
146
+ "state": {
147
+ "in": "spec",
148
+ "desc": "keep these states (full name as published, e.g. New Mexico; a list works)"
149
+ },
150
+ "from": {
151
+ "in": "spec",
152
+ "desc": "first year to keep (fiscal or calendar, as the file)"
153
+ },
154
+ "to": {
155
+ "in": "spec",
156
+ "desc": "last year to keep"
157
+ },
158
+ "product": {
159
+ "in": "spec",
160
+ "desc": "keep these products as published, e.g. Oil (bbl), Gas (mcf)"
161
+ }
162
+ },
163
+ "records": "$csv",
164
+ "paging": {
165
+ "type": "none"
166
+ },
167
+ "max_rows": 600000
168
+ },
169
+ "calendar_year_production": {
170
+ "path": "/downloads/calendar_year_production.csv",
171
+ "method": "GET",
172
+ "description": "The same production by calendar year.",
173
+ "params": {
174
+ "state": {
175
+ "in": "spec",
176
+ "desc": "keep these states (full name as published, e.g. New Mexico; a list works)"
177
+ },
178
+ "from": {
179
+ "in": "spec",
180
+ "desc": "first year to keep (fiscal or calendar, as the file)"
181
+ },
182
+ "to": {
183
+ "in": "spec",
184
+ "desc": "last year to keep"
185
+ },
186
+ "product": {
187
+ "in": "spec",
188
+ "desc": "keep these products as published, e.g. Oil (bbl), Gas (mcf)"
189
+ }
190
+ },
191
+ "records": "$csv",
192
+ "paging": {
193
+ "type": "none"
194
+ },
195
+ "max_rows": 600000
196
+ }
197
+ }
198
+ }
@@ -0,0 +1,91 @@
1
+ {
2
+ "id": "onrr",
3
+ "title": "ONRR natural resources revenue data: federal and Native American royalties, rents, bonuses, disbursements and production",
4
+ "agency": "U.S. Department of the Interior, Office of Natural Resources Revenue",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "energy",
8
+ "federal revenue",
9
+ "public lands"
10
+ ],
11
+ "keywords": [
12
+ "onrr",
13
+ "office of natural resources revenue",
14
+ "natural resources revenue",
15
+ "federal royalties",
16
+ "oil and gas royalties",
17
+ "coal royalties",
18
+ "offshore leasing",
19
+ "lease bonus",
20
+ "rents",
21
+ "federal lands",
22
+ "public lands revenue",
23
+ "mineral leasing",
24
+ "disbursements to states",
25
+ "gomesa",
26
+ "reclamation fund",
27
+ "land and water conservation fund",
28
+ "production on federal lands",
29
+ "native american mineral revenue"
30
+ ],
31
+ "questions": [
32
+ "federal oil and gas royalties by state",
33
+ "revenue from federal lands in New Mexico or Wyoming",
34
+ "offshore revenue in the Gulf",
35
+ "how much mineral revenue went to a state",
36
+ "oil and gas production on federal lands by year",
37
+ "coal royalties over time"
38
+ ],
39
+ "homepage": "https://revenuedata.onrr.gov/",
40
+ "terms_url": "https://revenuedata.onrr.gov/downloads/",
41
+ "auth": {
42
+ "env": null,
43
+ "required": "none",
44
+ "placement": "none",
45
+ "signup_url": null,
46
+ "echoes_key": false
47
+ },
48
+ "limits": {
49
+ "rps": 1,
50
+ "burst": 2,
51
+ "max_rows": 2000000
52
+ },
53
+ "routes": {
54
+ "api": {
55
+ "base": "https://revenuedata.onrr.gov",
56
+ "probe": "https://revenuedata.onrr.gov/downloads/fiscal_year_disbursements.csv",
57
+ "hosts": [
58
+ "revenuedata.onrr.gov",
59
+ "revenuedata.doi.gov"
60
+ ]
61
+ },
62
+ "manual": {
63
+ "how": "Open https://revenuedata.onrr.gov/downloads/ (Revenue, Disbursements or Production, then 'Downloads and documentation'), download the CSV and upload it with ingest."
64
+ }
65
+ },
66
+ "cache": {
67
+ "mode": "run_only",
68
+ "ttl_hours": null,
69
+ "why": "Files are refreshed monthly; data/ is reused within a run."
70
+ },
71
+ "redistribution": {
72
+ "mode": "allowed",
73
+ "why": "ONRR data are U.S. government works in the public domain (17 U.S.C. 105)."
74
+ },
75
+ "notices": [],
76
+ "retention": null,
77
+ "citation": "U.S. Department of the Interior, Office of Natural Resources Revenue, {dataset}, revenuedata.onrr.gov, retrieved {retrieved_long}.",
78
+ "audit_hooks": [],
79
+ "clarify": [
80
+ "Revenue or disbursements? Revenue is what companies paid ONRR (by where the resource is); disbursements are what ONRR paid out to the Treasury, states, counties and funds. Default: revenue for 'how much came from', disbursements for 'how much a state got'.",
81
+ "Fiscal or calendar year? Federal fiscal years run October-September. Default: fiscal year, said as such."
82
+ ],
83
+ "formats": [
84
+ "csv"
85
+ ],
86
+ "aliases": [
87
+ "ONRR",
88
+ "Natural Resources Revenue Data",
89
+ "revenuedata.doi.gov"
90
+ ]
91
+ }
@@ -0,0 +1,55 @@
1
+ # ONRR natural resources revenue data: federal and Native American royalties, rents, bonuses, disbursements and production
2
+ Depth: standard. Key: none. The Office of Natural Resources Revenue's open data (Natural Resources Revenue Data,
3
+ revenuedata.onrr.gov), as its CSV downloads:
4
+ - revenue companies paid for extracting oil, gas, coal, minerals and renewable energy on federal onshore and
5
+ offshore lands and Native American lands (royalties, rents, bonuses, other revenues), by year or month;
6
+ - disbursements of that money to the U.S. Treasury, states and counties, and federal funds;
7
+ - production volumes on those lands.
8
+
9
+ ## When to use
10
+ - Federal royalties and other mineral revenue by state, county, offshore region, commodity or year; offshore
11
+ lease bonuses; how much a state received (disbursements); oil, gas and coal produced on federal lands.
12
+ - Not here: total U.S. or state production and prices on all lands (eia); federal receipts in the budget
13
+ (treasury MTS, omb); severance taxes states levy themselves (census government finances).
14
+
15
+ ## Endpoints
16
+ All take `state` (full name), `from`/`to` (years).
17
+
18
+ | Endpoint | Columns | More filters |
19
+ |---|---|---|
20
+ | `fiscal_year_revenue`, `calendar_year_revenue`, `monthly_revenue` (`Date` MM/DD/YYYY; 42 MB) | Fiscal Year (or Calendar Year, Date), Land Class, Land Category, State, County, FIPS Code, Offshore Region, Revenue Type, Mineral Lease Type, Commodity, Product, Revenue | `commodity`, `revenue_type`, `land_class` |
21
+ | `fiscal_year_disbursements` | Fiscal Year, Fund Type, Source, State, County, Disbursement | `fund_type` |
22
+ | `fiscal_year_production`, `calendar_year_production` | Fiscal Year (or Calendar Year), Land Class, Land Category, State, County, FIPS Code, Offshore Region, Product, Volume | `product` |
23
+
24
+ ## Spec examples
25
+ ```json
26
+ {"description": "Federal mineral revenue in New Mexico by type and commodity, FY2025",
27
+ "requests": [{"name": "nm_fy2025", "endpoint": "fiscal_year_revenue", "params": {"state": "New Mexico", "from": "2025", "to": "2025"}}]}
28
+ ```
29
+ ```json
30
+ {"description": "Disbursements to Wyoming, FY2020-2025",
31
+ "requests": [{"name": "wy_disb", "endpoint": "fiscal_year_disbursements", "params": {"state": "Wyoming", "from": "2020", "to": "2025"}}]}
32
+ ```
33
+
34
+ ## Data format and traps
35
+ - Revenue is in dollars, as text. Rows can be negative (adjustments and refunds: 3,486 negative rows in the
36
+ fiscal-year file of 2026-11-07). Sum them with the rest; never drop them.
37
+ - Revenue by state is where the resource is (onshore federal land). Offshore revenue has no state (`State`
38
+ empty, `Offshore Region` = Gulf of America, Pacific, Alaska, Atlantic). Native American revenue (`Land Class`
39
+ Native American) has no state or county in these files. A state total from the State column
40
+ is federal onshore revenue only: say so.
41
+ - Revenue and disbursements are different money flows and years: disbursements follow collection, and offshore
42
+ revenue reaches states under their own formulas. Never subtract one from the other.
43
+ - `Product` carries the unit in production files (Oil (bbl), Gas (mcf), Coal (tons) ...): never add volumes
44
+ across products.
45
+ - The current fiscal year is partial. Fiscal years run October-September. Revenue starts FY2004 (CY2003);
46
+ older years are in separate historical files (not covered).
47
+
48
+ ## Access
49
+ No key. `fetch` reads revenuedata.onrr.gov/downloads/<file>.csv (revenuedata.doi.gov redirects there; verified
50
+ 2026-11-07; up to 42 MB) and filters it. Manual: the Downloads page, then `ingest`. The site's documentation
51
+ pages under /downloads/ describe each field.
52
+
53
+ ## Citation and terms
54
+ "U.S. Department of the Interior, Office of Natural Resources Revenue, Natural Resources Revenue Data, <file>,
55
+ revenuedata.onrr.gov, retrieved <date>." Public domain (17 U.S.C. 105).
@@ -59,7 +59,8 @@ The client retries 429 and 5xx with backoff.
59
59
  - **Data dictionary:** an Excel file (`FWD Data Dictionary.xlsx`); this pack ships it as
60
60
  `data_dictionary.csv` (copied into every `data/opm/` folder; `request_schema` lists its columns), so it
61
61
  never needs downloading.
62
- - **Reading Parquet:** `fetch` and `ingest` read the Parquet files themselves; no other software is needed.
62
+ - **Reading Parquet:** `fetch` and `ingest` read the Parquet files themselves, and `compute` takes the
63
+ `data/opm/*.parquet` it writes as inputs; no other software is needed.
63
64
 
64
65
  ## Other OPM tools (no API)
65
66
  - **Table Builder** (https://data.opm.gov/explore-data/data/table-builder): aggregate tables in a browser;
@@ -68,7 +68,8 @@ effective month also needs the next two or three files (late reports).
68
68
 
69
69
  ## Data format and traps
70
70
  `fetch` writes `data/opm/`: `opm_<dataset>.parquet`, `opm_<dataset>.csv` up to 1,000,000 rows (above that only
71
- the Parquet file, which `compute` cannot read: narrow the request or use `aggregate`), `manifest.json` (every
71
+ the Parquet file; `compute` reads it too: name only the columns you need, `{"path": "data/opm/opm_employment.parquet",
72
+ "columns": [...]}`, or narrow the request or use `aggregate`), `manifest.json` (every
72
73
  source file, version, publish date, SHA-256, rows, missing columns), `request.json`, `data_dictionary.csv`,
73
74
  `README.md`, `spec_used.json`. Read the result's `notes`: a filter column absent from an older file
74
75
  means those rows were NOT filtered; missing requested columns are filled with nulls.
package/packs/port.json CHANGED
@@ -14,6 +14,11 @@
14
14
  "fetch.py"
15
15
  ]
16
16
  },
17
+ "bjs": {
18
+ "python": [
19
+ "adapter.py"
20
+ ]
21
+ },
17
22
  "bls": {
18
23
  "python": [
19
24
  "fetch.py"
@@ -24,6 +29,16 @@
24
29
  "adapter.py"
25
30
  ]
26
31
  },
32
+ "cbp": {
33
+ "python": [
34
+ "adapter.py"
35
+ ]
36
+ },
37
+ "cdc-wonder": {
38
+ "python": [
39
+ "adapter.py"
40
+ ]
41
+ },
27
42
  "census": {
28
43
  "python": [
29
44
  "adapter.py"
@@ -54,6 +69,11 @@
54
69
  "adapter.py"
55
70
  ]
56
71
  },
72
+ "cpsc": {
73
+ "python": [
74
+ "adapter.py"
75
+ ]
76
+ },
57
77
  "dhs-ohss": {
58
78
  "python": [
59
79
  "adapter.py"
@@ -119,6 +139,16 @@
119
139
  "adapter.py"
120
140
  ]
121
141
  },
142
+ "fhwa": {
143
+ "python": [
144
+ "adapter.py"
145
+ ]
146
+ },
147
+ "foreign-assistance": {
148
+ "python": [
149
+ "adapter.py"
150
+ ]
151
+ },
122
152
  "fred": {
123
153
  "python": [
124
154
  "fetch.py"
@@ -144,6 +174,11 @@
144
174
  "adapter.py"
145
175
  ]
146
176
  },
177
+ "hud-ahar": {
178
+ "python": [
179
+ "adapter.py"
180
+ ]
181
+ },
147
182
  "ipums": {
148
183
  "python": [
149
184
  "fetch.py"
@@ -159,6 +194,11 @@
159
194
  "adapter.py"
160
195
  ]
161
196
  },
197
+ "nces": {
198
+ "python": [
199
+ "adapter.py"
200
+ ]
201
+ },
162
202
  "nhtsa": {
163
203
  "python": [
164
204
  "adapter.py"
@@ -179,6 +219,11 @@
179
219
  "adapter.py"
180
220
  ]
181
221
  },
222
+ "nsf": {
223
+ "python": [
224
+ "adapter.py"
225
+ ]
226
+ },
182
227
  "nws": {
183
228
  "python": [
184
229
  "adapter.py"
@@ -189,6 +234,11 @@
189
234
  "adapter.py"
190
235
  ]
191
236
  },
237
+ "onrr": {
238
+ "python": [
239
+ "adapter.py"
240
+ ]
241
+ },
192
242
  "open-payments": {
193
243
  "python": [
194
244
  "adapter.py"
@@ -214,6 +264,11 @@
214
264
  "adapter.py"
215
265
  ]
216
266
  },
267
+ "sba": {
268
+ "python": [
269
+ "adapter.py"
270
+ ]
271
+ },
217
272
  "sec": {
218
273
  "python": [
219
274
  "adapter.py"
@@ -234,6 +289,11 @@
234
289
  "fetch.py"
235
290
  ]
236
291
  },
292
+ "treasury-tic": {
293
+ "python": [
294
+ "adapter.py"
295
+ ]
296
+ },
237
297
  "treasury-yields": {
238
298
  "python": [
239
299
  "adapter.py"
@@ -244,6 +304,11 @@
244
304
  "adapter.py"
245
305
  ]
246
306
  },
307
+ "uscourts": {
308
+ "python": [
309
+ "adapter.py"
310
+ ]
311
+ },
247
312
  "usda-fna": {
248
313
  "python": [
249
314
  "adapter.py"
@@ -0,0 +1,34 @@
1
+ {
2
+ "base_url": "https://data.sba.gov",
3
+ "timeout": 600,
4
+ "catalog_path": "/data.json",
5
+ "endpoints": {
6
+ "list_files": {
7
+ "path": "/data.json",
8
+ "method": "GET",
9
+ "description": "The loan data files SBA currently publishes (from data.sba.gov's catalog): 7(a) and 504 FOIA loan-level CSVs (file names carry the as-of date), activity reports, disaster loan workbooks. One row per file with the path for the foia request.",
10
+ "params": {
11
+ "search": {"in": "spec", "desc": "keep files whose dataset or file title contains this (e.g. FOIA, 504, Disaster)"}
12
+ },
13
+ "records": "",
14
+ "paging": {"type": "none"},
15
+ "max_rows": 5000
16
+ },
17
+ "foia": {
18
+ "path": "/sites/default/files/{file}.csv",
19
+ "method": "GET",
20
+ "description": "SBA 7(a) or 504 loan-level FOIA data: one row per approved loan with borrower, lender (or CDC), amount, approval date and fiscal year, NAICS industry, project state and county, congressional district, loan status, charge-off. Read streaming and filtered: give at least one of state, fiscal_year, from, naics.",
21
+ "params": {
22
+ "file": {"in": "path", "raw": true, "required": true, "desc": "the file's path under /sites/default/files without .csv, from list_files, e.g. uploaded_resources/FOIA_504_FY2010_Present_asof_260930"},
23
+ "state": {"in": "spec", "desc": "keep loans with this ProjectState (two letters; a list works)"},
24
+ "fiscal_year": {"in": "spec", "desc": "keep these ApprovalFY values (a year or a list)"},
25
+ "from": {"in": "spec", "desc": "first ApprovalFY to keep"},
26
+ "to": {"in": "spec", "desc": "last ApprovalFY to keep"},
27
+ "naics": {"in": "spec", "desc": "keep NaicsCode values starting with these digits (e.g. 72; a list works)"}
28
+ },
29
+ "records": "$csv",
30
+ "paging": {"type": "none"},
31
+ "max_rows": 1000000
32
+ }
33
+ }
34
+ }
@@ -0,0 +1,86 @@
1
+ {
2
+ "id": "sba",
3
+ "title": "SBA loan data: 7(a) and 504 loans (FOIA loan-level files), activity reports, disaster loans",
4
+ "agency": "U.S. Small Business Administration",
5
+ "depth": "standard",
6
+ "domains": [
7
+ "small business",
8
+ "federal lending",
9
+ "federal spending"
10
+ ],
11
+ "keywords": [
12
+ "sba",
13
+ "small business administration",
14
+ "sba loans",
15
+ "7(a) loans",
16
+ "7a loans",
17
+ "504 loans",
18
+ "sba lenders",
19
+ "certified development company",
20
+ "small business lending",
21
+ "loan charge-offs",
22
+ "sba loans by state",
23
+ "sba loans by industry",
24
+ "disaster loans"
25
+ ],
26
+ "questions": [
27
+ "SBA 7(a) loans approved in a state by fiscal year",
28
+ "504 loans by industry (NAICS)",
29
+ "largest SBA lenders",
30
+ "charge-off rate of SBA loans",
31
+ "SBA loans to restaurants in a state"
32
+ ],
33
+ "homepage": "https://data.sba.gov/",
34
+ "terms_url": "https://data.sba.gov/",
35
+ "auth": {
36
+ "env": null,
37
+ "required": "none",
38
+ "placement": "none",
39
+ "signup_url": null,
40
+ "echoes_key": false
41
+ },
42
+ "limits": {
43
+ "rps": 1,
44
+ "burst": 2,
45
+ "max_rows": 1000000
46
+ },
47
+ "routes": {
48
+ "api": {
49
+ "base": "https://data.sba.gov",
50
+ "probe": "https://data.sba.gov/data.json",
51
+ "hosts": [
52
+ "data.sba.gov"
53
+ ]
54
+ },
55
+ "manual": {
56
+ "how": "Open https://data.sba.gov/, find '7(a) & 504 FOIA', download the CSV (FOIA - 7(a) or 504, by period) and upload it with ingest; the filters apply to the uploaded file."
57
+ }
58
+ },
59
+ "cache": {
60
+ "mode": "run_only",
61
+ "ttl_hours": null,
62
+ "why": "Files are replaced quarterly (the as-of date is in the file name)."
63
+ },
64
+ "redistribution": {
65
+ "mode": "allowed",
66
+ "why": "SBA publishes these loan records under FOIA as public data; U.S. government works (17 U.S.C. 105)."
67
+ },
68
+ "notices": [],
69
+ "retention": null,
70
+ "citation": "U.S. Small Business Administration, {dataset}, data.sba.gov, retrieved {retrieved_long}.",
71
+ "audit_hooks": [],
72
+ "clarify": [
73
+ "Which program? 7(a) (lender loans with an SBA guaranty) or 504 (CDC loans for real estate and equipment, with a third-party lender part): they are different files and different dollars. Default: 7(a), said as such.",
74
+ "Approvals or outcomes? GrossApproval is the approved amount (504: the CDC's part; ThirdPartyDollars is the lender's); LoanStatus and GrossChargeOffAmount describe outcomes as of the file's date. Default: count and dollars approved by ApprovalFY."
75
+ ],
76
+ "formats": [
77
+ "csv",
78
+ "json"
79
+ ],
80
+ "aliases": [
81
+ "SBA",
82
+ "Small Business Administration",
83
+ "7(a)",
84
+ "504"
85
+ ]
86
+ }