data-analyst-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -2
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +716 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +24 -0
- package/dist/report/scan_secrets.js +265 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +247 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +54 -4
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +170 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* National Weather Service (api.weather.gov) adapter.
|
|
3
|
+
*
|
|
4
|
+
* - fetch: forecast endpoints first resolve the point (/points/lat,lon) to its forecast grid, then read
|
|
5
|
+
* the forecast URL that the point names; everything else runs through the generic fetcher.
|
|
6
|
+
* - User-Agent: NWS asks every client for an identifying User-Agent. The shared one is sent; set
|
|
7
|
+
* NWS_USER_AGENT (e.g. "my-analysis (me@example.com)") to send your own.
|
|
8
|
+
* - normalize: GeoJSON features lose their "properties." prefix; alert polygons are dropped (they are
|
|
9
|
+
* large; geometry.type stays); station points get longitude/latitude columns.
|
|
10
|
+
*/
|
|
11
|
+
import { clean, clientFor, fetchRequest, manualItems as grManualItems, resolve, split } from "../core/generic-rest.js";
|
|
12
|
+
import { DataError } from "../core/http.js";
|
|
13
|
+
import { pyStr } from "../core/py.js";
|
|
14
|
+
const repr = (s) => (s === null || s === undefined ? "None" : `'${String(s).replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`);
|
|
15
|
+
function client(pack) {
|
|
16
|
+
const ua = (process.env.NWS_USER_AGENT || "").trim();
|
|
17
|
+
return clientFor(pack, undefined, ua ? { userAgent: ua } : {});
|
|
18
|
+
}
|
|
19
|
+
/** f"{x:.4f}" (exact binary value, ties to even, as Python formats). */
|
|
20
|
+
function fixed4(x) {
|
|
21
|
+
const s = x * 10000;
|
|
22
|
+
if (Number.isFinite(s) && Number.isInteger(s * 2) && !Number.isInteger(s)) {
|
|
23
|
+
const fl = Math.floor(s);
|
|
24
|
+
return ((fl % 2 === 0 ? fl : fl + 1) / 10000).toFixed(4);
|
|
25
|
+
}
|
|
26
|
+
return x.toFixed(4);
|
|
27
|
+
}
|
|
28
|
+
/** /points accepts at most 4 decimal places; more gives a 301 redirect to the rounded point. */
|
|
29
|
+
function round(params) {
|
|
30
|
+
for (const k of ["lat", "lon"]) {
|
|
31
|
+
const v = params[k];
|
|
32
|
+
if (v !== null && v !== undefined && v !== "")
|
|
33
|
+
params[k] = fixed4(Number(typeof v === "string" ? v.trim() : v)).replace(/0+$/, "").replace(/\.$/, "");
|
|
34
|
+
}
|
|
35
|
+
return params;
|
|
36
|
+
}
|
|
37
|
+
async function forecast(c, ep, params, maxRows) {
|
|
38
|
+
const [p] = split(ep, params);
|
|
39
|
+
const point = await c.getJson(p);
|
|
40
|
+
const props = point?.properties || {};
|
|
41
|
+
const url = props[ep.resolve];
|
|
42
|
+
if (!url) {
|
|
43
|
+
throw new DataError(`nws: the point ${pyStr(params.lat)},${pyStr(params.lon)} has no ${ep.resolve} ` +
|
|
44
|
+
"(NWS forecasts cover the U.S. and its territories only)", "bad_request");
|
|
45
|
+
}
|
|
46
|
+
const fc = await c.getJson(url);
|
|
47
|
+
const fp = fc?.properties || {};
|
|
48
|
+
const extra = { office: props.gridId ?? "", grid_x: props.gridX ?? "", grid_y: props.gridY ?? "",
|
|
49
|
+
generated_at: fp.generatedAt ?? "", update_time: fp.updateTime ?? "" };
|
|
50
|
+
const rows = clean(fp.periods || [], ep).map((r) => ({ ...r, ...extra }));
|
|
51
|
+
return maxRows ? rows.slice(0, Math.trunc(Number(maxRows))) : rows;
|
|
52
|
+
}
|
|
53
|
+
async function fetch(spec, pack) {
|
|
54
|
+
const c = client(pack);
|
|
55
|
+
const tables = {}, notes = [];
|
|
56
|
+
for (const r of spec.requests) {
|
|
57
|
+
let [ep, params, probs] = resolve(pack, r);
|
|
58
|
+
if (probs.length)
|
|
59
|
+
throw new DataError(probs.join("; "), "bad_request");
|
|
60
|
+
if ("lat" in params)
|
|
61
|
+
params = round(params);
|
|
62
|
+
let rows;
|
|
63
|
+
if (ep.resolve)
|
|
64
|
+
rows = await forecast(c, ep, params, r.max_rows);
|
|
65
|
+
else {
|
|
66
|
+
const [got, meta] = await fetchRequest(c, ep, params, r.max_rows);
|
|
67
|
+
rows = got;
|
|
68
|
+
if (meta.truncated)
|
|
69
|
+
notes.push(`${r.name}: stopped at ${rows.length.toLocaleString("en-US")} rows (max_rows or page limit); the source has more.`);
|
|
70
|
+
}
|
|
71
|
+
rows = normalize(r, ep, rows);
|
|
72
|
+
if (!rows.length && !r.allow_empty) {
|
|
73
|
+
if (ep.path === "/alerts/active")
|
|
74
|
+
notes.push(`${r.name}: no active alerts matched (a valid answer: nothing is in effect now).`);
|
|
75
|
+
else
|
|
76
|
+
throw new DataError(`request ${repr(r.name)} returned no rows: check the parameters (see source.md)`, "bad_request");
|
|
77
|
+
}
|
|
78
|
+
tables[r.name] = rows;
|
|
79
|
+
}
|
|
80
|
+
return [tables, notes];
|
|
81
|
+
}
|
|
82
|
+
function normalize(_req, ep, rows) {
|
|
83
|
+
if (ep.records !== "features")
|
|
84
|
+
return rows;
|
|
85
|
+
return rows.map((r) => {
|
|
86
|
+
const n = {};
|
|
87
|
+
for (let [k, v] of Object.entries(r)) {
|
|
88
|
+
if (k.startsWith("properties."))
|
|
89
|
+
k = k.slice("properties.".length);
|
|
90
|
+
if (k.startsWith("@"))
|
|
91
|
+
continue;
|
|
92
|
+
n[k] = v;
|
|
93
|
+
}
|
|
94
|
+
const coords = "geometry.coordinates" in n ? n["geometry.coordinates"] : "";
|
|
95
|
+
delete n["geometry.coordinates"];
|
|
96
|
+
if (n["geometry.type"] === "Point" && coords) {
|
|
97
|
+
const parts = pyStr(coords).split(";").map((p) => p.trim());
|
|
98
|
+
if (parts.length >= 2)
|
|
99
|
+
[n.longitude, n.latitude] = [parts[0], parts[1]];
|
|
100
|
+
}
|
|
101
|
+
return n;
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
function manualItems(spec, pack) {
|
|
105
|
+
const items = grManualItems(pack, spec);
|
|
106
|
+
const eps = pack.endpoints?.endpoints || {};
|
|
107
|
+
for (const it of items) {
|
|
108
|
+
const req = (spec.requests || []).find((r) => r.name === it.request) || {};
|
|
109
|
+
const ep = eps[req.endpoint || ""] || {};
|
|
110
|
+
if (ep.resolve) {
|
|
111
|
+
it.note = `This link returns the point. Open the URL in its properties.${ep.resolve} field and save ` +
|
|
112
|
+
`THAT page as ${it.filename}.`;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return items;
|
|
116
|
+
}
|
|
117
|
+
const adapter = { fetch: (spec, pack) => fetch(spec, pack), normalize, manualItems };
|
|
118
|
+
export default adapter;
|
|
119
|
+
//# sourceMappingURL=nws.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"nws.js","sourceRoot":"","sources":["../../src/adapters/nws.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,IAAI,aAAa,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACvH,OAAO,EAAU,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEpD,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;AAErI,SAAS,MAAM,CAAC,IAAU;IACxB,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACrD,OAAO,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,wEAAwE;AACxE,SAAS,MAAM,CAAC,CAAS;IACvB,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACpB,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACzB,OAAO,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACtB,CAAC;AAED,gGAAgG;AAChG,SAAS,KAAK,CAAC,MAA+B;IAC5C,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,EAAE;YAAE,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACxJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,CAAS,EAAE,EAAO,EAAE,MAA+B,EAAE,OAAgB;IAC3F,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACjC,MAAM,KAAK,GAAG,KAAK,EAAE,UAAU,IAAI,EAAE,CAAC;IACtC,MAAM,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;IAC9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,SAAS,CAAC,kBAAkB,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,OAAO,GAAG;YAClG,yDAAyD,EAAE,aAAa,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,MAAM,EAAE,GAAG,EAAE,EAAE,UAAU,IAAI,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE;QAC9F,YAAY,EAAE,EAAE,CAAC,WAAW,IAAI,EAAE,EAAE,WAAW,EAAE,EAAE,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC;IACzE,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,OAAO,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;IAC1E,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,IAAU,EAAE,IAAU;IACzC,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IACvB,MAAM,MAAM,GAAW,EAAE,EAAE,KAAK,GAAa,EAAE,CAAC;IAChD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9B,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3C,IAAI,KAAK,CAAC,MAAM;YAAE,MAAM,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,aAAa,CAAC,CAAC;QACvE,IAAI,KAAK,IAAI,MAAM;YAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,IAAW,CAAC;QAChB,IAAI,EAAE,CAAC,OAAO;YAAE,IAAI,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC;aAC5D,CAAC;YACJ,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,MAAM,YAAY,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC;YAClE,IAAI,GAAG,GAAG,CAAC;YACX,IAAI,IAAI,CAAC,SAAS;gBAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,gBAAgB,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,sDAAsD,CAAC,CAAC;QACrJ,CAAC;QACD,IAAI,GAAG,SAAS,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,IAAI,EAAE,CAAC,IAAI,KAAK,gBAAgB;gBAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,wEAAwE,CAAC,CAAC;;gBAC3H,MAAM,IAAI,SAAS,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,yDAAyD,EAAE,aAAa,CAAC,CAAC;QAC5H,CAAC;QACD,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACxB,CAAC;IACD,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,SAAS,CAAC,IAAiB,EAAE,EAAO,EAAE,IAAW;IACxD,IAAI,EAAE,CAAC,OAAO,KAAK,UAAU;QAAE,OAAO,IAAI,CAAC;IAC3C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACpB,MAAM,CAAC,GAAQ,EAAE,CAAC;QAClB,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACrC,IAAI,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YACnE,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YAChC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACX,CAAC;QACD,MAAM,MAAM,GAAG,sBAAsB,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,OAAO,CAAC,CAAC,sBAAsB,CAAC,CAAC;QACjC,IAAI,CAAC,CAAC,eAAe,CAAC,KAAK,OAAO,IAAI,MAAM,EAAE,CAAC;YAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC5D,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;gBAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW,CAAC,IAAU,EAAE,IAAU;IACzC,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,EAAE,SAAS,IAAI,EAAE,CAAC;IAC5C,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,OAAO,CAAC,IAAK,EAAkB,CAAC;QAC5F,MAAM,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACzC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;YACf,EAAE,CAAC,IAAI,GAAG,+DAA+D,EAAE,CAAC,OAAO,kBAAkB;gBACnG,gBAAgB,EAAE,CAAC,QAAQ,GAAG,CAAC;QACnC,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;AAC9F,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { Adapter } from "../core/packs.js";
|
|
2
|
+
import { parseGrid, type Grid } from "./_omb_grid.js";
|
|
3
|
+
type Sheets = Record<string, Grid>;
|
|
4
|
+
/** [sheet name, grid] of the sheet whose title names Historical Table s.t, or [null, null]. */
|
|
5
|
+
declare function findTable(sheets: Sheets, table: string): [string | null, Grid | null];
|
|
6
|
+
declare function kindOf(body: Buffer): string;
|
|
7
|
+
export declare function normTable(v: unknown): string;
|
|
8
|
+
export declare const _test: {
|
|
9
|
+
parseGrid: typeof parseGrid;
|
|
10
|
+
normTable: typeof normTable;
|
|
11
|
+
kindOf: typeof kindOf;
|
|
12
|
+
findTable: typeof findTable;
|
|
13
|
+
};
|
|
14
|
+
declare const adapter: Adapter;
|
|
15
|
+
export default adapter;
|
|
@@ -0,0 +1,517 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* omb adapter: the President's Budget data files that OMB publishes through GovInfo (no API, no key).
|
|
3
|
+
*
|
|
4
|
+
* GovInfo serves each budget volume as a package BUDGET-<FY>-<PKG> with stable URLs (verified 2026-10-30):
|
|
5
|
+
* https://www.govinfo.gov/content/pkg/BUDGET-<FY>-<PKG>/xls/BUDGET-<FY>-<PKG>-<granule>.xlsx one file
|
|
6
|
+
* https://www.govinfo.gov/content/pkg/BUDGET-<FY>-<PKG>.zip the package
|
|
7
|
+
* A file that does not exist answers HTTP 302 to https://www.govinfo.gov/error, an HTML page with status 200, so
|
|
8
|
+
* every body is checked: a workbook or CSV is read, an HTML page means "not there".
|
|
9
|
+
* - Historical Tables (PKG=TAB): Table s.t is granule TAB-<s+1>-<t> (Table 1.1 = TAB-2-1). Not every edition
|
|
10
|
+
* follows it (FY2026 publishes one workbook; FY2015 and earlier use other granule numbers and .xls files), so
|
|
11
|
+
* the granule is tried first and its title row must name the table; otherwise the package zip is read and the
|
|
12
|
+
* table found by its title.
|
|
13
|
+
* - Public Budget Database (PKG=DB): granule 1 budget authority, 2 outlays, 3 receipts (xlsx; older years such
|
|
14
|
+
* as FY2015 publish them as csv/BUDGET-<FY>-DB-<n>.csv only). FY2026 has no DB package on GovInfo.
|
|
15
|
+
* - Federal Credit Supplement (PKG=FCS): one workbook BUDGET-<FY>-FCS.xlsx with sheets Table1 ... Table10 (FY2024
|
|
16
|
+
* on); FY2018-FY2023 publish one granule per table, BUDGET-<FY>-FCS-<n>.xlsx.
|
|
17
|
+
* A sheet is read as published (see _omb_grid.ts).
|
|
18
|
+
*/
|
|
19
|
+
import fs from "node:fs";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
import { parse } from "csv-parse/sync";
|
|
22
|
+
import { unzipSync } from "fflate";
|
|
23
|
+
import * as gr from "../core/generic-rest.js";
|
|
24
|
+
import { DataError } from "../core/http.js";
|
|
25
|
+
import { decodeText, xlsSheets, xlsxSheets } from "../core/tabular.js";
|
|
26
|
+
import { parseGrid, txt } from "./_omb_grid.js";
|
|
27
|
+
import { comma, isDigits, naturalCompare, pyInt, pyRepr, pyStrip, toTable } from "./_omb_util.js";
|
|
28
|
+
const BASE = "https://www.govinfo.gov/content/pkg";
|
|
29
|
+
const DB_KINDS = { budget_authority: 1, outlays: 2, receipts: 3 };
|
|
30
|
+
const DB_UNITS = "thousands of dollars";
|
|
31
|
+
const FIRST_FY = 1996; // GovInfo's BUDGET collection starts with the FY1996 budget
|
|
32
|
+
const WH_HIST = "https://www.whitehouse.gov/omb/information-resources/budget/historical-tables/";
|
|
33
|
+
const WH_SUPP = "https://www.whitehouse.gov/omb/information-resources/budget/supplemental-materials/";
|
|
34
|
+
/** The edition (package) does not exist on GovInfo: the next older edition may be tried. */
|
|
35
|
+
class NoPackage extends DataError {
|
|
36
|
+
constructor(msg) {
|
|
37
|
+
super(msg, "not_found", 404);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
const empty = (v) => v === null || v === undefined || v === "";
|
|
41
|
+
// ---------------------------------------------------------------------- reading published grids
|
|
42
|
+
function firstText(grid, rows = 4) {
|
|
43
|
+
for (const r of grid.slice(0, rows)) {
|
|
44
|
+
for (const v of r) {
|
|
45
|
+
const t = txt(v);
|
|
46
|
+
if (t && t.toLowerCase() !== "return to table of contents")
|
|
47
|
+
return t;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return "";
|
|
51
|
+
}
|
|
52
|
+
function titlePat(table) {
|
|
53
|
+
const [s, t] = table.split(".");
|
|
54
|
+
return new RegExp(`^\\s*Table\\s+${Number(s)}\\.${Number(t)}(?![\\d.])`, "i");
|
|
55
|
+
}
|
|
56
|
+
/** [sheet name, grid] of the sheet whose title names Historical Table s.t, or [null, null]. */
|
|
57
|
+
function findTable(sheets, table) {
|
|
58
|
+
const pat = titlePat(table);
|
|
59
|
+
for (const [name, grid] of Object.entries(sheets))
|
|
60
|
+
if (pat.test(firstText(grid)))
|
|
61
|
+
return [name, grid];
|
|
62
|
+
return [null, null];
|
|
63
|
+
}
|
|
64
|
+
function titles(sheets) {
|
|
65
|
+
const out = [];
|
|
66
|
+
for (const g of Object.values(sheets)) {
|
|
67
|
+
const m = /^\s*Table\s+(\d+\.\d+)/i.exec(firstText(g));
|
|
68
|
+
if (m)
|
|
69
|
+
out.push(m[1]);
|
|
70
|
+
}
|
|
71
|
+
return out;
|
|
72
|
+
}
|
|
73
|
+
// ---------------------------------------------------------------------- bodies
|
|
74
|
+
function zipNames(body) {
|
|
75
|
+
const names = [];
|
|
76
|
+
try {
|
|
77
|
+
unzipSync(body, { filter: (f) => (names.push(f.name), false) });
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
return names;
|
|
83
|
+
}
|
|
84
|
+
const XLS_MAGIC = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]);
|
|
85
|
+
function kindOf(body) {
|
|
86
|
+
if (body.subarray(0, 4).equals(Buffer.from("PK\x03\x04", "latin1"))) {
|
|
87
|
+
const names = zipNames(body);
|
|
88
|
+
if (names === null)
|
|
89
|
+
return "other";
|
|
90
|
+
return names.includes("xl/workbook.xml") ? "xlsx" : "zip";
|
|
91
|
+
}
|
|
92
|
+
if (body.subarray(0, 8).equals(XLS_MAGIC))
|
|
93
|
+
return "xls";
|
|
94
|
+
const head = body.subarray(0, 400).toString("latin1").replace(/^[ \t\n\r\v\f]+/, "").toLowerCase();
|
|
95
|
+
if (head.startsWith("<!doctype html") || head.startsWith("<html") || head.includes("<html"))
|
|
96
|
+
return "html";
|
|
97
|
+
return "text";
|
|
98
|
+
}
|
|
99
|
+
function sheetsOf(body, kind) {
|
|
100
|
+
if (kind === "xlsx")
|
|
101
|
+
return xlsxSheets(body);
|
|
102
|
+
if (kind === "xls")
|
|
103
|
+
return xlsSheets(body);
|
|
104
|
+
throw new DataError(`expected a workbook, got ${kind}`, "api");
|
|
105
|
+
}
|
|
106
|
+
function csvGrid(body) {
|
|
107
|
+
const text = decodeText(body);
|
|
108
|
+
return parse(text, { relax_column_count: true, relax_quotes: true, skip_empty_lines: false });
|
|
109
|
+
}
|
|
110
|
+
/** [body, kind] of a GovInfo file, or [null, "missing"] when GovInfo answers its error page or 404. */
|
|
111
|
+
async function get(c, url) {
|
|
112
|
+
let body;
|
|
113
|
+
try {
|
|
114
|
+
body = (await c.get(url)).body;
|
|
115
|
+
}
|
|
116
|
+
catch (e) {
|
|
117
|
+
if (e instanceof DataError && e.kind === "not_found")
|
|
118
|
+
return [null, "missing"];
|
|
119
|
+
throw e;
|
|
120
|
+
}
|
|
121
|
+
const k = kindOf(body);
|
|
122
|
+
return k === "html" ? [null, "missing"] : [body, k];
|
|
123
|
+
}
|
|
124
|
+
function* zipMembers(body) {
|
|
125
|
+
const z = unzipSync(body);
|
|
126
|
+
for (const n of Object.keys(z).sort(naturalCompare)) {
|
|
127
|
+
const low = n.toLowerCase();
|
|
128
|
+
if ((low.endsWith(".xlsx") || low.endsWith(".xls")) && !n.endsWith("/"))
|
|
129
|
+
yield [n, z[n]];
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
const memberKind = (member) => (member.toLowerCase().endsWith(".xls") ? "xls" : "xlsx");
|
|
133
|
+
async function histTable(c, fy, table) {
|
|
134
|
+
const [s, t] = table.split(".").map(Number);
|
|
135
|
+
const url = `${BASE}/BUDGET-${fy}-TAB/xls/BUDGET-${fy}-TAB-${s + 1}-${t}.xlsx`;
|
|
136
|
+
const notes = [];
|
|
137
|
+
const [body, k] = await get(c, url);
|
|
138
|
+
if (body !== null && (k === "xlsx" || k === "xls")) {
|
|
139
|
+
const [, grid] = findTable(sheetsOf(body, k), table);
|
|
140
|
+
if (grid !== null)
|
|
141
|
+
return [grid, url, notes];
|
|
142
|
+
notes.push(`${url} holds ${pyRepr(firstText(Object.values(sheetsOf(body, k))[0]).slice(0, 60))}, not Table ` +
|
|
143
|
+
`${table}: the table was looked up by its title in the package zip`);
|
|
144
|
+
}
|
|
145
|
+
const zurl = `${BASE}/BUDGET-${fy}-TAB.zip`;
|
|
146
|
+
const [zbody, zk] = await get(c, zurl);
|
|
147
|
+
if (zbody === null || zk !== "zip") {
|
|
148
|
+
throw new NoPackage(`GovInfo has no Historical Tables for the FY${fy} budget (BUDGET-${fy}-TAB): ${url} and ` +
|
|
149
|
+
`${zurl} answer GovInfo's error page`);
|
|
150
|
+
}
|
|
151
|
+
const found = [];
|
|
152
|
+
for (const [member, data] of zipMembers(zbody)) {
|
|
153
|
+
const sh = sheetsOf(data, memberKind(member));
|
|
154
|
+
const [name, grid] = findTable(sh, table);
|
|
155
|
+
if (grid !== null) {
|
|
156
|
+
const multi = Object.keys(sh).length > 1;
|
|
157
|
+
if (body === null) {
|
|
158
|
+
notes.push(`the FY${fy} edition has no file ${url.split("/").pop()} (GovInfo answers its error ` +
|
|
159
|
+
`page): Table ${table} was read from ${member} in ${zurl}` + (multi ? `, sheet ${name}` : ""));
|
|
160
|
+
}
|
|
161
|
+
return [grid, `${zurl}#${member}` + (multi ? `!${name}` : ""), notes];
|
|
162
|
+
}
|
|
163
|
+
found.push(...titles(sh));
|
|
164
|
+
}
|
|
165
|
+
const key = (x) => x.split(".").map(Number);
|
|
166
|
+
const have = [...new Set(found)].sort((a, b) => {
|
|
167
|
+
const [ka, kb] = [key(a), key(b)];
|
|
168
|
+
for (let i = 0; i < Math.min(ka.length, kb.length); i++)
|
|
169
|
+
if (ka[i] !== kb[i])
|
|
170
|
+
return ka[i] - kb[i];
|
|
171
|
+
return ka.length - kb.length;
|
|
172
|
+
});
|
|
173
|
+
throw new DataError(`Table ${table} is not in the FY${fy} Historical Tables (GovInfo package BUDGET-${fy}-TAB); ` +
|
|
174
|
+
`its tables: ${have.join(", ") || "none found"}. Check the number against source.md (Table ` +
|
|
175
|
+
"s.t, e.g. 1.1, 3.2, 8.5)", "not_found", 404);
|
|
176
|
+
}
|
|
177
|
+
async function budgetDb(c, fy, kind) {
|
|
178
|
+
const n = DB_KINDS[kind];
|
|
179
|
+
const tried = [];
|
|
180
|
+
for (const [url, fmt] of [[`${BASE}/BUDGET-${fy}-DB/xls/BUDGET-${fy}-DB-${n}.xlsx`, "xlsx"],
|
|
181
|
+
[`${BASE}/BUDGET-${fy}-DB/csv/BUDGET-${fy}-DB-${n}.csv`, "csv"]]) {
|
|
182
|
+
const [body, k] = await get(c, url);
|
|
183
|
+
tried.push(url);
|
|
184
|
+
if (body === null)
|
|
185
|
+
continue;
|
|
186
|
+
if (fmt === "csv" && k === "text")
|
|
187
|
+
return [csvGrid(body), url, [`FY${fy} publishes this file as CSV only (${url})`]];
|
|
188
|
+
if (k === "xlsx" || k === "xls")
|
|
189
|
+
return [Object.values(sheetsOf(body, k))[0], url, []];
|
|
190
|
+
}
|
|
191
|
+
throw new NoPackage(`GovInfo has no Public Budget Database ${kind} file for the FY${fy} budget (${tried.join(" and ")} ` +
|
|
192
|
+
"answer GovInfo's error page; the FY2026 budget published no database there)");
|
|
193
|
+
}
|
|
194
|
+
function fcsN(table) {
|
|
195
|
+
const m = /^\s*(?:table\s*)?(\d{1,2})\s*$/i.exec(pyStrFor(table));
|
|
196
|
+
if (!m) {
|
|
197
|
+
throw new DataError(`credit_supplement table ${pyRepr(table)}: give the table number, e.g. 1 (Table 1)`, "bad_request");
|
|
198
|
+
}
|
|
199
|
+
return Number(m[1]);
|
|
200
|
+
}
|
|
201
|
+
/** str(v) as Python writes it for the values a spec carries. */
|
|
202
|
+
function pyStrFor(v) {
|
|
203
|
+
if (v === null || v === undefined)
|
|
204
|
+
return "None";
|
|
205
|
+
if (v === true)
|
|
206
|
+
return "True";
|
|
207
|
+
if (v === false)
|
|
208
|
+
return "False";
|
|
209
|
+
return String(v);
|
|
210
|
+
}
|
|
211
|
+
function fcsSheet(sheets, n) {
|
|
212
|
+
for (const [name, grid] of Object.entries(sheets)) {
|
|
213
|
+
if (new RegExp(`^table\\s*${n}$`, "i").test(name.trim()))
|
|
214
|
+
return [name, grid];
|
|
215
|
+
}
|
|
216
|
+
const pat = new RegExp(`^\\s*Table\\s+${n}\\.`, "i");
|
|
217
|
+
for (const [name, grid] of Object.entries(sheets))
|
|
218
|
+
if (pat.test(firstText(grid)))
|
|
219
|
+
return [name, grid];
|
|
220
|
+
return [null, null];
|
|
221
|
+
}
|
|
222
|
+
async function creditTable(c, fy, table) {
|
|
223
|
+
const n = fcsN(table);
|
|
224
|
+
const url = `${BASE}/BUDGET-${fy}-FCS/xls/BUDGET-${fy}-FCS.xlsx`;
|
|
225
|
+
let [body, k] = await get(c, url);
|
|
226
|
+
if (body !== null && (k === "xlsx" || k === "xls")) {
|
|
227
|
+
const sh = sheetsOf(body, k);
|
|
228
|
+
const [name, grid] = fcsSheet(sh, n);
|
|
229
|
+
if (grid === null) {
|
|
230
|
+
throw new DataError(`the FY${fy} Federal Credit Supplement has no Table ${n} (sheets: ${Object.keys(sh).join(", ")})`, "not_found", 404);
|
|
231
|
+
}
|
|
232
|
+
return [grid, `${url}!${name}`, []];
|
|
233
|
+
}
|
|
234
|
+
const url2 = `${BASE}/BUDGET-${fy}-FCS/xls/BUDGET-${fy}-FCS-${n}.xlsx`;
|
|
235
|
+
[body, k] = await get(c, url2);
|
|
236
|
+
if (body !== null && (k === "xlsx" || k === "xls")) {
|
|
237
|
+
const sh = sheetsOf(body, k);
|
|
238
|
+
let [, grid] = fcsSheet(sh, n);
|
|
239
|
+
if (grid === null)
|
|
240
|
+
grid = Object.values(sh)[0];
|
|
241
|
+
return [grid, url2, [`FY${fy} publishes one file per credit supplement table (${url2})`]];
|
|
242
|
+
}
|
|
243
|
+
throw new NoPackage(`GovInfo has no Federal Credit Supplement Table ${n} for the FY${fy} budget (${url} and ${url2} ` +
|
|
244
|
+
"answer GovInfo's error page)");
|
|
245
|
+
}
|
|
246
|
+
// ---------------------------------------------------------------------- spec handling
|
|
247
|
+
function fyCandidates(params) {
|
|
248
|
+
if (!empty(params.fy))
|
|
249
|
+
return [pyInt(params.fy)];
|
|
250
|
+
const y = new Date().getFullYear();
|
|
251
|
+
return [y + 1, y, y - 1]; // the budget for FY y+1 is released early in calendar year y
|
|
252
|
+
}
|
|
253
|
+
export function normTable(v) {
|
|
254
|
+
const m = /^\s*(?:table\s*)?(\d{1,2})\s*[.\-_ ]\s*(\d{1,2})\s*$/i.exec(pyStrFor(v));
|
|
255
|
+
return m ? `${Number(m[1])}.${Number(m[2])}` : "";
|
|
256
|
+
}
|
|
257
|
+
function checkSpec(spec) {
|
|
258
|
+
const probs = [];
|
|
259
|
+
const hi = new Date().getFullYear() + 1;
|
|
260
|
+
for (const r of spec.requests || []) {
|
|
261
|
+
const p = r.params || {}, n = pyRepr(r.name);
|
|
262
|
+
if (!empty(p.fy)) {
|
|
263
|
+
try {
|
|
264
|
+
const fy = pyInt(p.fy);
|
|
265
|
+
if (!(FIRST_FY <= fy && fy <= hi)) {
|
|
266
|
+
probs.push(`request ${n}: fy ${fy} is outside ${FIRST_FY}-${hi} (the budget's fiscal year, ` +
|
|
267
|
+
"e.g. 2027 for the budget released in 2026)");
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
catch {
|
|
271
|
+
probs.push(`request ${n}: fy is a four-digit budget fiscal year, e.g. 2027`);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
for (const k of ["header_row", "header_rows"]) {
|
|
275
|
+
if (!empty(p[k]) && !isDigits(pyStrFor(p[k])))
|
|
276
|
+
probs.push(`request ${n}: ${k} is a positive whole number`);
|
|
277
|
+
}
|
|
278
|
+
const ep = r.endpoint;
|
|
279
|
+
if (ep === "historical_table" && !empty(p.table) && !normTable(p.table) && !pyStrFor(p.table).startsWith("table_")) {
|
|
280
|
+
probs.push(`request ${n}: table ${pyRepr(p.table)} is not a Historical Table number like "1.1" or ` +
|
|
281
|
+
'"8.5" (section.table)');
|
|
282
|
+
}
|
|
283
|
+
if (ep === "credit_supplement" && !empty(p.table)) {
|
|
284
|
+
try {
|
|
285
|
+
fcsN(p.table);
|
|
286
|
+
}
|
|
287
|
+
catch (e) {
|
|
288
|
+
probs.push(`request ${n}: ${e.message}`);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
return probs;
|
|
293
|
+
}
|
|
294
|
+
/** [grid, url, notes, fy, label] for one request, trying older editions when fy is not given. */
|
|
295
|
+
async function one(c, epName, params) {
|
|
296
|
+
let last = null;
|
|
297
|
+
for (const fy of fyCandidates(params)) {
|
|
298
|
+
try {
|
|
299
|
+
if (epName === "historical_table") {
|
|
300
|
+
const [g, u, n] = await histTable(c, fy, normTable(params.table));
|
|
301
|
+
return [g, u, n, fy, `Historical Table ${normTable(params.table)}`];
|
|
302
|
+
}
|
|
303
|
+
if (epName === "budget_database") {
|
|
304
|
+
const [g, u, n] = await budgetDb(c, fy, params.kind);
|
|
305
|
+
return [g, u, n, fy, `Public Budget Database, ${String(params.kind).replace(/_/g, " ")}`];
|
|
306
|
+
}
|
|
307
|
+
if (epName === "credit_supplement") {
|
|
308
|
+
const [g, u, n] = await creditTable(c, fy, params.table);
|
|
309
|
+
return [g, u, n, fy, `Federal Credit Supplement Table ${fcsN(params.table)}`];
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
catch (e) {
|
|
313
|
+
if (!(e instanceof NoPackage))
|
|
314
|
+
throw e;
|
|
315
|
+
last = e;
|
|
316
|
+
if (!empty(params.fy))
|
|
317
|
+
throw e;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
throw last ?? new DataError(`unknown endpoint ${pyRepr(epName)}`, "bad_request");
|
|
321
|
+
}
|
|
322
|
+
const UNIT_BLOCK = /^(in|as)\b.*\b(dollars|percent|percentages)\b.*:$/i;
|
|
323
|
+
/**
|
|
324
|
+
* Tables 3.1 and 6.1 stack blocks in different units under label rows ("In millions of dollars:", "As
|
|
325
|
+
* percentages of GDP:"), so "Total, Federal outlays" appears once per block: add `units_block` (the label of the
|
|
326
|
+
* block a row is in) when a table has such label rows.
|
|
327
|
+
*/
|
|
328
|
+
function unitBlocks(rows) {
|
|
329
|
+
if (!rows.length)
|
|
330
|
+
return rows;
|
|
331
|
+
const first = rows[0].keys().next().value;
|
|
332
|
+
const blank = (r) => ![...r].some(([k, v]) => k !== first && k !== "sheet_row" && v);
|
|
333
|
+
const label = (r) => pyStrip(String(r.get(first)));
|
|
334
|
+
if (!rows.some((r) => blank(r) && UNIT_BLOCK.test(label(r))))
|
|
335
|
+
return rows;
|
|
336
|
+
let cur = "";
|
|
337
|
+
return rows.map((r) => {
|
|
338
|
+
if (blank(r) && UNIT_BLOCK.test(label(r)))
|
|
339
|
+
cur = label(r).replace(/:+$/, "");
|
|
340
|
+
const m = new Map([...r].filter(([k]) => k !== "sheet_row"));
|
|
341
|
+
m.set("units_block", cur);
|
|
342
|
+
m.set("sheet_row", r.get("sheet_row"));
|
|
343
|
+
return m;
|
|
344
|
+
});
|
|
345
|
+
}
|
|
346
|
+
function shape(epName, grid, params) {
|
|
347
|
+
const [rows, meta] = parseGrid(grid, params.header_row, params.header_rows, { stripDots: epName !== "budget_database", tocCells: true });
|
|
348
|
+
return [epName === "historical_table" ? unitBlocks(rows) : rows, meta];
|
|
349
|
+
}
|
|
350
|
+
function notesFor(name, epName, label, fy, url, meta, rows) {
|
|
351
|
+
const out = [];
|
|
352
|
+
const top = meta.top.join(" | ").slice(0, 400);
|
|
353
|
+
const head = meta.header;
|
|
354
|
+
const span = !!head && head[1] > head[0];
|
|
355
|
+
out.push(`${name}: ${label}, FY${fy} budget (${url})` + (top ? `: ${top}` : "") +
|
|
356
|
+
(head ? `; header row${span ? "s" : ""} ${head[0]}${span ? "-" + head[1] : ""} of the sheet` : ""));
|
|
357
|
+
if (epName === "budget_database") {
|
|
358
|
+
out.push(`${name}: amounts are in ${DB_UNITS}; columns 1962 (1976 for budget authority) ... are fiscal ` +
|
|
359
|
+
`years and TQ is the 1976 transition quarter; years after FY${fy - 2} are estimates (FY${fy - 1} ` +
|
|
360
|
+
`the current-year estimate, FY${fy} on the President's proposals)`);
|
|
361
|
+
}
|
|
362
|
+
if (epName === "historical_table" && rows.length) {
|
|
363
|
+
const first = rows[0].keys().next().value;
|
|
364
|
+
const estRows = rows.filter((r) => String(r.get(first) ?? "").toLowerCase().includes("estimate")).map((r) => String(r.get(first)));
|
|
365
|
+
const estCols = [...rows[0].keys()].filter((k) => k.toLowerCase().includes("estimate"));
|
|
366
|
+
if (estRows.length || estCols.length) {
|
|
367
|
+
const e = estRows.length ? estRows : estCols;
|
|
368
|
+
out.push(`${name}: ${e.length} ${estRows.length ? "rows" : "columns"} marked estimate (${e[0]} to ${e[e.length - 1]}): ` +
|
|
369
|
+
"projections of the President's budget, not actual results; the budget year and later are proposals");
|
|
370
|
+
}
|
|
371
|
+
else
|
|
372
|
+
out.push(`${name}: no estimate years in this table (this edition's table ends with actual years)`);
|
|
373
|
+
}
|
|
374
|
+
if (meta.footer.length)
|
|
375
|
+
out.push(`${name}: footnotes: ` + meta.footer.join(" | ").slice(0, 1200));
|
|
376
|
+
return out;
|
|
377
|
+
}
|
|
378
|
+
async function fetch(spec, pack, _route) {
|
|
379
|
+
const c = gr.clientFor(pack);
|
|
380
|
+
const tables = {};
|
|
381
|
+
const notes = [];
|
|
382
|
+
const cap = Number(pack.policy.limits.max_rows || 50000);
|
|
383
|
+
for (const r of spec.requests) {
|
|
384
|
+
const [, params, probs] = gr.resolve(pack, r);
|
|
385
|
+
if (probs.length)
|
|
386
|
+
throw new DataError(probs.join("; "), "bad_request");
|
|
387
|
+
let epName = r.endpoint || "";
|
|
388
|
+
if (!epName)
|
|
389
|
+
epName = pack.endpoints?.catalog?.[r.dataset ?? ""]?.endpoint ?? "";
|
|
390
|
+
let got;
|
|
391
|
+
try {
|
|
392
|
+
got = await one(c, epName, params);
|
|
393
|
+
}
|
|
394
|
+
catch (e) {
|
|
395
|
+
if (!(e instanceof DataError))
|
|
396
|
+
throw e;
|
|
397
|
+
throw new DataError(`request ${pyRepr(r.name)} (${epName}): ${e.message}`, e.kind, e.status);
|
|
398
|
+
}
|
|
399
|
+
const [grid, url, extra, fy, label] = got;
|
|
400
|
+
let [rows, meta] = shape(epName, grid, params);
|
|
401
|
+
if (!rows.length && !r.allow_empty) {
|
|
402
|
+
throw new DataError(`request ${pyRepr(r.name)}: ${label} (FY${fy}, ${url}) has no data rows under the detected ` +
|
|
403
|
+
"header: set header_row (the 1-based row of the column names)", "bad_request");
|
|
404
|
+
}
|
|
405
|
+
const limit = Math.min(Number(r.max_rows || cap), cap);
|
|
406
|
+
if (rows.length > limit) {
|
|
407
|
+
notes.push(`${r.name}: stopped at ${comma(limit)} rows (max_rows) of ${comma(rows.length)}. Raise max_rows.`);
|
|
408
|
+
rows = rows.slice(0, limit);
|
|
409
|
+
}
|
|
410
|
+
notes.push(...extra.map((x) => `${r.name}: ${x}`), ...notesFor(r.name, epName, label, fy, url, meta, rows));
|
|
411
|
+
tables[r.name] = rows;
|
|
412
|
+
}
|
|
413
|
+
return [Object.fromEntries(Object.entries(tables).map(([k, v]) => [k, toTable(v, "omb")])), notes];
|
|
414
|
+
}
|
|
415
|
+
// ---------------------------------------------------------------------- manual route
|
|
416
|
+
function manualItems(spec, pack) {
|
|
417
|
+
const items = [];
|
|
418
|
+
const y = new Date().getFullYear();
|
|
419
|
+
for (const r of spec.requests || []) {
|
|
420
|
+
const [, params, probs] = gr.resolve(pack, r);
|
|
421
|
+
if (probs.length)
|
|
422
|
+
continue;
|
|
423
|
+
const name = r.name, epn = r.endpoint;
|
|
424
|
+
const fy = !empty(params.fy) ? pyInt(params.fy) : y + 1;
|
|
425
|
+
let url, note, ext;
|
|
426
|
+
if (epn === "historical_table") {
|
|
427
|
+
const t = normTable(params.table);
|
|
428
|
+
if (!t)
|
|
429
|
+
continue;
|
|
430
|
+
const [s, tt] = t.split(".");
|
|
431
|
+
url = `${BASE}/BUDGET-${fy}-TAB/xls/BUDGET-${fy}-TAB-${Number(s) + 1}-${tt}.xlsx`;
|
|
432
|
+
note = `Table ${t} of the FY${fy} Historical Tables. If GovInfo shows an error page (some editions, e.g. ` +
|
|
433
|
+
`FY2026, have one workbook instead), download ${BASE}/BUDGET-${fy}-TAB.zip and upload it as ` +
|
|
434
|
+
`${name}.zip, or save hist${String(Number(s)).padStart(2, "0")}z${tt}_fy${fy}.xlsx from ${WH_HIST} as ${name}.xlsx`;
|
|
435
|
+
ext = [".zip", ".xls"];
|
|
436
|
+
}
|
|
437
|
+
else if (epn === "budget_database") {
|
|
438
|
+
const n = DB_KINDS[String(params.kind)] ?? 2;
|
|
439
|
+
url = `${BASE}/BUDGET-${fy}-DB/xls/BUDGET-${fy}-DB-${n}.xlsx`;
|
|
440
|
+
note = `Public Budget Database (${pyStrFor(params.kind)}), FY${fy}. Older editions publish CSV: ` +
|
|
441
|
+
`${BASE}/BUDGET-${fy}-DB/csv/BUDGET-${fy}-DB-${n}.csv (upload as ${name}.csv). The same file is on ` +
|
|
442
|
+
`${WH_SUPP} (budauth/outlays/receipts_fy${fy}.xlsx)`;
|
|
443
|
+
ext = [".csv"];
|
|
444
|
+
}
|
|
445
|
+
else {
|
|
446
|
+
let n;
|
|
447
|
+
try {
|
|
448
|
+
n = fcsN(params.table);
|
|
449
|
+
}
|
|
450
|
+
catch {
|
|
451
|
+
continue;
|
|
452
|
+
}
|
|
453
|
+
url = `${BASE}/BUDGET-${fy}-FCS/xls/BUDGET-${fy}-FCS.xlsx`;
|
|
454
|
+
note = `Federal Credit Supplement, FY${fy} (the pack reads Table ${n}). FY2018-FY2023: ` +
|
|
455
|
+
`${BASE}/BUDGET-${fy}-FCS/xls/BUDGET-${fy}-FCS-${n}.xlsx. Also on ${WH_SUPP} (cr_supp_fy${fy}.xlsx)`;
|
|
456
|
+
ext = [];
|
|
457
|
+
}
|
|
458
|
+
items.push({ name: `${name}#1`, request: name, url, filename: `${name}.xlsx`, also_ext: ext, needs_key: false,
|
|
459
|
+
method: "GET", body: null, note });
|
|
460
|
+
}
|
|
461
|
+
return items;
|
|
462
|
+
}
|
|
463
|
+
function parseUpload(pack, req, paths) {
|
|
464
|
+
const [, params, probs] = gr.resolve(pack, req);
|
|
465
|
+
if (probs.length)
|
|
466
|
+
throw new DataError(probs.join("; "), "bad_request");
|
|
467
|
+
const epn = req.endpoint || "";
|
|
468
|
+
const rows = [];
|
|
469
|
+
for (const p of paths) {
|
|
470
|
+
const body = fs.readFileSync(p);
|
|
471
|
+
const k = kindOf(body);
|
|
472
|
+
const base = path.basename(p);
|
|
473
|
+
let grid = null;
|
|
474
|
+
if (epn === "historical_table") {
|
|
475
|
+
const t = normTable(params.table);
|
|
476
|
+
if (k === "xlsx" || k === "xls")
|
|
477
|
+
grid = findTable(sheetsOf(body, k), t)[1];
|
|
478
|
+
else if (k === "zip") {
|
|
479
|
+
for (const [member, data] of zipMembers(body)) {
|
|
480
|
+
grid = findTable(sheetsOf(data, memberKind(member)), t)[1];
|
|
481
|
+
if (grid !== null)
|
|
482
|
+
break;
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
if (grid === null) {
|
|
486
|
+
throw new DataError(`${base}: no sheet titled "Table ${t} - ..." (upload the Historical Tables file ` +
|
|
487
|
+
"for that table, or the package zip)", "bad_request");
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
else if (epn === "budget_database") {
|
|
491
|
+
if (k === "text")
|
|
492
|
+
grid = csvGrid(body);
|
|
493
|
+
else if (k === "xlsx" || k === "xls")
|
|
494
|
+
grid = Object.values(sheetsOf(body, k))[0];
|
|
495
|
+
else
|
|
496
|
+
throw new DataError(`${base}: not a Public Budget Database .xlsx or .csv file`, "bad_request");
|
|
497
|
+
}
|
|
498
|
+
else {
|
|
499
|
+
if (k !== "xlsx" && k !== "xls")
|
|
500
|
+
throw new DataError(`${base}: not a Federal Credit Supplement workbook`, "bad_request");
|
|
501
|
+
const sh = sheetsOf(body, k);
|
|
502
|
+
const n = fcsN(params.table);
|
|
503
|
+
grid = fcsSheet(sh, n)[1];
|
|
504
|
+
if (grid === null && Object.keys(sh).length === 1)
|
|
505
|
+
grid = Object.values(sh)[0];
|
|
506
|
+
if (grid === null)
|
|
507
|
+
throw new DataError(`${base}: no sheet Table${n} (sheets: ${Object.keys(sh).join(", ")})`, "bad_request");
|
|
508
|
+
}
|
|
509
|
+
rows.push(...shape(epn, grid, params)[0]);
|
|
510
|
+
}
|
|
511
|
+
// a FileTable when the columns are years (package.write takes either)
|
|
512
|
+
return toTable(rows, "omb");
|
|
513
|
+
}
|
|
514
|
+
export const _test = { parseGrid, normTable, kindOf, findTable };
|
|
515
|
+
const adapter = { fetch, checkSpec, manualItems, parseUpload };
|
|
516
|
+
export default adapter;
|
|
517
|
+
//# sourceMappingURL=omb.js.map
|