data-analyst-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -2
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +716 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +24 -0
- package/dist/report/scan_secrets.js +265 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +247 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +54 -4
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +170 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { Client } from "../core/http.js";
|
|
2
|
+
import type { Adapter, Pack, Row, Spec, SpecRequest, Tables } from "../core/packs.js";
|
|
3
|
+
import { El } from "./_govinfo_xml.js";
|
|
4
|
+
/** Hooks the tests replace (the Python tests patch the module's gr.fetch and _client). */
|
|
5
|
+
export declare const internals: {
|
|
6
|
+
fetchSpec: (pack: Pack, spec: Spec, client?: Client) => Promise<[Tables, string[]]>;
|
|
7
|
+
client: (base: string, name: string) => Client;
|
|
8
|
+
};
|
|
9
|
+
/** The Congress whose term includes a date (January 3 since 1935, March 4 before). today: [year, month, day]. */
|
|
10
|
+
export declare function currentCongress(today?: [number, number, number]): number;
|
|
11
|
+
export declare function congressYears(n: number): [number, number];
|
|
12
|
+
/** Calendar year of a regular session (session 1 = the odd year, session 2 = the even year). */
|
|
13
|
+
export declare function sessionYear(congress: unknown, session: unknown): number;
|
|
14
|
+
export declare function ordinal(n0: unknown): string;
|
|
15
|
+
/** title_contains -> a test on a title, or null. Each phrase matches as given: its words in that order, each at the
|
|
16
|
+
* START of a word, any case, any spacing or punctuation between words. A list matches any of its phrases. */
|
|
17
|
+
export declare function titleMatcher(value: unknown): ((title: unknown) => boolean) | null;
|
|
18
|
+
/** For a multi-word title_contains that kept nothing: the words must stand together and in that order. */
|
|
19
|
+
export declare function phraseOrderHint(req: SpecRequest): string | null;
|
|
20
|
+
/** title_contains on a bill or law list reads Congress.gov's display title only: say so, with the title search. */
|
|
21
|
+
export declare function displayTitleNote(req: SpecRequest): string | null;
|
|
22
|
+
export declare function houseUrl(year: unknown, roll: unknown): string;
|
|
23
|
+
export declare function senateUrl(congress: unknown, session: unknown, roll: unknown): string;
|
|
24
|
+
export declare function senateMenuUrl(congress: unknown, session: unknown): string;
|
|
25
|
+
/** Python's refused_hosts: when the spec needs Senate vote XML, try senate.gov once; a refusal is returned as the
|
|
26
|
+
* error a fetch would raise (a route check can run it before the fetch). */
|
|
27
|
+
export declare function refusedHostsProbe(spec: Spec): Promise<string>;
|
|
28
|
+
export declare function parseHouse(root: El, url: string): [Row, Row[]];
|
|
29
|
+
export declare function parseSenate(root: El, url: string): [Row, Row[]];
|
|
30
|
+
export declare function parseVoteXml(root: El, url: string): [Row, Row[]];
|
|
31
|
+
export declare function parseSenateMenu(root: El, _url: string): Row[];
|
|
32
|
+
declare const adapter: Adapter;
|
|
33
|
+
export default adapter;
|
|
@@ -0,0 +1,589 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Congress.gov pack hooks, plus roll-call votes from the House Clerk and Senate XML files.
|
|
3
|
+
*
|
|
4
|
+
* - beforeRequest: "congress": "current" becomes the current Congress number (a Congress starts at noon on
|
|
5
|
+
* January 3 of an odd year); title_contains is removed (applied in normalize); a "+" in sort becomes a space
|
|
6
|
+
* (Congress.gov silently ignores an encoded "+").
|
|
7
|
+
* - normalize: title_contains filter (the phrase as given, each word matching at a word start, any case; a list
|
|
8
|
+
* matches any of its phrases); text_plain added to CRS summaries (HTML stripped).
|
|
9
|
+
* - fetch: roll_call, senate_votes and bill_votes read the official XML (clerk.house.gov 1990 on, www.senate.gov
|
|
10
|
+
* 1989 on; no key). roll_call for the House falls back to the Congress.gov house-vote API (115th Congress on)
|
|
11
|
+
* when the Clerk file cannot be read. Every other endpoint goes through generic-rest.
|
|
12
|
+
* - manualItems / parseUpload: XML links for the vote endpoints; DEMO_KEY on Congress.gov links.
|
|
13
|
+
* - The Python refused_hosts probe (senate.gov refusing scripts) is refusedHostsProbe here: the Adapter's
|
|
14
|
+
* refusedHosts hook is synchronous and cannot probe, so fetch adds the same guidance to a blocked Senate request.
|
|
15
|
+
*/
|
|
16
|
+
import fs from "node:fs";
|
|
17
|
+
import path from "node:path";
|
|
18
|
+
import { PACKS_DIR } from "../core/env.js";
|
|
19
|
+
import * as gr from "../core/generic-rest.js";
|
|
20
|
+
import { Client, DataError } from "../core/http.js";
|
|
21
|
+
import { readJson } from "../core/policy.js";
|
|
22
|
+
import { parsePyJson, pyStr, urlsplit } from "../core/py.js";
|
|
23
|
+
import { pyJsonClient } from "./_govinfo_json.js";
|
|
24
|
+
import { htmlUnescape, pyInt, pyRepr, pySplit, pyStrip, truthy } from "./_govinfo_py.js";
|
|
25
|
+
import { fromstring, XmlParseError } from "./_govinfo_xml.js";
|
|
26
|
+
const HERE = path.join(PACKS_DIR, "congress");
|
|
27
|
+
const CLERK = "https://clerk.house.gov";
|
|
28
|
+
const SENATE = "https://www.senate.gov";
|
|
29
|
+
const CUSTOM = ["roll_call", "senate_votes", "bill_votes"];
|
|
30
|
+
const DEMO_LIMIT = 8;
|
|
31
|
+
const PARTY = { republican: "R", democratic: "D", democrat: "D", independent: "I" };
|
|
32
|
+
const CLIENTS = new Map();
|
|
33
|
+
const HOUSE_API_FIRST = 115; // Congress.gov house-vote data start with the 115th Congress (2017)
|
|
34
|
+
/** Hooks the tests replace (the Python tests patch the module's gr.fetch and _client). */
|
|
35
|
+
export const internals = {
|
|
36
|
+
fetchSpec: gr.fetchSpec,
|
|
37
|
+
client: (base, name) => {
|
|
38
|
+
if (!CLIENTS.has(base))
|
|
39
|
+
CLIENTS.set(base, new Client(base, { name, rps: 2, burst: 4 }));
|
|
40
|
+
return CLIENTS.get(base);
|
|
41
|
+
},
|
|
42
|
+
};
|
|
43
|
+
// ---------------------------------------------------------------------- congress numbers
|
|
44
|
+
/** The Congress whose term includes a date (January 3 since 1935, March 4 before). today: [year, month, day]. */
|
|
45
|
+
export function currentCongress(today) {
|
|
46
|
+
const d = today ?? (() => {
|
|
47
|
+
const t = new Date();
|
|
48
|
+
return [t.getFullYear(), t.getMonth() + 1, t.getDate()];
|
|
49
|
+
})();
|
|
50
|
+
const [y, m, day] = d;
|
|
51
|
+
let n = Math.floor((y - 1789) / 2) + 1;
|
|
52
|
+
const start = y >= 1935 ? [1, 3] : [3, 4];
|
|
53
|
+
if (y % 2 === 1 && (m < start[0] || (m === start[0] && day < start[1])) && n > 1)
|
|
54
|
+
n -= 1;
|
|
55
|
+
return n;
|
|
56
|
+
}
|
|
57
|
+
export function congressYears(n) {
|
|
58
|
+
const start = 1789 + 2 * (n - 1);
|
|
59
|
+
return [start, start + 1];
|
|
60
|
+
}
|
|
61
|
+
/** Calendar year of a regular session (session 1 = the odd year, session 2 = the even year). */
|
|
62
|
+
export function sessionYear(congress, session) {
|
|
63
|
+
return congressYears(pyInt(congress))[0] + pyInt(session) - 1;
|
|
64
|
+
}
|
|
65
|
+
export function ordinal(n0) {
|
|
66
|
+
const n = pyInt(n0);
|
|
67
|
+
const suf = n % 100 >= 10 && n % 100 <= 20 ? "th" : { 1: "st", 2: "nd", 3: "rd" }[((n % 10) + 10) % 10] ?? "th";
|
|
68
|
+
return `${n}${suf}`;
|
|
69
|
+
}
|
|
70
|
+
function eps() {
|
|
71
|
+
return readJson(HERE, "endpoints.json").endpoints;
|
|
72
|
+
}
|
|
73
|
+
// ---------------------------------------------------------------------- titles
|
|
74
|
+
function stripHtml(s) {
|
|
75
|
+
let t = pyStr(truthy(s) ? s : "").replace(/<br\s*\/?>|<\/p>|<\/li>/gi, "\n");
|
|
76
|
+
t = htmlUnescape(t.replace(/<[^>]+>/g, ""));
|
|
77
|
+
return pyStrip(t.replace(/\n\s*\n+/g, "\n").replace(/[ \t]+/g, " "));
|
|
78
|
+
}
|
|
79
|
+
const W = "[\\p{L}\\p{N}_]";
|
|
80
|
+
const esc = (s) => s.replace(/[\\^$.*+?()[\]{}|/]/g, "\\$&");
|
|
81
|
+
/** title_contains -> a test on a title, or null. Each phrase matches as given: its words in that order, each at the
|
|
82
|
+
* START of a word, any case, any spacing or punctuation between words. A list matches any of its phrases. */
|
|
83
|
+
export function titleMatcher(value) {
|
|
84
|
+
const phrases = Array.isArray(value) ? value : [value];
|
|
85
|
+
const pats = phrases.filter((p) => pySplit(pyStr(truthy(p) ? p : "")).length)
|
|
86
|
+
.map((p) => `(?<!${W})` + pySplit(pyStr(p)).map(esc).join(`${W}*[^\\p{L}\\p{N}_]+`));
|
|
87
|
+
if (!pats.length)
|
|
88
|
+
return null;
|
|
89
|
+
const rx = new RegExp(pats.join("|"), "iu");
|
|
90
|
+
return (title) => rx.test(pyStr(truthy(title) ? title : ""));
|
|
91
|
+
}
|
|
92
|
+
/** For a multi-word title_contains that kept nothing: the words must stand together and in that order. */
|
|
93
|
+
export function phraseOrderHint(req) {
|
|
94
|
+
const v = (req.params || {}).title_contains;
|
|
95
|
+
const multi = (Array.isArray(v) ? v : [v]).filter((p) => pySplit(pyStr(truthy(p) ? p : "")).length > 1).map((p) => pyStr(p));
|
|
96
|
+
if (!multi.length)
|
|
97
|
+
return null;
|
|
98
|
+
return `title_contains needs each phrase's words adjacent and in that order (${multi.map(pyRepr).join(", ")}); ` +
|
|
99
|
+
"to match them in another order or apart, list each form as its own phrase (a list matches any of " +
|
|
100
|
+
"them), or filter on one distinctive word and check the titles in the CSV.";
|
|
101
|
+
}
|
|
102
|
+
const TITLE_ROUTE = { bills: "BILLS", bills_congress: "BILLS", laws: "PLAW", laws_congress: "PLAW" };
|
|
103
|
+
/** title_contains on a bill or law list reads Congress.gov's display title only: say so, with the title search. */
|
|
104
|
+
export function displayTitleNote(req) {
|
|
105
|
+
const p = req.params || {};
|
|
106
|
+
const v = p.title_contains;
|
|
107
|
+
const phrases = (Array.isArray(v) ? v : [v]).filter((x) => pyStrip(pyStr(truthy(x) ? x : ""))).map((x) => pyStrip(pyStr(x)));
|
|
108
|
+
const coll = TITLE_ROUTE[req.endpoint || ""];
|
|
109
|
+
if (!phrases.length || !coll)
|
|
110
|
+
return null;
|
|
111
|
+
const c = p.congress;
|
|
112
|
+
const cong = /^\d+$/.test(pyStr(truthy(c) ? c : "")) ? `"congress": ${pyStr(c)}` : '"congress": <the Congress number>';
|
|
113
|
+
const search = phrases.slice(0, 3).map((x) => `{"collection": "${coll}", ${cong}, "title": "${x}"}`).join("; ");
|
|
114
|
+
return `${req._group || req.name}: title_contains matched the list's display title only, not official or short titles, so ` +
|
|
115
|
+
"bills whose official title alone holds the phrase are missing here. For 'in the title' answers, search " +
|
|
116
|
+
`titles at GovInfo: source govinfo, endpoint search, params ${search}; strip the version suffix from ` +
|
|
117
|
+
"packageId (BILLS-<congress><type><number><version>) to get each bill, deduplicate, then confirm each " +
|
|
118
|
+
"with this pack's bill_titles (for_each on congress, billType, billNumber) and say which title the " +
|
|
119
|
+
"count uses.";
|
|
120
|
+
}
|
|
121
|
+
function normalize(req, _ep, rows) {
|
|
122
|
+
const match = titleMatcher((req.params || {}).title_contains);
|
|
123
|
+
if (match)
|
|
124
|
+
rows = rows.filter((r) => match(r.title));
|
|
125
|
+
if (req.endpoint === "bill_summaries" || req.endpoint === "summaries")
|
|
126
|
+
rows = rows.map((r) => ({ ...r, text_plain: stripHtml(r.text) }));
|
|
127
|
+
return rows;
|
|
128
|
+
}
|
|
129
|
+
// ---------------------------------------------------------------------- XML votes
|
|
130
|
+
async function xml(url) {
|
|
131
|
+
const u = urlsplit(url);
|
|
132
|
+
const base = `${u.scheme}://${u.netloc}`;
|
|
133
|
+
if (base !== CLERK && base !== SENATE)
|
|
134
|
+
throw new DataError(`congress: not fetching vote XML from ${base}`, "api");
|
|
135
|
+
const c = internals.client(base, base === CLERK ? "house-clerk" : "senate-lis");
|
|
136
|
+
let body;
|
|
137
|
+
try {
|
|
138
|
+
body = (await c.get(u.path)).body;
|
|
139
|
+
}
|
|
140
|
+
catch (e) {
|
|
141
|
+
if (e instanceof DataError && e.kind === "blocked" && base === SENATE) {
|
|
142
|
+
throw new DataError(`${e.message}. Senate vote XML cannot be fetched from here; House votes can (bill_votes with ` +
|
|
143
|
+
'"chamber": "house"), and source.md (Access) gives the Congressional Record route for Senate roll calls', "blocked", e.status);
|
|
144
|
+
}
|
|
145
|
+
throw e;
|
|
146
|
+
}
|
|
147
|
+
try {
|
|
148
|
+
return fromstring(body);
|
|
149
|
+
}
|
|
150
|
+
catch (e) {
|
|
151
|
+
if (!(e instanceof XmlParseError))
|
|
152
|
+
throw e;
|
|
153
|
+
throw new DataError(`congress: ${u.path} is not valid XML (${e.message}); the site may be blocking automated requests`, "api");
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
export function houseUrl(year, roll) {
|
|
157
|
+
return `${CLERK}/evs/${pyInt(year)}/roll${String(pyInt(roll)).padStart(3, "0")}.xml`;
|
|
158
|
+
}
|
|
159
|
+
export function senateUrl(congress, session, roll) {
|
|
160
|
+
const c = pyInt(congress), s = pyInt(session);
|
|
161
|
+
return `${SENATE}/legislative/LIS/roll_call_votes/vote${c}${s}/vote_${c}_${s}_${String(pyInt(roll)).padStart(5, "0")}.xml`;
|
|
162
|
+
}
|
|
163
|
+
export function senateMenuUrl(congress, session) {
|
|
164
|
+
return `${SENATE}/legislative/LIS/roll_call_lists/vote_menu_${pyInt(congress)}_${pyInt(session)}.xml`;
|
|
165
|
+
}
|
|
166
|
+
/** Python's refused_hosts: when the spec needs Senate vote XML, try senate.gov once; a refusal is returned as the
|
|
167
|
+
* error a fetch would raise (a route check can run it before the fetch). */
|
|
168
|
+
export async function refusedHostsProbe(spec) {
|
|
169
|
+
const senate = (r) => {
|
|
170
|
+
const p = r.params || {}, ep = r.endpoint;
|
|
171
|
+
const ch = pyStr(p.chamber || "").toLowerCase();
|
|
172
|
+
return ep === "senate_votes" || (ep === "roll_call" && ch === "senate") || (ep === "bill_votes" && ch !== "house");
|
|
173
|
+
};
|
|
174
|
+
if (!(spec.requests || []).some((r) => r && typeof r === "object" && senate(r)))
|
|
175
|
+
return "";
|
|
176
|
+
try {
|
|
177
|
+
await internals.client(SENATE, "senate-lis").get(urlsplit(senateMenuUrl(119, 1)).path);
|
|
178
|
+
}
|
|
179
|
+
catch (e) {
|
|
180
|
+
if (e instanceof DataError && e.kind === "blocked") {
|
|
181
|
+
return `${e.message}. Senate vote XML cannot be fetched from here; House votes can (bill_votes with ` +
|
|
182
|
+
'"chamber": "house"), and source.md (Access) gives the Congressional Record route for Senate roll calls';
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return "";
|
|
186
|
+
}
|
|
187
|
+
function t(el, p, dflt = "") {
|
|
188
|
+
const x = el ? el.find(p) : null;
|
|
189
|
+
return x ? pySplit([...x.itertext()].join("")).join(" ") : dflt;
|
|
190
|
+
}
|
|
191
|
+
const MONTHS = ["january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december"];
|
|
192
|
+
const D = "(3[01]|[12]\\d|0[1-9]|[1-9]| [1-9])";
|
|
193
|
+
/** datetime.strptime(s, fmt).date().isoformat() for "%d-%b-%Y" and "%B %d, %Y" ("" when no format matches). */
|
|
194
|
+
function isoDate(s0, fmts) {
|
|
195
|
+
const s = pySplit(s0).join(" ");
|
|
196
|
+
for (const f of fmts) {
|
|
197
|
+
let m = null, y = 0, mo = 0, d = 0;
|
|
198
|
+
if (f === "%d-%b-%Y") {
|
|
199
|
+
m = new RegExp(`^${D}-(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)-(\\d{4})$`, "i").exec(s);
|
|
200
|
+
if (m)
|
|
201
|
+
[d, mo, y] = [Number(m[1]), MONTHS.findIndex((x) => x.startsWith(m[2].toLowerCase())) + 1, Number(m[3])];
|
|
202
|
+
}
|
|
203
|
+
else if (f === "%B %d, %Y") {
|
|
204
|
+
m = new RegExp(`^(${MONTHS.join("|")})\\s+${D},\\s+(\\d{4})$`, "i").exec(s);
|
|
205
|
+
if (m)
|
|
206
|
+
[mo, d, y] = [MONTHS.indexOf(m[1].toLowerCase()) + 1, Number(m[2]), Number(m[3])];
|
|
207
|
+
}
|
|
208
|
+
if (!m)
|
|
209
|
+
continue;
|
|
210
|
+
const dt = new Date(Date.UTC(2000, mo - 1, d));
|
|
211
|
+
dt.setUTCFullYear(y);
|
|
212
|
+
if (y < 1 || dt.getUTCMonth() !== mo - 1 || dt.getUTCDate() !== d)
|
|
213
|
+
continue;
|
|
214
|
+
return `${String(y).padStart(4, "0")}-${String(mo).padStart(2, "0")}-${String(d).padStart(2, "0")}`;
|
|
215
|
+
}
|
|
216
|
+
return "";
|
|
217
|
+
}
|
|
218
|
+
function position(v) {
|
|
219
|
+
const k = pyStrip(v).toLowerCase();
|
|
220
|
+
return { yea: "yea", aye: "yea", nay: "nay", no: "nay", present: "present", "not voting": "not_voting" }[k] ?? "other";
|
|
221
|
+
}
|
|
222
|
+
export function parseHouse(root, url) {
|
|
223
|
+
const m = root.find("vote-metadata");
|
|
224
|
+
const sess = t(m, "session").replace(/\D/g, "");
|
|
225
|
+
const dateRaw = t(m, "action-date");
|
|
226
|
+
const vote = { chamber: "house", congress: t(m, "congress"), session: sess, roll: t(m, "rollcall-num"),
|
|
227
|
+
date: isoDate(dateRaw, ["%d-%b-%Y"]), date_published: dateRaw, question: t(m, "vote-question"),
|
|
228
|
+
vote_type: t(m, "vote-type"), result: t(m, "vote-result"), legislation: t(m, "legis-num"), description: t(m, "vote-desc") };
|
|
229
|
+
const tv = m ? m.find("vote-totals/totals-by-vote") : null;
|
|
230
|
+
Object.assign(vote, { yea: t(tv, "yea-total"), nay: t(tv, "nay-total"), present: t(tv, "present-total"),
|
|
231
|
+
not_voting: t(tv, "not-voting-total") });
|
|
232
|
+
for (const tp of m ? m.findall("vote-totals/totals-by-party") : []) {
|
|
233
|
+
const p = PARTY[t(tp, "party").toLowerCase()] ?? (t(tp, "party").slice(0, 1).toUpperCase() || "?");
|
|
234
|
+
for (const [k, tag] of [["yea", "yea-total"], ["nay", "nay-total"], ["present", "present-total"], ["not_voting", "not-voting-total"]]) {
|
|
235
|
+
vote[`${k}_${p}`] = t(tp, tag);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
vote.source_url = url;
|
|
239
|
+
const members = [];
|
|
240
|
+
for (const rv of root.findall("vote-data/recorded-vote")) {
|
|
241
|
+
const leg = rv.find("legislator");
|
|
242
|
+
const a = leg ? leg.attrib : {};
|
|
243
|
+
const cast = t(rv, "vote");
|
|
244
|
+
members.push({ chamber: "house", congress: vote.congress, session: sess, roll: vote.roll, date: vote.date,
|
|
245
|
+
member_id: a["name-id"] ?? "", name: a["unaccented-name"] || t(rv, "legislator"), party: a.party ?? "",
|
|
246
|
+
state: a.state ?? "", vote: cast, position: position(cast), vote_source: "clerk.house.gov" });
|
|
247
|
+
}
|
|
248
|
+
return [vote, members];
|
|
249
|
+
}
|
|
250
|
+
export function parseSenate(root, url) {
|
|
251
|
+
const dateRaw = t(root, "vote_date");
|
|
252
|
+
const parts = dateRaw.split(",");
|
|
253
|
+
const d = isoDate(parts.length >= 2 ? parts[0] + "," + parts[1] : dateRaw, ["%B %d, %Y"]);
|
|
254
|
+
const vote = { chamber: "senate", congress: t(root, "congress"), session: t(root, "session"),
|
|
255
|
+
roll: String(pyInt(t(root, "vote_number") || 0)), date: d, date_published: dateRaw,
|
|
256
|
+
question: t(root, "vote_question_text") || t(root, "question"), vote_type: t(root, "majority_requirement"),
|
|
257
|
+
result: t(root, "vote_result"), legislation: t(root, "document/document_name"),
|
|
258
|
+
description: t(root, "vote_title") || t(root, "vote_document_text"),
|
|
259
|
+
yea: t(root, "count/yeas"), nay: t(root, "count/nays"), present: t(root, "count/present"),
|
|
260
|
+
not_voting: t(root, "count/absent"), tie_breaker: t(root, "tie_breaker/by_whom") };
|
|
261
|
+
const members = [];
|
|
262
|
+
for (const mm of root.findall("members/member")) {
|
|
263
|
+
const cast = t(mm, "vote_cast");
|
|
264
|
+
members.push({ chamber: "senate", congress: vote.congress, session: vote.session, roll: vote.roll, date: d,
|
|
265
|
+
member_id: t(mm, "lis_member_id"), name: pyStrip(`${t(mm, "first_name")} ${t(mm, "last_name")}`), party: t(mm, "party"),
|
|
266
|
+
state: t(mm, "state"), vote: cast, position: position(cast), vote_source: "senate.gov" });
|
|
267
|
+
}
|
|
268
|
+
const parties = [...new Set(members.map((x) => String(x.party)))];
|
|
269
|
+
const tally = new Map();
|
|
270
|
+
for (const pos of ["yea", "nay", "present", "not_voting"])
|
|
271
|
+
for (const p of parties)
|
|
272
|
+
tally.set(JSON.stringify([pos, p]), [pos, p, 0]);
|
|
273
|
+
for (const x of members) {
|
|
274
|
+
const k = JSON.stringify([x.position, x.party]);
|
|
275
|
+
const cur = tally.get(k) ?? [String(x.position), String(x.party), 0];
|
|
276
|
+
cur[2]++;
|
|
277
|
+
tally.set(k, cur);
|
|
278
|
+
}
|
|
279
|
+
const sorted = [...tally.values()].sort((a, b) => (a[0] !== b[0] ? (a[0] < b[0] ? -1 : 1) : a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0));
|
|
280
|
+
for (const [pos, party, n] of sorted)
|
|
281
|
+
vote[`${pos}_${party}`] = n;
|
|
282
|
+
for (const pos of ["present", "not_voting"]) { // the XML leaves <present/> and <absent/> empty when the count is 0
|
|
283
|
+
if (members.length && vote[pos] === "")
|
|
284
|
+
vote[pos] = String(members.filter((x) => x.position === pos).length);
|
|
285
|
+
}
|
|
286
|
+
vote.source_url = url;
|
|
287
|
+
return [vote, members];
|
|
288
|
+
}
|
|
289
|
+
export function parseVoteXml(root, url) {
|
|
290
|
+
if (root.tag === "rollcall-vote")
|
|
291
|
+
return parseHouse(root, url);
|
|
292
|
+
if (root.tag === "roll_call_vote")
|
|
293
|
+
return parseSenate(root, url);
|
|
294
|
+
throw new DataError(`congress: ${url} is not a roll-call vote file`, "api");
|
|
295
|
+
}
|
|
296
|
+
export function parseSenateMenu(root, _url) {
|
|
297
|
+
const year = t(root, "congress_year");
|
|
298
|
+
const rows = [];
|
|
299
|
+
for (const v of root.findall("votes/vote")) {
|
|
300
|
+
// an en bloc vote nests issue, question and result under <en_bloc><matter>
|
|
301
|
+
const matters = v.findall("en_bloc/matter");
|
|
302
|
+
const pick = (tag) => t(v, tag) || [...new Set(matters.map((m) => t(m, tag)).filter(Boolean))].join("; ");
|
|
303
|
+
rows.push({ chamber: "senate", congress: t(root, "congress"), session: t(root, "session"),
|
|
304
|
+
roll: String(pyInt(t(v, "vote_number") || 0)), date: isoDate(`${t(v, "vote_date")}-${year}`, ["%d-%b-%Y"]),
|
|
305
|
+
issue: pick("issue"), question: pick("question"), result: pick("result"), en_bloc: matters.length ? String(matters.length) : "",
|
|
306
|
+
yea: t(v, "vote_tally/yeas"), nay: t(v, "vote_tally/nays"), title: t(v, "title"),
|
|
307
|
+
source_url: senateUrl(t(root, "congress"), t(root, "session"), t(v, "vote_number") || 0) });
|
|
308
|
+
}
|
|
309
|
+
return rows;
|
|
310
|
+
}
|
|
311
|
+
async function houseApiMembers(pack, congress, session, roll) {
|
|
312
|
+
const data = await gr.clientFor(pack).getJson(`/house-vote/${congress}/${session}/${pyInt(roll)}/members`, { format: "json" });
|
|
313
|
+
const v = data.houseRollCallVoteMemberVotes || {};
|
|
314
|
+
return (v.results || []).map((m) => {
|
|
315
|
+
const cast = pyStr(m.voteCast ?? "");
|
|
316
|
+
return { chamber: "house", congress: String(congress), session: String(session), roll: pyStr(roll),
|
|
317
|
+
date: pyStr(v.startDate ?? "").slice(0, 10), member_id: m.bioguideID ?? "",
|
|
318
|
+
name: pyStrip(`${pyStr(m.firstName ?? "")} ${pyStr(m.lastName ?? "")}`), party: m.voteParty ?? "",
|
|
319
|
+
state: m.voteState ?? "", vote: cast, position: position(cast), vote_source: "api.congress.gov" };
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
const yes = (v) => ["true", "1", "yes"].includes(pyStr(v ?? "").toLowerCase());
|
|
323
|
+
async function rollCall(pack, p) {
|
|
324
|
+
const ch = pyStr(p.chamber).toLowerCase();
|
|
325
|
+
// summary=true: one row for the vote itself (question, result, type, totals) instead of one per member
|
|
326
|
+
const part = yes(p.summary) ? 0 : 1;
|
|
327
|
+
if (ch === "senate") {
|
|
328
|
+
const url = senateUrl(p.congress, p.session, p.roll);
|
|
329
|
+
const got = parseSenate(await xml(url), url);
|
|
330
|
+
return [part ? got[1] : [got[0]], []];
|
|
331
|
+
}
|
|
332
|
+
const year = pyInt(truthy(p.year) ? p.year : sessionYear(pyInt(p.congress), pyInt(p.session)));
|
|
333
|
+
const url = houseUrl(year, p.roll);
|
|
334
|
+
try {
|
|
335
|
+
const got = parseHouse(await xml(url), url);
|
|
336
|
+
return [part ? got[1] : [got[0]], []];
|
|
337
|
+
}
|
|
338
|
+
catch (e) {
|
|
339
|
+
if (!(e instanceof DataError) || !part)
|
|
340
|
+
throw e;
|
|
341
|
+
const congress = pyInt(truthy(p.congress) ? p.congress : currentCongress([year, 6, 1]));
|
|
342
|
+
const session = pyInt(truthy(p.session) ? p.session : year % 2 ? 1 : 2);
|
|
343
|
+
if (congress < HOUSE_API_FIRST)
|
|
344
|
+
throw e;
|
|
345
|
+
const rows = await houseApiMembers(pack, congress, session, p.roll);
|
|
346
|
+
return [rows, [`House Clerk file unavailable (${e.message}); member votes came from the Congress.gov house-vote API`]];
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
async function billVotes(pack, req) {
|
|
350
|
+
const p = { ...(req.params || {}) };
|
|
351
|
+
if (pyStr(p.congress ?? "").toLowerCase() === "current")
|
|
352
|
+
p.congress = currentCongress();
|
|
353
|
+
const ep = eps().bill_actions;
|
|
354
|
+
const params = { ...(ep.defaults || {}) };
|
|
355
|
+
for (const k of ["congress", "billType", "billNumber"]) {
|
|
356
|
+
if (!(k in p))
|
|
357
|
+
throw new DataError(`request ${pyRepr(req.name)}: bill_votes needs ${k}`, "bad_request");
|
|
358
|
+
params[k] = p[k];
|
|
359
|
+
}
|
|
360
|
+
const [actions] = await gr.fetchRequest(pyJsonClient(gr.clientFor(pack), pack, ep.records), ep, params, 5000);
|
|
361
|
+
const want = pyStr(p.chamber || "").toLowerCase();
|
|
362
|
+
const seen = new Set();
|
|
363
|
+
const votes = [];
|
|
364
|
+
for (const a of actions) {
|
|
365
|
+
const rv = a.recordedVotes;
|
|
366
|
+
for (const v of typeof rv === "string" && rv.startsWith("[") ? parsePyJson(rv) : []) {
|
|
367
|
+
const url = pyStr(v.url || "");
|
|
368
|
+
const ch = pyStr(v.chamber ?? "").toLowerCase();
|
|
369
|
+
if (!url || seen.has(url) || (want && ch !== want))
|
|
370
|
+
continue;
|
|
371
|
+
seen.add(url);
|
|
372
|
+
votes.push([v, a]);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
votes.sort((x, y) => {
|
|
376
|
+
const [dx, dy] = [pyStr(x[0].date ?? ""), pyStr(y[0].date ?? "")];
|
|
377
|
+
if (dx !== dy)
|
|
378
|
+
return dx < dy ? -1 : 1;
|
|
379
|
+
return Number(x[0].rollNumber ?? 0) - Number(y[0].rollNumber ?? 0);
|
|
380
|
+
});
|
|
381
|
+
let rows = [];
|
|
382
|
+
const members = yes(p.members);
|
|
383
|
+
for (const [v, a] of votes) {
|
|
384
|
+
const [vote, mem] = parseVoteXml(await xml(v.url), v.url);
|
|
385
|
+
const ctx = { bill: `${pyStr(p.congress)} ${pyStr(p.billType).toUpperCase()} ${pyStr(p.billNumber)}`,
|
|
386
|
+
action_date: a.actionDate ?? "", action_text: a.text ?? "" };
|
|
387
|
+
if (members)
|
|
388
|
+
rows = rows.concat(mem.map((m) => ({ ...ctx, ...m })));
|
|
389
|
+
else
|
|
390
|
+
rows.push({ ...ctx, ...vote });
|
|
391
|
+
}
|
|
392
|
+
let keys = [];
|
|
393
|
+
for (const r of rows)
|
|
394
|
+
keys = keys.concat(Object.keys(r).filter((k) => !keys.includes(k)));
|
|
395
|
+
const party = keys.filter((k) => /^(yea|nay|present|not_voting|other)_.+$/s.test(k)).sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
|
|
396
|
+
keys = [...keys.filter((k) => !party.includes(k) && k !== "source_url"), ...party, ...(keys.includes("source_url") ? ["source_url"] : [])];
|
|
397
|
+
return rows.map((r) => Object.fromEntries(keys.map((k) => [k, k in r ? r[k] : ""])));
|
|
398
|
+
}
|
|
399
|
+
// ---------------------------------------------------------------------- keys, manual route
|
|
400
|
+
function calls(spec) {
|
|
401
|
+
const e = eps();
|
|
402
|
+
let n = 0;
|
|
403
|
+
for (const r of spec.requests || []) {
|
|
404
|
+
if (r.endpoint === "roll_call" || r.endpoint === "senate_votes")
|
|
405
|
+
continue; // official XML: no key, no quota
|
|
406
|
+
const pg = (e[r.endpoint || ""] || {}).paging || {};
|
|
407
|
+
const size = pyInt(pg.size || 250);
|
|
408
|
+
n += pg.type === "offset" ? Math.max(1, Math.ceil(pyInt(r.max_rows || size) / size)) : 1;
|
|
409
|
+
}
|
|
410
|
+
return n;
|
|
411
|
+
}
|
|
412
|
+
const adapter = {
|
|
413
|
+
normalize,
|
|
414
|
+
checkSpec(spec) {
|
|
415
|
+
const probs = [];
|
|
416
|
+
for (const r of spec.requests || []) {
|
|
417
|
+
const p = r.params || {}, n = pyRepr(r.name ?? null);
|
|
418
|
+
if (r.endpoint === "roll_call") {
|
|
419
|
+
const ch = pyStr(p.chamber ?? "").toLowerCase();
|
|
420
|
+
if (ch === "senate" && !(truthy(p.congress) && truthy(p.session)))
|
|
421
|
+
probs.push(`request ${n}: a Senate roll_call needs congress and session`);
|
|
422
|
+
if (ch === "house" && !(truthy(p.year) || (truthy(p.congress) && truthy(p.session)))) {
|
|
423
|
+
probs.push(`request ${n}: a House roll_call needs year, or congress and session`);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
const c = p.congress;
|
|
427
|
+
if (!(c === null || c === undefined || c === "" || c === "current") && !/^\d+$/.test(pyStr(c))) {
|
|
428
|
+
probs.push(`request ${n}: congress must be a number or 'current'`);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
return probs;
|
|
432
|
+
},
|
|
433
|
+
beforeRequest(_req, _ep, params) {
|
|
434
|
+
const out = { ...params };
|
|
435
|
+
if (pyStr(out.congress ?? "").toLowerCase() === "current")
|
|
436
|
+
out.congress = currentCongress();
|
|
437
|
+
delete out.title_contains;
|
|
438
|
+
// Congress.gov reads "introducedDate+asc" with a literal "+" (a space); sent encoded as %2B the sort is ignored
|
|
439
|
+
if (typeof out.sort === "string")
|
|
440
|
+
out.sort = pySplit(out.sort.split("+").join(" ")).join(" ");
|
|
441
|
+
return out;
|
|
442
|
+
},
|
|
443
|
+
async fetch(spec, pack, _route) {
|
|
444
|
+
const tables = {};
|
|
445
|
+
let notes = [];
|
|
446
|
+
const E = eps();
|
|
447
|
+
for (const r of spec.requests) {
|
|
448
|
+
const name = r.endpoint;
|
|
449
|
+
if (!CUSTOM.includes(name || "")) {
|
|
450
|
+
const hint = phraseOrderHint(r);
|
|
451
|
+
let tt, n;
|
|
452
|
+
try {
|
|
453
|
+
const client = pyJsonClient(gr.clientFor(pack), pack, (E[name || ""] || {}).records ?? "");
|
|
454
|
+
[tt, n] = await internals.fetchSpec(pack, { requests: [r] }, client);
|
|
455
|
+
}
|
|
456
|
+
catch (e) {
|
|
457
|
+
if (e instanceof DataError && hint && e.message.includes("returned no rows")) {
|
|
458
|
+
throw new DataError(`${e.message} ${hint}`, e.kind);
|
|
459
|
+
}
|
|
460
|
+
throw e;
|
|
461
|
+
}
|
|
462
|
+
Object.assign(tables, tt);
|
|
463
|
+
const got = tt[r.name];
|
|
464
|
+
notes = notes.concat(n, hint && !(Array.isArray(got) ? got.length : got) ? [`${r.name}: no titles matched. ${hint}`] : []);
|
|
465
|
+
const titleNote = displayTitleNote(r);
|
|
466
|
+
if (titleNote && !notes.includes(titleNote))
|
|
467
|
+
notes.push(titleNote); // for_each: once
|
|
468
|
+
continue;
|
|
469
|
+
}
|
|
470
|
+
const p = r.params || {};
|
|
471
|
+
let rows;
|
|
472
|
+
if (name === "roll_call") {
|
|
473
|
+
const [got, n] = await rollCall(pack, p);
|
|
474
|
+
rows = got;
|
|
475
|
+
notes = notes.concat(n);
|
|
476
|
+
}
|
|
477
|
+
else if (name === "senate_votes") {
|
|
478
|
+
const url = senateMenuUrl(p.congress, p.session);
|
|
479
|
+
rows = parseSenateMenu(await xml(url), url);
|
|
480
|
+
}
|
|
481
|
+
else
|
|
482
|
+
rows = await billVotes(pack, r);
|
|
483
|
+
if (!rows.length && !r.allow_empty) {
|
|
484
|
+
throw new DataError(`request ${pyRepr(r.name)} returned no rows (no recorded votes, or a wrong roll number)`, "bad_request");
|
|
485
|
+
}
|
|
486
|
+
tables[r.name] = rows;
|
|
487
|
+
}
|
|
488
|
+
return [tables, notes];
|
|
489
|
+
},
|
|
490
|
+
keyOnlyNeeds(spec) {
|
|
491
|
+
const n = calls(spec);
|
|
492
|
+
return n > DEMO_LIMIT ? [`about ${n} Congress.gov API calls (the keyless DEMO_KEY allows only about 10 an hour)`] : [];
|
|
493
|
+
},
|
|
494
|
+
manualItems(spec, pack) {
|
|
495
|
+
const items = [];
|
|
496
|
+
for (const r of spec.requests || []) {
|
|
497
|
+
const p = r.params || {}, name = r.name;
|
|
498
|
+
try {
|
|
499
|
+
if (r.endpoint === "roll_call") {
|
|
500
|
+
const url = pyStr(p.chamber ?? "None").toLowerCase() === "senate" ? senateUrl(p.congress, p.session, p.roll)
|
|
501
|
+
: houseUrl(truthy(p.year) ? p.year : sessionYear(pyInt(p.congress), pyInt(p.session)), p.roll);
|
|
502
|
+
items.push({ name: `${name}#1`, request: name, url, filename: `${name}.xml`, needs_key: false, method: "GET", body: null,
|
|
503
|
+
note: "official roll-call XML, no key" });
|
|
504
|
+
continue;
|
|
505
|
+
}
|
|
506
|
+
if (r.endpoint === "senate_votes") {
|
|
507
|
+
items.push({ name: `${name}#1`, request: name, url: senateMenuUrl(p.congress, p.session), filename: `${name}.xml`,
|
|
508
|
+
needs_key: false, method: "GET", body: null, note: "Senate vote list XML, no key" });
|
|
509
|
+
continue;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
catch {
|
|
513
|
+
continue;
|
|
514
|
+
}
|
|
515
|
+
let rr = r;
|
|
516
|
+
if (r.endpoint === "bill_votes") {
|
|
517
|
+
rr = { ...r, endpoint: "bill_actions", params: Object.fromEntries(Object.entries(p).filter(([k]) => ["congress", "billType", "billNumber"].includes(k))) };
|
|
518
|
+
}
|
|
519
|
+
if (pyStr((rr.params || {}).congress ?? "").toLowerCase() === "current")
|
|
520
|
+
rr = { ...rr, params: { ...rr.params, congress: currentCongress() } };
|
|
521
|
+
rr = { ...rr, params: Object.fromEntries(Object.entries(rr.params || {}).filter(([k]) => k !== "title_contains")) };
|
|
522
|
+
for (const it of gr.manualItems(pack, { requests: [rr] })) {
|
|
523
|
+
if (!it.url.includes("api_key="))
|
|
524
|
+
it.url += (it.url.includes("?") ? "&" : "?") + "api_key=DEMO_KEY";
|
|
525
|
+
let note = "DEMO_KEY is api.data.gov's public demo key (about 10 requests an hour); replace it with your own key";
|
|
526
|
+
if (r.endpoint === "bill_votes") {
|
|
527
|
+
note = "the bill's actions; each recordedVotes entry links a vote XML file (clerk.house.gov or " +
|
|
528
|
+
"senate.gov) to save as well, from a web browser if a script is refused; " + note;
|
|
529
|
+
}
|
|
530
|
+
it.note = (it.note ? it.note + "; " : "") + note;
|
|
531
|
+
items.push(it);
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
return items;
|
|
535
|
+
},
|
|
536
|
+
parseUpload(pack, req, files) {
|
|
537
|
+
const ep = req.endpoint;
|
|
538
|
+
const readXml = (f) => {
|
|
539
|
+
try {
|
|
540
|
+
return fromstring(fs.readFileSync(f));
|
|
541
|
+
}
|
|
542
|
+
catch (e) {
|
|
543
|
+
if (e instanceof XmlParseError)
|
|
544
|
+
throw new DataError(`congress: ${path.basename(f)} is not valid XML (${e.message})`, "api");
|
|
545
|
+
throw e;
|
|
546
|
+
}
|
|
547
|
+
};
|
|
548
|
+
if (ep === "roll_call" || ep === "senate_votes") {
|
|
549
|
+
let rows = [];
|
|
550
|
+
for (const f of files) {
|
|
551
|
+
const root = readXml(f);
|
|
552
|
+
const summary = yes((req.params || {}).summary);
|
|
553
|
+
rows = rows.concat(ep === "senate_votes" ? parseSenateMenu(root, path.basename(f))
|
|
554
|
+
: summary ? [parseVoteXml(root, path.basename(f))[0]] : parseVoteXml(root, path.basename(f))[1]);
|
|
555
|
+
}
|
|
556
|
+
return rows;
|
|
557
|
+
}
|
|
558
|
+
if (ep === "bill_votes") {
|
|
559
|
+
const rows = [];
|
|
560
|
+
for (const f of files) {
|
|
561
|
+
if (f.toLowerCase().endsWith(".xml")) {
|
|
562
|
+
rows.push(parseVoteXml(readXml(f), path.basename(f))[0]);
|
|
563
|
+
continue;
|
|
564
|
+
}
|
|
565
|
+
const data = parsePyJson(fs.readFileSync(f, "utf8").replace(/^/, ""));
|
|
566
|
+
for (const a of data.actions || []) {
|
|
567
|
+
for (const v of a.recordedVotes || []) {
|
|
568
|
+
rows.push({ action_date: a.actionDate ?? "", chamber: pyStr(v.chamber ?? "").toLowerCase(), congress: v.congress ?? "",
|
|
569
|
+
session: v.sessionNumber ?? "", roll: v.rollNumber ?? "", date: pyStr(v.date ?? "").slice(0, 10),
|
|
570
|
+
source_url: v.url ?? "", action_text: a.text ?? "" });
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
const seen = new Set();
|
|
575
|
+
const out = [];
|
|
576
|
+
for (const r of rows) { // actions repeat each vote: keep one row per vote
|
|
577
|
+
const k = truthy(r.source_url) ? r.source_url : r;
|
|
578
|
+
if (!seen.has(k)) {
|
|
579
|
+
seen.add(k);
|
|
580
|
+
out.push(r);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
return out;
|
|
584
|
+
}
|
|
585
|
+
return gr.parseUpload(pack, req, files);
|
|
586
|
+
},
|
|
587
|
+
};
|
|
588
|
+
export default adapter;
|
|
589
|
+
//# sourceMappingURL=congress.js.map
|