data-analyst-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -2
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +716 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +24 -0
- package/dist/report/scan_secrets.js +265 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +247 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +54 -4
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +170 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,636 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GovInfo pack hooks.
|
|
3
|
+
*
|
|
4
|
+
* - search: GovInfo filters by collection, congress, bill type, publisher and date only through field:value terms
|
|
5
|
+
* INSIDE the query string, so beforeRequest builds the query. User text is escaped: query operators, field syntax
|
|
6
|
+
* and punctuation are removed from `text`, `phrase` and `title`. `query` is raw syntax, passed unchanged.
|
|
7
|
+
* - text: fetch a package's summary, then its text link (api.govinfo.gov, with key) or the public content link
|
|
8
|
+
* (www.govinfo.gov, never with key), strip the HTML and return one row.
|
|
9
|
+
* - sections: a public law's USLM XML (113th Congress on), one row per section, with the U.S. Code references split
|
|
10
|
+
* into amended, cited and classified (margin notes). Sections inside <quotedContent> are amended Code text.
|
|
11
|
+
* - manualItems: api links carry the public DEMO_KEY; text and sections items use keyless www.govinfo.gov links.
|
|
12
|
+
*/
|
|
13
|
+
import fs from "node:fs";
|
|
14
|
+
import path from "node:path";
|
|
15
|
+
import { PACKS_DIR } from "../core/env.js";
|
|
16
|
+
import * as gr from "../core/generic-rest.js";
|
|
17
|
+
import { Client, DataError } from "../core/http.js";
|
|
18
|
+
import { readJson } from "../core/policy.js";
|
|
19
|
+
import { parseQsl, pyQuote, pyStr, pyUrlencode, urlsplit, urlunsplit } from "../core/py.js";
|
|
20
|
+
import { pyJsonClient } from "./_govinfo_json.js";
|
|
21
|
+
import { htmlUnescape, pyInt, pyRepr, pySplit, pyStrip, pyTupleRepr, pyWsSub, stripChars, truthy } from "./_govinfo_py.js";
|
|
22
|
+
import { fromstring, XmlParseError } from "./_govinfo_xml.js";
|
|
23
|
+
const HERE = path.join(PACKS_DIR, "govinfo");
|
|
24
|
+
const BILL_TYPES = ["hr", "s", "hjres", "sjres", "hconres", "sconres", "hres", "sres"];
|
|
25
|
+
const SEARCH_KEYS = ["text", "phrase", "all_words", "any", "title", "query", "collection", "congress", "billtype",
|
|
26
|
+
"publisher", "date_from", "date_to", "sort"];
|
|
27
|
+
const SORTS = {
|
|
28
|
+
relevance: [{ field: "score", sortOrder: "DESC" }],
|
|
29
|
+
newest: [{ field: "publishdate", sortOrder: "DESC" }],
|
|
30
|
+
oldest: [{ field: "publishdate", sortOrder: "ASC" }]
|
|
31
|
+
};
|
|
32
|
+
const OPERATORS = new Set(["and", "or", "not"]);
|
|
33
|
+
const DEMO_LIMIT = 8;
|
|
34
|
+
const cmp = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
35
|
+
/** Hooks the tests replace (the Python tests patch the module's gr.client_for and _fetch_text). */
|
|
36
|
+
export const internals = {
|
|
37
|
+
client: (pack) => gr.clientFor(pack),
|
|
38
|
+
fetchText: (pack, req) => fetchText(pack, req),
|
|
39
|
+
};
|
|
40
|
+
function eps() {
|
|
41
|
+
return readJson(HERE, "endpoints.json");
|
|
42
|
+
}
|
|
43
|
+
/** len() and [:n] of a str (code points, not UTF-16 units). */
|
|
44
|
+
const SURR = /[\ud800-\udfff]/;
|
|
45
|
+
const pyLen = (s) => (SURR.test(s) ? [...s].length : s.length);
|
|
46
|
+
const pyCut = (s, n) => (SURR.test(s) ? [...s].slice(0, n).join("") : s.slice(0, n));
|
|
47
|
+
function collections() {
|
|
48
|
+
return new Set(Object.values(eps().catalog).map((c) => c.id).filter((id) => id === id.toUpperCase() &&
|
|
49
|
+
/[A-Z]/.test(id) && !id.includes(" ")));
|
|
50
|
+
}
|
|
51
|
+
/** {collection: [first year, last year]} for collections whose catalog entry names a closed range. */
|
|
52
|
+
function coverage() {
|
|
53
|
+
const out = {};
|
|
54
|
+
for (const c of Object.values(eps().catalog)) {
|
|
55
|
+
const m = /\((\d{4})-(\d{4})\)/.exec(c.description || "");
|
|
56
|
+
if (m && c.id === c.id.toUpperCase() && /[A-Z]/.test(c.id))
|
|
57
|
+
out[c.id] = [Number(m[1]), Number(m[2])];
|
|
58
|
+
}
|
|
59
|
+
return out;
|
|
60
|
+
}
|
|
61
|
+
function list(v) {
|
|
62
|
+
if (Array.isArray(v))
|
|
63
|
+
return v.map((x) => pyStrip(pyStr(x))).filter(Boolean);
|
|
64
|
+
return pyStr(v).split(",").map((x) => pyStrip(x)).filter(Boolean);
|
|
65
|
+
}
|
|
66
|
+
/** A date window wholly outside a closed collection's years returns nothing: say so before fetching. */
|
|
67
|
+
function outsideCoverage(name, p) {
|
|
68
|
+
const years = ["start_date", "end_date", "date_from", "date_to"].filter((k) => /^\d{4}/.test(pyStr(truthy(p[k]) ? p[k] : "")))
|
|
69
|
+
.map((k) => Number(pyStr(p[k]).slice(0, 4)));
|
|
70
|
+
if (!years.length)
|
|
71
|
+
return [];
|
|
72
|
+
const cov = coverage(), out = [];
|
|
73
|
+
const lo = Math.min(...years), hi = Math.max(...years);
|
|
74
|
+
for (const c of list(truthy(p.collection) ? p.collection : [])) {
|
|
75
|
+
const rng = cov[c.toUpperCase()];
|
|
76
|
+
if (rng && (lo > rng[1] || hi < rng[0])) {
|
|
77
|
+
out.push(`request ${pyRepr(name)}: ${c.toUpperCase()} on GovInfo covers ${rng[0]}-${rng[1]}; nothing is dated ` +
|
|
78
|
+
`${lo}-${hi} (later items are on the agency's own site, which no pack serves)`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return out;
|
|
82
|
+
}
|
|
83
|
+
/** User text reduced to plain words: no field syntax, operators, quotes, wildcards or brackets. */
|
|
84
|
+
export function words(text) {
|
|
85
|
+
const t = pyStr(text).replace(/[()[\]{}:"'\\^~*?!/<>=|&+]/g, " ");
|
|
86
|
+
const out = [];
|
|
87
|
+
for (let w of pySplit(t)) {
|
|
88
|
+
w = stripChars(w, "-,;.");
|
|
89
|
+
if (w && !OPERATORS.has(w.toLowerCase()))
|
|
90
|
+
out.push(w);
|
|
91
|
+
}
|
|
92
|
+
return out;
|
|
93
|
+
}
|
|
94
|
+
function date(v) {
|
|
95
|
+
const s = pyStrip(pyStr(v));
|
|
96
|
+
if (!/^\d{4}-\d{2}-\d{2}$/.test(s))
|
|
97
|
+
throw new DataError(`govinfo: dates must be YYYY-MM-DD, got ${pyRepr(s)}`, "bad_request");
|
|
98
|
+
return s;
|
|
99
|
+
}
|
|
100
|
+
/** `any`: a list of words or phrases (or one string separated by ;), any of which may match. */
|
|
101
|
+
function alternatives(v) {
|
|
102
|
+
return Array.isArray(v) ? v.map(pyStr) : pyStr(v).split(";");
|
|
103
|
+
}
|
|
104
|
+
/** The GovInfo query string for search parameters (filters first, then the escaped text). */
|
|
105
|
+
export function buildQuery(p) {
|
|
106
|
+
const terms = [];
|
|
107
|
+
const colls = list(truthy(p.collection) ? p.collection : []).map((c) => c.toUpperCase());
|
|
108
|
+
if (colls.length)
|
|
109
|
+
terms.push(colls.length === 1 ? `collection:${colls[0]}` : "(" + colls.map((c) => `collection:${c}`).join(" OR ") + ")");
|
|
110
|
+
if (truthy(p.congress))
|
|
111
|
+
terms.push(`congress:${pyInt(p.congress)}`);
|
|
112
|
+
if (truthy(p.billtype))
|
|
113
|
+
terms.push(`billtype:${pyStr(p.billtype).toLowerCase()}`);
|
|
114
|
+
if (truthy(p.publisher))
|
|
115
|
+
terms.push('publisher:"' + words(p.publisher).join(" ") + '"');
|
|
116
|
+
if (truthy(p.date_from) || truthy(p.date_to)) {
|
|
117
|
+
terms.push(`publishdate:range(${truthy(p.date_from) ? date(p.date_from) : ""},${truthy(p.date_to) ? date(p.date_to) : ""})`);
|
|
118
|
+
}
|
|
119
|
+
if (truthy(p.text))
|
|
120
|
+
terms.push("(" + words(p.text).join(" ") + ")");
|
|
121
|
+
if (truthy(p.phrase)) {
|
|
122
|
+
const w = words(p.phrase);
|
|
123
|
+
// one word is searched unquoted: quotes match only that exact form, and fewer documents
|
|
124
|
+
terms.push(w.length > 1 ? '("' + w.join(" ") + '")' : "(" + w.join(" ") + ")");
|
|
125
|
+
}
|
|
126
|
+
if (truthy(p.all_words))
|
|
127
|
+
terms.push("(" + words(p.all_words).join(" ") + ")");
|
|
128
|
+
if (truthy(p.title)) {
|
|
129
|
+
const w = words(p.title);
|
|
130
|
+
terms.push(w.length > 1 ? 'title:"' + w.join(" ") + '"' : `title:${w[0]}`);
|
|
131
|
+
}
|
|
132
|
+
if (truthy(p.any)) {
|
|
133
|
+
const alts = alternatives(p.any).map(words).filter((w) => w.length);
|
|
134
|
+
terms.push("(" + alts.map((w) => (w.length > 1 ? '"' + w.join(" ") + '"' : w[0])).join(" OR ") + ")");
|
|
135
|
+
}
|
|
136
|
+
if (truthy(p.query))
|
|
137
|
+
terms.push("(" + pyStr(p.query) + ")");
|
|
138
|
+
return terms.join(" AND ");
|
|
139
|
+
}
|
|
140
|
+
const PHRASE_GAP_MIN = 20; // documents: a smaller gap is within what word order and adjacency alone explain
|
|
141
|
+
/** One count request for the same search with all the words (pageSize 1): a note when the quoted result is under
|
|
142
|
+
* half of it and at least PHRASE_GAP_MIN documents short. */
|
|
143
|
+
export async function phraseCountNote(pack, req, got) {
|
|
144
|
+
const p = req.params || {};
|
|
145
|
+
if (words(truthy(p.phrase) ? p.phrase : "").length < 2)
|
|
146
|
+
return null;
|
|
147
|
+
const alt = { ...Object.fromEntries(Object.entries(p).filter(([k]) => k !== "phrase")), all_words: p.phrase };
|
|
148
|
+
let total;
|
|
149
|
+
try {
|
|
150
|
+
const [ep, params] = gr.resolve(pack, { ...req, params: alt });
|
|
151
|
+
const [pth, query, body0] = gr.split(ep, adapter.beforeRequest(req, ep, params));
|
|
152
|
+
const body = { ...(body0 || {}), pageSize: 1 };
|
|
153
|
+
const data = (await internals.client(pack).request("POST", pth, Object.keys(query).length ? query : null, { jsonBody: body })).json();
|
|
154
|
+
total = pyInt(truthy(data?.count) ? data.count : 0);
|
|
155
|
+
}
|
|
156
|
+
catch (e) {
|
|
157
|
+
if (e instanceof DataError || e instanceof TypeError || e instanceof SyntaxError)
|
|
158
|
+
return null;
|
|
159
|
+
throw e;
|
|
160
|
+
}
|
|
161
|
+
if (got * 2 > total || total - got < PHRASE_GAP_MIN)
|
|
162
|
+
return null;
|
|
163
|
+
const fmt = (n) => n.toLocaleString("en-US");
|
|
164
|
+
return `${req.name}: the quoted search returned ${fmt(got)} documents, under half of the ${fmt(total)} containing all ` +
|
|
165
|
+
`the words of '${words(p.phrase).join(" ")}', so it may be missing documents that hold the phrase ` +
|
|
166
|
+
"(GovInfo's quoted search can skip them): use all_words for the complete candidate list (an upper bound: " +
|
|
167
|
+
"the words may stand apart) and confirm the phrase in the text where the count matters. A full-text " +
|
|
168
|
+
"phrase search is not a title search: for 'X in the title' search the title field (param title).";
|
|
169
|
+
}
|
|
170
|
+
export function billPackageId(p) {
|
|
171
|
+
if (truthy(p.packageId))
|
|
172
|
+
return pyStrip(pyStr(p.packageId));
|
|
173
|
+
const miss = ["congress", "billtype", "number"].filter((k) => !truthy(p[k]));
|
|
174
|
+
if (miss.length) {
|
|
175
|
+
throw new DataError(`govinfo text: give packageId (e.g. CRPT-118hrpt730, PLAW-118publ31), or congress, billtype ` +
|
|
176
|
+
`and number for a bill; missing ${pyRepr(miss)} (params given: ${pyRepr(Object.keys(p).sort(cmp))})`, "bad_request");
|
|
177
|
+
}
|
|
178
|
+
return `BILLS-${pyInt(p.congress)}${pyStr(p.billtype).toLowerCase()}${pyInt(p.number)}${pyStr(truthy(p.version) ? p.version : "enr").toLowerCase()}`;
|
|
179
|
+
}
|
|
180
|
+
function htmlToText(raw) {
|
|
181
|
+
let t = raw.replace(/<(script|style)[^>]*>[\s\S]*?<\/\1>/gi, "");
|
|
182
|
+
t = t.replace(/<br\s*\/?>|<\/p>|<\/div>|<\/h\d>/gi, "\n");
|
|
183
|
+
t = htmlUnescape(t.replace(/<[^>]+>/g, ""));
|
|
184
|
+
t = t.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, ""); // NULs in Daily Digest renditions (2025)
|
|
185
|
+
t = t.replace(/[ \t]+\n/g, "\n");
|
|
186
|
+
return pyStrip(t.replace(/\n{3,}/g, "\n\n"));
|
|
187
|
+
}
|
|
188
|
+
export { htmlToText };
|
|
189
|
+
const MAX_GRANULES = 25; // a package whose text is only at granule level (CRPT, CHRG): read at most this many
|
|
190
|
+
// GPO's text rendition of a document it did not convert holds a cover page and this marker
|
|
191
|
+
const PDF_ONLY = /\[\s*TEXT\s+NOT\s+AVAILABLE\b[^\]]{0,60}\]/i;
|
|
192
|
+
const COVER_CHARS = 5000;
|
|
193
|
+
const THIN_PER_PAGE = 100;
|
|
194
|
+
/** full; pdf_only; partial_pdf; near_empty. */
|
|
195
|
+
export function textStatus(text, pages0) {
|
|
196
|
+
let pages;
|
|
197
|
+
try {
|
|
198
|
+
pages = pyInt(pages0);
|
|
199
|
+
}
|
|
200
|
+
catch {
|
|
201
|
+
pages = 0;
|
|
202
|
+
}
|
|
203
|
+
const n = pyLen(text);
|
|
204
|
+
if (PDF_ONLY.test(text))
|
|
205
|
+
return n < COVER_CHARS || (pages && n < 3 * THIN_PER_PAGE * pages) ? "pdf_only" : "partial_pdf";
|
|
206
|
+
return pages >= 5 && n < THIN_PER_PAGE * pages ? "near_empty" : "full";
|
|
207
|
+
}
|
|
208
|
+
/** GovInfo's public (keyless) PDF of a package or granule. */
|
|
209
|
+
export function pdfUrl(pid, gid = "") {
|
|
210
|
+
return `https://www.govinfo.gov/content/pkg/${pid}/pdf/${gid || pid}.pdf`;
|
|
211
|
+
}
|
|
212
|
+
export function textRow(pid, summary, text, link, maxChars, committees = "", gid = "") {
|
|
213
|
+
const full = pyLen(text);
|
|
214
|
+
const status = textStatus(text, summary.pages ?? "");
|
|
215
|
+
if (truthy(maxChars))
|
|
216
|
+
text = pyCut(text, pyInt(maxChars));
|
|
217
|
+
return { package_id: pid, title: summary.title ?? "", collection: summary.collectionCode ?? "",
|
|
218
|
+
date_issued: summary.dateIssued ?? "", pages: summary.pages ?? "", committees, text_url: link, characters: full,
|
|
219
|
+
truncated: truthy(maxChars) && full > pyInt(maxChars), text_status: status, pdf_url: pdfUrl(pid, gid), text };
|
|
220
|
+
}
|
|
221
|
+
/** A note naming the rows whose text is not the document's text: a term count over them is unread, not zero. */
|
|
222
|
+
function textNotes(name, rows) {
|
|
223
|
+
const out = [];
|
|
224
|
+
const bad = rows.filter((r) => r.text_status === "pdf_only" || r.text_status === "near_empty");
|
|
225
|
+
const part = rows.filter((r) => r.text_status === "partial_pdf");
|
|
226
|
+
const ids = (rs) => rs.slice(0, 10).map((r) => pyStr(truthy(r.granule_id) ? r.granule_id : r.package_id)).join(", ") +
|
|
227
|
+
(rs.length > 10 ? ` and ${rs.length - 10} more` : "");
|
|
228
|
+
if (bad.length) {
|
|
229
|
+
out.push(`${name}${gr.INCOMPLETE} ${bad.length} of ${rows.length} rows have no usable text (${ids(bad)}): GovInfo's ` +
|
|
230
|
+
"text rendition is GPO's 'TEXT NOT AVAILABLE, REFER TO PDF' placeholder or far too short for the " +
|
|
231
|
+
"page count (text_status pdf_only / near_empty). They were not searched: read each one's pdf_url, " +
|
|
232
|
+
"and count them as unread, never as 0 matches or as full text.");
|
|
233
|
+
}
|
|
234
|
+
if (part.length) {
|
|
235
|
+
out.push(`${name}: ${part.length} of ${rows.length} rows have parts only in the PDF (${ids(part)}; text_status ` +
|
|
236
|
+
"partial_pdf): GPO left a 'TEXT NOT AVAILABLE' placeholder inside the text, so a term may be " +
|
|
237
|
+
"missing from the text but present in the pdf_url.");
|
|
238
|
+
}
|
|
239
|
+
return out;
|
|
240
|
+
}
|
|
241
|
+
/** [body, the link shown without a key] of a GovInfo link. */
|
|
242
|
+
async function getRaw(api, link) {
|
|
243
|
+
const u = urlsplit(link);
|
|
244
|
+
const host = u.netloc.replace(/^.*@/, "").replace(/:\d*$/, "").toLowerCase();
|
|
245
|
+
if (u.scheme !== "https" || !["api.govinfo.gov", "www.govinfo.gov"].includes(host)) {
|
|
246
|
+
throw new DataError(`govinfo: not fetching a non-GovInfo link ${u.scheme}://${host || "None"}`, "api");
|
|
247
|
+
}
|
|
248
|
+
const q = {};
|
|
249
|
+
for (const [k, v] of parseQsl(u.query))
|
|
250
|
+
if (k.toLowerCase() !== "api_key" && v !== "")
|
|
251
|
+
q[k] = v;
|
|
252
|
+
let raw;
|
|
253
|
+
if (host === "api.govinfo.gov")
|
|
254
|
+
raw = (await api.get(u.path, q)).text();
|
|
255
|
+
else { // the key is only ever sent to api.govinfo.gov
|
|
256
|
+
const www = new Client("https://www.govinfo.gov", { name: "govinfo-content", rps: 2, burst: 4 });
|
|
257
|
+
raw = (await www.get(u.path, q)).text();
|
|
258
|
+
}
|
|
259
|
+
return [raw, urlunsplit({ ...u, query: pyUrlencode(q) })];
|
|
260
|
+
}
|
|
261
|
+
async function linkText(api, link) {
|
|
262
|
+
const [raw, shown] = await getRaw(api, link);
|
|
263
|
+
return [htmlToText(raw), shown];
|
|
264
|
+
}
|
|
265
|
+
/** Texts, links and committees of a package whose text renditions are per granule. */
|
|
266
|
+
async function granuleTexts(api, pid) {
|
|
267
|
+
const got = await api.getJson(`/packages/${pyQuote(pid)}/granules`, { offsetMark: "*", pageSize: "100" });
|
|
268
|
+
const grans = got.granules || [];
|
|
269
|
+
if (grans.length > MAX_GRANULES) {
|
|
270
|
+
const n = truthy(got.count) ? pyStr(got.count) : `over ${grans.length}`;
|
|
271
|
+
throw new DataError(`govinfo: package ${pid} has ${n} granules and no package-level text: list them (endpoint ` +
|
|
272
|
+
"granules), then ask endpoint text for the ones you need with packageId and granuleId " +
|
|
273
|
+
"(for_each over the granules table)", "bad_request");
|
|
274
|
+
}
|
|
275
|
+
const texts = [], links = [], committees = [];
|
|
276
|
+
for (const g of grans) {
|
|
277
|
+
const gs = await api.getJson(`/packages/${pyQuote(pid)}/granules/${pyQuote(pyStr(g.granuleId))}/summary`);
|
|
278
|
+
committees.push(...(gs.committees || []).filter((c) => truthy(c.committeeName)).map((c) => c.committeeName ?? ""));
|
|
279
|
+
const d = gs.download || {};
|
|
280
|
+
const link = d.txtLink || d.htmlLink;
|
|
281
|
+
if (link) {
|
|
282
|
+
const [tx, shown] = await linkText(api, link);
|
|
283
|
+
texts.push(tx);
|
|
284
|
+
links.push(shown);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
return [texts, links, [...new Set(committees)]];
|
|
288
|
+
}
|
|
289
|
+
async function fetchText(pack, req) {
|
|
290
|
+
const p = req.params || {};
|
|
291
|
+
const pid = billPackageId(p);
|
|
292
|
+
const api = gr.clientFor(pack);
|
|
293
|
+
if (truthy(p.granuleId)) { // one granule: a Record article, a CFR part, a report part
|
|
294
|
+
const gid = pyStr(p.granuleId);
|
|
295
|
+
const gs = await api.getJson(`/packages/${pyQuote(pid)}/granules/${pyQuote(gid)}/summary`);
|
|
296
|
+
const d = gs.download || {};
|
|
297
|
+
const link = d.txtLink || d.htmlLink || "";
|
|
298
|
+
if (!link)
|
|
299
|
+
throw new DataError(`govinfo: granule ${gid} has no text rendition (PDF only); see its pdfLink`, "bad_request");
|
|
300
|
+
const [text, shown] = await linkText(api, link);
|
|
301
|
+
const row = textRow(pid, gs, text, shown, p.max_chars, (gs.committees || []).filter((c) => truthy(c.committeeName)).map((c) => c.committeeName ?? "").join("; "), gid);
|
|
302
|
+
return [{ granule_id: gid, granule_class: gs.granuleClass ?? "", ...row }];
|
|
303
|
+
}
|
|
304
|
+
let summary;
|
|
305
|
+
try {
|
|
306
|
+
summary = await api.getJson(`/packages/${pyQuote(pid)}/summary`);
|
|
307
|
+
}
|
|
308
|
+
catch (e) {
|
|
309
|
+
if (e instanceof DataError && e.kind === "not_found") {
|
|
310
|
+
throw new DataError(`govinfo: package ${pid} not found (check the version code: ih, rh, eh, es, enr, ...)`, "bad_request");
|
|
311
|
+
}
|
|
312
|
+
throw e;
|
|
313
|
+
}
|
|
314
|
+
const d = summary.download || {};
|
|
315
|
+
const link = d.txtLink || d.htmlLink || summary.txtLink || "";
|
|
316
|
+
if (link) {
|
|
317
|
+
const [text, shown] = await linkText(api, link);
|
|
318
|
+
return [textRow(pid, summary, text, shown, p.max_chars)];
|
|
319
|
+
}
|
|
320
|
+
const [texts, links, committees] = await granuleTexts(api, pid);
|
|
321
|
+
if (!texts.length)
|
|
322
|
+
throw new DataError(`govinfo: package ${pid} has no text rendition (PDF only); see its pdfLink`, "bad_request");
|
|
323
|
+
const shown = links[0] + (links.length > 1 ? ` (+${links.length - 1} granules)` : "");
|
|
324
|
+
return [textRow(pid, summary, texts.join("\n\n"), shown, p.max_chars, committees.join("; "))];
|
|
325
|
+
}
|
|
326
|
+
// ---------------------------------------------------------------------- USLM sections
|
|
327
|
+
const USLM_NS = "{http://schemas.gpo.gov/xml/uslm}";
|
|
328
|
+
const CONTAINERS = ["division", "title", "subtitle", "part", "subpart", "chapter", "subchapter"];
|
|
329
|
+
const BLOCKS = ["chapeau", "content", "continuation", "p"]; // where "Section X of title Y is amended" sits
|
|
330
|
+
const SPACED = [...BLOCKS, "heading", "subsection", "paragraph", "subparagraph", "clause", "subclause", "item"];
|
|
331
|
+
const NO_USLM = "GovInfo has USLM XML for public laws from the 113th Congress (2013) on, not for bills, Statutes at " +
|
|
332
|
+
"Large granules or most private laws; check the id, or use endpoint text";
|
|
333
|
+
const local = (tag) => tag.slice(tag.lastIndexOf("}") + 1);
|
|
334
|
+
/** el and its descendants, not entering amended (quoted) text or margin notes. */
|
|
335
|
+
function* walk(el, skip = ["quotedContent", "sidenote"]) {
|
|
336
|
+
yield el;
|
|
337
|
+
for (const c of el.children)
|
|
338
|
+
if (!skip.includes(local(c.tag)))
|
|
339
|
+
yield* walk(c, skip);
|
|
340
|
+
}
|
|
341
|
+
/** Text of an element without its margin notes, whitespace and soft hyphens normalised. */
|
|
342
|
+
function plain(el) {
|
|
343
|
+
const parts = [];
|
|
344
|
+
const go = (e) => {
|
|
345
|
+
parts.push(e.text || "");
|
|
346
|
+
for (const c of e.children) {
|
|
347
|
+
if (local(c.tag) !== "sidenote") {
|
|
348
|
+
go(c);
|
|
349
|
+
if (SPACED.includes(local(c.tag)))
|
|
350
|
+
parts.push(" "); // "HEADING.(a)" -> "HEADING. (a)"
|
|
351
|
+
}
|
|
352
|
+
parts.push(c.tail || "");
|
|
353
|
+
}
|
|
354
|
+
};
|
|
355
|
+
go(el);
|
|
356
|
+
return pyStrip(pyWsSub(parts.join("").split("\u00ad").join(""), " "));
|
|
357
|
+
}
|
|
358
|
+
/** /us/usc/t31/s1120/a/2 -> 31 U.S.C. 1120(a)(2); /us/usc/t31/ch11 -> 31 U.S.C. ch. 11. */
|
|
359
|
+
export function uscCite(href) {
|
|
360
|
+
const m = /^\/us\/usc\/t([\p{L}\p{N}_]+)\/(s|ch)([^/]+)((?:\/[^/]+)*)$/u.exec(href);
|
|
361
|
+
if (!m)
|
|
362
|
+
return href;
|
|
363
|
+
if (m[2] === "ch")
|
|
364
|
+
return `${m[1]} U.S.C. ch. ${m[3]}`;
|
|
365
|
+
return `${m[1]} U.S.C. ${m[3]}` + m[4].split("/").filter(Boolean).map((x) => `(${x})`).join("");
|
|
366
|
+
}
|
|
367
|
+
function childText(el, name) {
|
|
368
|
+
const c = el.find(USLM_NS + name);
|
|
369
|
+
return c ? plain(c) : "";
|
|
370
|
+
}
|
|
371
|
+
const AMEND_WORDS = /\b(?:is|are)\s+(?:hereby\s+)?(?:further\s+)?amended\b|\bby\s+(?:striking|inserting|adding|redesignating)\b/;
|
|
372
|
+
function usc(refs) {
|
|
373
|
+
const out = [];
|
|
374
|
+
for (const r of refs)
|
|
375
|
+
if (local(r.tag) === "ref" && r.get("href").startsWith("/us/usc/"))
|
|
376
|
+
out.push(r.get("href"));
|
|
377
|
+
return out;
|
|
378
|
+
}
|
|
379
|
+
const uniq = (xs) => [...new Set(xs)];
|
|
380
|
+
function sectionRow(pid, sec, pth, maxChars) {
|
|
381
|
+
const cited = uniq(usc(walk(sec)));
|
|
382
|
+
let amended = [];
|
|
383
|
+
for (const b of walk(sec)) {
|
|
384
|
+
// GPO marks most actions <amendingAction>, but not every one ("as amended by this Act, is further amended")
|
|
385
|
+
if (BLOCKS.includes(local(b.tag)) && ([...walk(b)].some((x) => local(x.tag) === "amendingAction") || AMEND_WORDS.test(plain(b)))) {
|
|
386
|
+
amended = amended.concat(usc(walk(b)));
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
amended = uniq(amended);
|
|
390
|
+
const classified = uniq(usc(walk(sec, ["quotedContent"]))).filter((h) => !cited.includes(h));
|
|
391
|
+
const num = sec.find(USLM_NS + "num");
|
|
392
|
+
const text = plain(sec);
|
|
393
|
+
return { package_id: pid, identifier: sec.get("identifier"),
|
|
394
|
+
num: num ? ("value" in num.attrib ? num.attrib.value : plain(num)) : "", heading: childText(sec, "heading"),
|
|
395
|
+
within: pth.join(" > "),
|
|
396
|
+
usc_amended: amended.map(uscCite).join("; "),
|
|
397
|
+
usc_cited: cited.filter((h) => !amended.includes(h)).map(uscCite).join("; "),
|
|
398
|
+
usc_classified: classified.map(uscCite).join("; "),
|
|
399
|
+
amending_actions: [...walk(sec)].filter((x) => local(x.tag) === "amendingAction").length,
|
|
400
|
+
characters: pyLen(text), text: truthy(maxChars) ? pyCut(text, pyInt(maxChars)) : text };
|
|
401
|
+
}
|
|
402
|
+
/** One row per section of a USLM law (a law without sections gives one row for its body). */
|
|
403
|
+
export function uslmSections(pid, raw, maxChars) {
|
|
404
|
+
let root;
|
|
405
|
+
try {
|
|
406
|
+
root = fromstring(raw);
|
|
407
|
+
}
|
|
408
|
+
catch (e) {
|
|
409
|
+
if (e instanceof XmlParseError)
|
|
410
|
+
throw new DataError(`govinfo: ${pid} USLM is not well-formed XML (${e.message})`, "api");
|
|
411
|
+
throw e;
|
|
412
|
+
}
|
|
413
|
+
const main = root.find(USLM_NS + "main");
|
|
414
|
+
if (!main)
|
|
415
|
+
throw new DataError(`govinfo: ${pid} has no <main> in its USLM; use endpoint text`, "api");
|
|
416
|
+
const rows = [];
|
|
417
|
+
const go = (el, pth) => {
|
|
418
|
+
for (const c of el.children) {
|
|
419
|
+
const name = local(c.tag);
|
|
420
|
+
if (name === "section")
|
|
421
|
+
rows.push(sectionRow(pid, c, pth, maxChars));
|
|
422
|
+
else if (CONTAINERS.includes(name))
|
|
423
|
+
go(c, [...pth, [childText(c, "num"), childText(c, "heading")].filter(Boolean).join(" ")]);
|
|
424
|
+
else if (name !== "quotedContent" && name !== "sidenote")
|
|
425
|
+
go(c, pth);
|
|
426
|
+
}
|
|
427
|
+
};
|
|
428
|
+
go(main, []);
|
|
429
|
+
if (!rows.length) {
|
|
430
|
+
rows.push(sectionRow(pid, main, [], maxChars));
|
|
431
|
+
rows[0].identifier = rows[0].identifier || pid;
|
|
432
|
+
}
|
|
433
|
+
return rows;
|
|
434
|
+
}
|
|
435
|
+
async function fetchSections(pack, req) {
|
|
436
|
+
const p = req.params || {};
|
|
437
|
+
const pid = pyStrip(pyStr(truthy(p.packageId) ? p.packageId : ""));
|
|
438
|
+
const api = gr.clientFor(pack);
|
|
439
|
+
let raw;
|
|
440
|
+
try {
|
|
441
|
+
[raw] = await getRaw(api, `https://api.govinfo.gov/packages/${pyQuote(pid)}/uslm`);
|
|
442
|
+
}
|
|
443
|
+
catch (e) {
|
|
444
|
+
if (e instanceof DataError && (e.kind === "not_found" || e.kind === "bad_request")) { // a missing rendition: HTTP 400
|
|
445
|
+
throw new DataError(`govinfo: no USLM XML for ${pid}. ${NO_USLM}`, "bad_request");
|
|
446
|
+
}
|
|
447
|
+
throw e;
|
|
448
|
+
}
|
|
449
|
+
return uslmSections(pid, raw, p.max_chars);
|
|
450
|
+
}
|
|
451
|
+
/** Python's _calls: API calls a spec makes (the DEMO_KEY allows about 10 an hour). */
|
|
452
|
+
export function calls(spec) {
|
|
453
|
+
const E = eps().endpoints;
|
|
454
|
+
let n = 0;
|
|
455
|
+
for (const r of spec.requests || []) {
|
|
456
|
+
const ep = E[r.endpoint || ""] || {};
|
|
457
|
+
const size = pyInt((ep.paging || {}).size || 1000);
|
|
458
|
+
const text = r.endpoint === "text";
|
|
459
|
+
n += text ? (truthy((r.params || {}).granuleId) ? 2 : 4) : r.endpoint === "sections" ? 1
|
|
460
|
+
: Math.max(1, Math.ceil(pyInt(r.max_rows || size) / size));
|
|
461
|
+
if (r.endpoint === "search" && words(truthy((r.params || {}).phrase) ? r.params.phrase : "").length > 1)
|
|
462
|
+
n += 1;
|
|
463
|
+
}
|
|
464
|
+
return n;
|
|
465
|
+
}
|
|
466
|
+
const adapter = {
|
|
467
|
+
checkSpec(spec) {
|
|
468
|
+
const probs = [];
|
|
469
|
+
const known = collections();
|
|
470
|
+
for (const r of spec.requests || []) {
|
|
471
|
+
const p = r.params || {}, ep = r.endpoint;
|
|
472
|
+
const n = pyRepr(r.name ?? null);
|
|
473
|
+
if (ep === "search") {
|
|
474
|
+
if (!["text", "phrase", "all_words", "any", "title", "query", "collection", "publisher"].some((k) => truthy(p[k]))) {
|
|
475
|
+
probs.push(`request ${n}: search needs text, phrase, all_words, any, title, query, collection or publisher`);
|
|
476
|
+
}
|
|
477
|
+
if (p.any !== null && p.any !== undefined && !alternatives(p.any).some((x) => words(x).length)) {
|
|
478
|
+
probs.push(`request ${n}: any needs a list of words or phrases`);
|
|
479
|
+
}
|
|
480
|
+
if (/\bdateissued\s*:/i.test(pyStr(truthy(p.query) ? p.query : ""))) {
|
|
481
|
+
probs.push(`request ${n}: GovInfo answers HTTP 500 to dateIssued: inside query (verified ` +
|
|
482
|
+
"2026-10-28): use date_from / date_to, which the pack sends as publishdate:range(...)");
|
|
483
|
+
}
|
|
484
|
+
if (["text", "phrase", "title"].some((k) => truthy(p[k]) && !words(p[k]).length)) {
|
|
485
|
+
probs.push(`request ${n}: text/phrase/title has no searchable words`);
|
|
486
|
+
}
|
|
487
|
+
if (["text", "phrase", "all_words", "any", "title", "query"].map((k) => pyStr(truthy(p[k]) ? p[k] : "")).join(" ").includes("*")) {
|
|
488
|
+
probs.push(`request ${n}: GovInfo search answers a wildcard (*) with HTTP 500: list the word ` +
|
|
489
|
+
"forms instead (title:tariff OR title:tariffs)");
|
|
490
|
+
}
|
|
491
|
+
const bad = list(truthy(p.collection) ? p.collection : []).filter((c) => !known.has(c.toUpperCase()));
|
|
492
|
+
if (bad.length)
|
|
493
|
+
probs.push(`request ${n}: unknown collection ${pyRepr(bad)} (search the catalog with request_schema)`);
|
|
494
|
+
if (truthy(p.billtype) && !BILL_TYPES.includes(pyStr(p.billtype).toLowerCase())) {
|
|
495
|
+
probs.push(`request ${n}: billtype must be one of ${pyTupleRepr(BILL_TYPES)}`);
|
|
496
|
+
}
|
|
497
|
+
if (truthy(p.sort) && !(typeof p.sort === "string" && p.sort in SORTS)) {
|
|
498
|
+
probs.push(`request ${n}: sort must be one of ${pyRepr(Object.keys(SORTS).sort(cmp))}`);
|
|
499
|
+
}
|
|
500
|
+
for (const k of ["date_from", "date_to"]) {
|
|
501
|
+
if (truthy(p[k]) && !/^\d{4}-\d{2}-\d{2}$/.test(pyStr(p[k])))
|
|
502
|
+
probs.push(`request ${n}: ${k} must be YYYY-MM-DD`);
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
else if (ep === "text") {
|
|
506
|
+
if (!truthy(p.packageId) && !["congress", "billtype", "number"].every((k) => truthy(p[k]))) {
|
|
507
|
+
probs.push(`request ${n}: text needs packageId, or congress + billtype + number`);
|
|
508
|
+
}
|
|
509
|
+
else if (!truthy(p.packageId) && !BILL_TYPES.includes(pyStr(p.billtype).toLowerCase())) {
|
|
510
|
+
probs.push(`request ${n}: billtype must be one of ${pyTupleRepr(BILL_TYPES)}`);
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
else if (ep === "sections") {
|
|
514
|
+
if (!/^PLAW-\d+(publ|pvtl)\d+$/.test(pyStr(truthy(p.packageId) ? p.packageId : ""))) {
|
|
515
|
+
probs.push(`request ${n}: sections needs a PLAW packageId (e.g. PLAW-117publ169). ${NO_USLM}`);
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
probs.push(...outsideCoverage(r.name ?? null, p));
|
|
519
|
+
}
|
|
520
|
+
return probs;
|
|
521
|
+
},
|
|
522
|
+
beforeRequest(_req, ep, params) {
|
|
523
|
+
if (String(ep.path ?? "").startsWith("/published")) {
|
|
524
|
+
// GovInfo's end date is exclusive: the spec's end_date is inclusive, so ask for the next day
|
|
525
|
+
const end = pyStr(truthy(params.end_date) ? params.end_date : "");
|
|
526
|
+
if (/^\d{4}-\d{2}-\d{2}$/.test(end)) {
|
|
527
|
+
const [y, m, d] = end.split("-").map(Number);
|
|
528
|
+
const dt = new Date(Date.UTC(2000, m - 1, d));
|
|
529
|
+
dt.setUTCFullYear(y);
|
|
530
|
+
if (dt.getUTCMonth() !== m - 1)
|
|
531
|
+
throw new RangeError(`day is out of range for month: ${end}`);
|
|
532
|
+
dt.setUTCDate(dt.getUTCDate() + 1);
|
|
533
|
+
const nxt = `${String(dt.getUTCFullYear()).padStart(4, "0")}-${String(dt.getUTCMonth() + 1).padStart(2, "0")}-${String(dt.getUTCDate()).padStart(2, "0")}`;
|
|
534
|
+
params = { ...params, end_date: nxt };
|
|
535
|
+
}
|
|
536
|
+
return params;
|
|
537
|
+
}
|
|
538
|
+
if (ep.path !== "/search")
|
|
539
|
+
return params;
|
|
540
|
+
const out = Object.fromEntries(Object.entries(params).filter(([k]) => !SEARCH_KEYS.includes(k)));
|
|
541
|
+
out.query = buildQuery(params);
|
|
542
|
+
out.sorts = SORTS[truthy(params.sort) ? pyStr(params.sort) : "relevance"];
|
|
543
|
+
return out;
|
|
544
|
+
},
|
|
545
|
+
async fetch(spec, pack, _route) {
|
|
546
|
+
const tables = {};
|
|
547
|
+
let notes = [];
|
|
548
|
+
const texts = new Map();
|
|
549
|
+
for (const r of spec.requests) {
|
|
550
|
+
if (r.endpoint === "text") {
|
|
551
|
+
const rows = await internals.fetchText(pack, r);
|
|
552
|
+
tables[r.name] = rows;
|
|
553
|
+
const g = r._group || r.name; // for_each: one note
|
|
554
|
+
texts.set(g, [...(texts.get(g) || []), ...rows]);
|
|
555
|
+
}
|
|
556
|
+
else if (r.endpoint === "sections") {
|
|
557
|
+
tables[r.name] = await fetchSections(pack, r);
|
|
558
|
+
}
|
|
559
|
+
else {
|
|
560
|
+
const rec = (eps().endpoints[r.endpoint || ""] || {}).records ?? "";
|
|
561
|
+
const [t, n] = await gr.fetchSpec(pack, { requests: [r] }, pyJsonClient(gr.clientFor(pack), pack, rec));
|
|
562
|
+
Object.assign(tables, t);
|
|
563
|
+
notes = notes.concat(n);
|
|
564
|
+
const capped = n.some((x) => x.startsWith(`${r.name}: stopped at`));
|
|
565
|
+
if (r.endpoint === "search" && !capped) {
|
|
566
|
+
const got = t[r.name];
|
|
567
|
+
const note = await phraseCountNote(pack, r, Array.isArray(got) ? got.length : 0);
|
|
568
|
+
if (note)
|
|
569
|
+
notes.push(note);
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
for (const [name, rows] of texts)
|
|
574
|
+
notes = notes.concat(textNotes(name, rows));
|
|
575
|
+
return [tables, notes];
|
|
576
|
+
},
|
|
577
|
+
keyOnlyNeeds(spec) {
|
|
578
|
+
const n = calls(spec);
|
|
579
|
+
return n > DEMO_LIMIT ? [`about ${n} API calls (the keyless DEMO_KEY allows about 10 an hour on GovInfo)`] : [];
|
|
580
|
+
},
|
|
581
|
+
manualItems(spec, pack) {
|
|
582
|
+
const items = [];
|
|
583
|
+
for (let r of spec.requests || []) {
|
|
584
|
+
if (r.endpoint === "text") {
|
|
585
|
+
let pid;
|
|
586
|
+
try {
|
|
587
|
+
pid = billPackageId(r.params || {});
|
|
588
|
+
}
|
|
589
|
+
catch {
|
|
590
|
+
continue;
|
|
591
|
+
}
|
|
592
|
+
items.push({ name: `${r.name}#1`, request: r.name, filename: `${r.name}.htm`,
|
|
593
|
+
url: `https://www.govinfo.gov/content/pkg/${pid}/html/${pid}.htm`, needs_key: false, method: "GET", body: null,
|
|
594
|
+
note: "public text page, no key; if it is missing, open https://www.govinfo.gov/app/details/" + pid });
|
|
595
|
+
continue;
|
|
596
|
+
}
|
|
597
|
+
if (r.endpoint === "sections") {
|
|
598
|
+
const pid = pyStr(truthy((r.params || {}).packageId) ? r.params.packageId : "");
|
|
599
|
+
items.push({ name: `${r.name}#1`, request: r.name, filename: `${r.name}.xml`,
|
|
600
|
+
url: `https://www.govinfo.gov/content/pkg/${pid}/uslm/${pid}.xml`, needs_key: false, method: "GET", body: null,
|
|
601
|
+
note: "public USLM XML, no key" });
|
|
602
|
+
continue;
|
|
603
|
+
}
|
|
604
|
+
let ep = {};
|
|
605
|
+
if (r.endpoint === "search") {
|
|
606
|
+
const [e, params] = gr.resolve(pack, r);
|
|
607
|
+
ep = e;
|
|
608
|
+
r = { ...r, params: { ...adapter.beforeRequest(r, ep, params) } };
|
|
609
|
+
}
|
|
610
|
+
for (const it of gr.manualItems(pack, { requests: [r] })) {
|
|
611
|
+
if (it.body !== null && it.body !== undefined && r.endpoint === "search")
|
|
612
|
+
it.body = { ...(ep.body || {}), ...it.body, pageSize: 100 };
|
|
613
|
+
it.url += (it.url.includes("?") ? "&" : "?") + "api_key=DEMO_KEY";
|
|
614
|
+
it.note = (it.note ? it.note + "; " : "") +
|
|
615
|
+
"DEMO_KEY is api.data.gov's public demo key (about 10 requests an hour); replace it with your own key for more";
|
|
616
|
+
items.push(it);
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
return items;
|
|
620
|
+
},
|
|
621
|
+
parseUpload(pack, req, files) {
|
|
622
|
+
const read = (f) => {
|
|
623
|
+
const b = fs.readFileSync(f);
|
|
624
|
+
return (b[0] === 0xef && b[1] === 0xbb && b[2] === 0xbf ? b.subarray(3) : b).toString("utf8");
|
|
625
|
+
};
|
|
626
|
+
if (req.endpoint === "sections") {
|
|
627
|
+
const p = req.params || {};
|
|
628
|
+
return files.flatMap((f) => uslmSections(pyStr(truthy(p.packageId) ? p.packageId : ""), read(f), p.max_chars));
|
|
629
|
+
}
|
|
630
|
+
if (req.endpoint !== "text")
|
|
631
|
+
return gr.parseUpload(pack, req, files);
|
|
632
|
+
return files.map((f) => textRow(billPackageId(req.params || {}), {}, htmlToText(read(f)), path.basename(f), (req.params || {}).max_chars));
|
|
633
|
+
},
|
|
634
|
+
};
|
|
635
|
+
export default adapter;
|
|
636
|
+
//# sourceMappingURL=govinfo.js.map
|