data-analyst-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -2
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +716 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +24 -0
- package/dist/report/scan_secrets.js +265 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +247 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +54 -4
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +170 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,639 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Declarative fetching for JSON/CSV REST APIs (Socrata, DKAN, CKAN, plain REST) from a pack's endpoints.json.
|
|
3
|
+
* The endpoints.json format:
|
|
4
|
+
*
|
|
5
|
+
* {
|
|
6
|
+
* "base_url": "https://data.cdc.gov/resource", "timeout": 30, "retry_html_400": 3, "headers": {...},
|
|
7
|
+
* "endpoints": {
|
|
8
|
+
* "<name>": {
|
|
9
|
+
* "path": "/{dataset}.json", "method": "GET" | "POST", "description": "...",
|
|
10
|
+
* "params": {"dataset": {"in": "path", "required": true, "desc": "...", "catalog": true},
|
|
11
|
+
* // "in": "query" (GET default) | "path" | "body" (POST default) | "spec" (never sent) | "records"
|
|
12
|
+
* // "flag": true (a bare ?flagName, sent only when true), "raw": true (path value keeps slashes)
|
|
13
|
+
* "$where": {"desc": "SoQL filter"}, "state": {"desc": "...", "enum": [...]}},
|
|
14
|
+
* "defaults": {"$order": ":id"}, "body": {...},
|
|
15
|
+
* "records": "results" | "" | "$table" | "$csv" | "$xlsx" | "$xls" | "$zip:<glob>",
|
|
16
|
+
* "sheet": "Table 1", "header_row": 1, "total": "meta.count",
|
|
17
|
+
* "paging": {"type": "offset", "limit_param": "$limit", "offset_param": "$offset", "size": 1000, "manual_size": 50000}
|
|
18
|
+
* | {"type": "page", "page_param": "page", "size_param": "per_page", "size": 100, "start": 1, "min_size": 5}
|
|
19
|
+
* | {"type": "cursor", "cursor_param": "cursor", "next": "pagination.next_cursor", "size_param": "limit"}
|
|
20
|
+
* | {"type": "link", "next": "links.next"} | {"type": "none"},
|
|
21
|
+
* "max_rows": 100000, "unique_key": "appl_id", "truncated_note": {"when_default": "sort_field", "text": "..."},
|
|
22
|
+
* "missing": ["", "(NA)", "."], "flatten": true
|
|
23
|
+
* }
|
|
24
|
+
* },
|
|
25
|
+
* "catalog": {"<alias>": {"id": "bi63-dtpu", "name": "...", "description": "...", "endpoint": "<name>"}}
|
|
26
|
+
* }
|
|
27
|
+
*
|
|
28
|
+
* spec: {"description": "...", "requests": [{"name": "deaths", "endpoint": "query",
|
|
29
|
+
* "params": {"dataset": "leading_death", "$where": "year = '2017'"}, "max_rows": 5000}]}
|
|
30
|
+
*/
|
|
31
|
+
import fs from "node:fs";
|
|
32
|
+
import path from "node:path";
|
|
33
|
+
import { Cache } from "./cache.js";
|
|
34
|
+
import { Client, DataError } from "./http.js";
|
|
35
|
+
import { parsePyJson, pyJsonDumps, pyQuote, pyStr, pyUrlencode, sleep } from "./py.js";
|
|
36
|
+
import { decodeText, fnmatch, readCsv, sheetRecords, unzip, xlsRecords } from "./tabular.js";
|
|
37
|
+
/** A note containing this marks its table incomplete (status partial), like a max_rows cut. */
|
|
38
|
+
export const INCOMPLETE = ": incomplete:";
|
|
39
|
+
export const FOR_EACH_MAX = 2000;
|
|
40
|
+
export function dig(obj, p) {
|
|
41
|
+
if (p === "" || p === null || p === undefined)
|
|
42
|
+
return obj;
|
|
43
|
+
for (const part of p.split(".")) {
|
|
44
|
+
if (obj === null || obj === undefined)
|
|
45
|
+
return null;
|
|
46
|
+
if (Array.isArray(obj)) {
|
|
47
|
+
const i = Number(part);
|
|
48
|
+
obj = Number.isInteger(i) && i >= 0 && i < obj.length ? obj[i] : null;
|
|
49
|
+
}
|
|
50
|
+
else
|
|
51
|
+
obj = obj && typeof obj === "object" ? obj[part] ?? null : null;
|
|
52
|
+
}
|
|
53
|
+
return obj;
|
|
54
|
+
}
|
|
55
|
+
const scalar = (x) => typeof x === "string" || typeof x === "number" || typeof x === "boolean";
|
|
56
|
+
export function flatten(rec, prefix = "") {
|
|
57
|
+
if (!rec || typeof rec !== "object" || Array.isArray(rec))
|
|
58
|
+
return { [prefix || "value"]: rec };
|
|
59
|
+
const out = {};
|
|
60
|
+
for (const [k, v] of Object.entries(rec)) {
|
|
61
|
+
const key = prefix ? `${prefix}.${k}` : k;
|
|
62
|
+
if (v && typeof v === "object" && !Array.isArray(v))
|
|
63
|
+
Object.assign(out, flatten(v, key));
|
|
64
|
+
else if (Array.isArray(v)) {
|
|
65
|
+
out[key] = v.length && !v.every(scalar) ? pyJsonDumps(v, false, false) : v.map(pyStr).join("; ");
|
|
66
|
+
}
|
|
67
|
+
else
|
|
68
|
+
out[key] = v;
|
|
69
|
+
}
|
|
70
|
+
return out;
|
|
71
|
+
}
|
|
72
|
+
export function clientFor(pack, eps, extra = {}) {
|
|
73
|
+
const P = pack.policy;
|
|
74
|
+
const e = eps ?? pack.endpoints ?? {};
|
|
75
|
+
const L = P.limits;
|
|
76
|
+
return new Client(e.base_url || P.routes?.api?.base || "", {
|
|
77
|
+
name: pack.id, auth: P.auth, rps: L.rps || 5, burst: L.burst || 10,
|
|
78
|
+
checkError: pack.adapter?.checkError ?? null, errorsPerMin: L.errors_per_min ?? null,
|
|
79
|
+
bytesPerMin: L.bytes_per_min ?? null, headers: e.headers ?? null, timeout: e.timeout ?? 30,
|
|
80
|
+
retryHtml400: e.retry_html_400 ?? 0,
|
|
81
|
+
// a persistent cache only where the source's terms allow it (cache.mode "persistent"); never for test fixtures
|
|
82
|
+
cache: P.cache.mode === "persistent" && !process.env.DATA_ANALYST_FIXTURES ? new Cache(pack.id, P) : null, ...extra,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
// ---------------------------------------------------------------------- spec -> requests
|
|
86
|
+
/** [endpoint definition, merged params, problems] for one spec request. */
|
|
87
|
+
export function resolve(pack, req) {
|
|
88
|
+
const eps = pack.endpoints?.endpoints || {};
|
|
89
|
+
const cat = pack.endpoints?.catalog || {};
|
|
90
|
+
const probs = [];
|
|
91
|
+
let name = req.endpoint;
|
|
92
|
+
if (!name && req.dataset && cat[req.dataset])
|
|
93
|
+
name = cat[req.dataset].endpoint;
|
|
94
|
+
const ep = eps[name || ""];
|
|
95
|
+
if (!ep) {
|
|
96
|
+
return [{}, {}, [`request ${JSON.stringify(req.name)}: unknown endpoint ${JSON.stringify(name ?? null)} (known: ${Object.keys(eps).sort().join(", ")})`]];
|
|
97
|
+
}
|
|
98
|
+
const given = req.params || {};
|
|
99
|
+
const defaults = { ...(ep.defaults || {}) };
|
|
100
|
+
if ("$group" in given && !("$order" in given))
|
|
101
|
+
delete defaults.$order; // SoQL: an aggregate cannot keep a row-id order
|
|
102
|
+
const params = { ...defaults, ...given };
|
|
103
|
+
for (const [pname, d] of Object.entries(ep.params || {})) {
|
|
104
|
+
if (d.catalog) {
|
|
105
|
+
let v = pname in params ? params[pname] : req.dataset;
|
|
106
|
+
if (typeof v === "string" && cat[v])
|
|
107
|
+
v = cat[v].id;
|
|
108
|
+
if (v !== undefined && v !== null)
|
|
109
|
+
params[pname] = v;
|
|
110
|
+
}
|
|
111
|
+
const v = params[pname];
|
|
112
|
+
if (d.required && (v === undefined || v === null || v === "")) {
|
|
113
|
+
probs.push(`request ${JSON.stringify(req.name)}: parameter ${JSON.stringify(pname)} is required (${d.desc || ""})`);
|
|
114
|
+
}
|
|
115
|
+
if (d.enum && v !== undefined && v !== null && v !== "" && !d.enum.map(String).includes(pyStr(v))) {
|
|
116
|
+
probs.push(`request ${JSON.stringify(req.name)}: ${pname}=${JSON.stringify(v)} is not one of ${JSON.stringify(d.enum)}`);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (!req.name || !/^[A-Za-z0-9_-]+$/.test(String(req.name))) {
|
|
120
|
+
probs.push("every request needs a name of letters, digits, _ or - (it names the CSV file)");
|
|
121
|
+
}
|
|
122
|
+
return [ep, params, probs];
|
|
123
|
+
}
|
|
124
|
+
/** [endpoint path with {x} filled, query params, JSON body]. */
|
|
125
|
+
export function split(ep, params) {
|
|
126
|
+
const pdefs = ep.params || {};
|
|
127
|
+
let p = ep.path;
|
|
128
|
+
const query = {};
|
|
129
|
+
let body = ep.body !== undefined && ep.body !== null ? structuredClone(ep.body) : null;
|
|
130
|
+
const flags = [];
|
|
131
|
+
for (const [k, v] of Object.entries(params)) {
|
|
132
|
+
const d = pdefs[k] || {};
|
|
133
|
+
const where = d.in || (ep.method === "POST" ? "body" : "query");
|
|
134
|
+
if (where === "spec" || where === "records")
|
|
135
|
+
continue;
|
|
136
|
+
if (d.flag) {
|
|
137
|
+
if (v === true || ["true", "1", "yes"].includes(String(v).trim().toLowerCase()))
|
|
138
|
+
flags.push(pyQuote(k));
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
if (where === "path" || p.includes(`{${k}}`))
|
|
142
|
+
p = p.replace(`{${k}}`, pyQuote(pyStr(v), d.raw ? "/" : ""));
|
|
143
|
+
else if (where === "body") {
|
|
144
|
+
body ??= {};
|
|
145
|
+
body[k] = v;
|
|
146
|
+
}
|
|
147
|
+
else
|
|
148
|
+
query[k] = v;
|
|
149
|
+
}
|
|
150
|
+
if (flags.length)
|
|
151
|
+
p += (p.includes("?") ? "&" : "?") + flags.join("&");
|
|
152
|
+
return [p, query, body];
|
|
153
|
+
}
|
|
154
|
+
export function checkSpec(pack, spec) {
|
|
155
|
+
const probs = [];
|
|
156
|
+
const reqs = spec.requests || [];
|
|
157
|
+
if (!reqs.length)
|
|
158
|
+
probs.push('spec needs a non-empty "requests" list');
|
|
159
|
+
const names = reqs.map((r) => r.name);
|
|
160
|
+
if (new Set(names).size !== names.length)
|
|
161
|
+
probs.push("request names must be unique");
|
|
162
|
+
for (const r of reqs)
|
|
163
|
+
probs.push(...resolve(pack, r)[2]);
|
|
164
|
+
if (pack.adapter?.checkSpec)
|
|
165
|
+
probs.push(...(pack.adapter.checkSpec(spec, pack) || []));
|
|
166
|
+
return probs;
|
|
167
|
+
}
|
|
168
|
+
// ---------------------------------------------------------------------- one request per row of a table
|
|
169
|
+
/** for_each requests as one request per distinct value (or combination) of a column of an earlier fetch's CSV. */
|
|
170
|
+
export function expandForEach(spec, base) {
|
|
171
|
+
const reqs = [];
|
|
172
|
+
for (const r of spec.requests || []) {
|
|
173
|
+
const fe = r.for_each;
|
|
174
|
+
if (!fe) {
|
|
175
|
+
reqs.push(r);
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
const csvPath = path.isAbsolute(fe.csv || "") ? fe.csv : path.join(base, fe.csv || "");
|
|
179
|
+
if (!fs.existsSync(csvPath)) {
|
|
180
|
+
throw new DataError(`request ${JSON.stringify(r.name)}: for_each csv ${JSON.stringify(fe.csv)} not found (fetch that ` +
|
|
181
|
+
"table first; the path is relative to the workspace)", "bad_request");
|
|
182
|
+
}
|
|
183
|
+
const pairs = fe.map && Object.keys(fe.map).length ? { ...fe.map } : { [fe.param || fe.column]: fe.column };
|
|
184
|
+
const rows = readCsv(decodeText(fs.readFileSync(csvPath)));
|
|
185
|
+
const cols = rows.length ? Object.keys(rows[0]) : [];
|
|
186
|
+
const miss = Object.values(pairs).filter((c) => !cols.includes(c));
|
|
187
|
+
if (miss.length) {
|
|
188
|
+
throw new DataError(`request ${JSON.stringify(r.name)}: for_each column(s) ${JSON.stringify(miss)} not in ${fe.csv} ` +
|
|
189
|
+
`(columns: ${JSON.stringify(cols)})`, "bad_request");
|
|
190
|
+
}
|
|
191
|
+
const low = !!fe.lower, pre = String(fe.strip || "");
|
|
192
|
+
const combos = new Map();
|
|
193
|
+
for (const x of rows) {
|
|
194
|
+
let vals = Object.values(pairs).map((c) => (low ? String(x[c] ?? "").trim().toLowerCase() : String(x[c] ?? "").trim()));
|
|
195
|
+
vals = vals.map((v) => (pre && v.startsWith(pre) ? v.slice(pre.length) : v));
|
|
196
|
+
if (vals.every(Boolean))
|
|
197
|
+
combos.set(JSON.stringify(vals), vals);
|
|
198
|
+
}
|
|
199
|
+
const max = Number(fe.max ?? FOR_EACH_MAX);
|
|
200
|
+
if (combos.size > max) {
|
|
201
|
+
throw new DataError(`request ${JSON.stringify(r.name)}: for_each gives ${combos.size} requests (max ${max}): filter the ` +
|
|
202
|
+
'table first, or set "max"', "bad_request");
|
|
203
|
+
}
|
|
204
|
+
const names = Object.keys(pairs);
|
|
205
|
+
let i = 0;
|
|
206
|
+
for (const vals of combos.values()) {
|
|
207
|
+
i++;
|
|
208
|
+
const { for_each: _drop, ...rest } = r;
|
|
209
|
+
const key = names.map((n, j) => [n, vals[j]]);
|
|
210
|
+
reqs.push({ ...rest, name: `${r.name}__${i}`, params: { ...(r.params || {}), ...Object.fromEntries(key) },
|
|
211
|
+
_group: r.name, _key: key, _orig: r });
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
return { ...spec, requests: reqs };
|
|
215
|
+
}
|
|
216
|
+
/** The spec as written (each for_each request once). */
|
|
217
|
+
export function collapseForEach(spec) {
|
|
218
|
+
const reqs = [];
|
|
219
|
+
const seen = new Set();
|
|
220
|
+
for (const r of spec.requests || []) {
|
|
221
|
+
if (r._group === undefined)
|
|
222
|
+
reqs.push(r);
|
|
223
|
+
else if (!seen.has(r._group)) {
|
|
224
|
+
seen.add(r._group);
|
|
225
|
+
reqs.push(r._orig);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
return { ...spec, requests: reqs };
|
|
229
|
+
}
|
|
230
|
+
/** The expanded requests' tables as one table per for_each request; each row keeps the value it was asked for. */
|
|
231
|
+
export function mergeForEach(spec, tables) {
|
|
232
|
+
const out = {};
|
|
233
|
+
for (const r of spec.requests || []) {
|
|
234
|
+
const rows = tables[r.name] || [];
|
|
235
|
+
if (r._group === undefined) {
|
|
236
|
+
out[r.name] = rows;
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
if (!Array.isArray(rows))
|
|
240
|
+
throw new DataError(`request '${r.name}': for_each cannot merge a streamed table`, "spec");
|
|
241
|
+
const asked = Object.fromEntries(r._key);
|
|
242
|
+
const merged = (out[r._group] ??= []);
|
|
243
|
+
for (const row of rows) {
|
|
244
|
+
const add = Object.fromEntries(Object.entries(asked).filter(([k]) => !(k in row)));
|
|
245
|
+
merged.push({ ...add, ...row });
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
const named = new Set((spec.requests || []).map((r) => r.name));
|
|
249
|
+
for (const [name, rows] of Object.entries(tables))
|
|
250
|
+
if (!named.has(name) && !(name in out))
|
|
251
|
+
out[name] = rows;
|
|
252
|
+
return out;
|
|
253
|
+
}
|
|
254
|
+
export function records(ep, resp) {
|
|
255
|
+
const p = ep.records ?? "";
|
|
256
|
+
if (p === "$csv")
|
|
257
|
+
return readCsv(resp.text());
|
|
258
|
+
if (p === "$xlsx")
|
|
259
|
+
return sheetRecords(ep, resp.body);
|
|
260
|
+
if (p === "$xls") {
|
|
261
|
+
if (resp.body.subarray(0, 4).equals(Buffer.from("PK\x03\x04", "latin1")))
|
|
262
|
+
return sheetRecords(ep, resp.body);
|
|
263
|
+
if (!resp.body.subarray(0, 8).equals(Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]))) {
|
|
264
|
+
throw new DataError("expected an .xls workbook, got something else (an HTML page?)", "api");
|
|
265
|
+
}
|
|
266
|
+
return xlsRecords(ep, resp.body);
|
|
267
|
+
}
|
|
268
|
+
if (p.startsWith("$zip:")) {
|
|
269
|
+
const z = unzip(resp.body);
|
|
270
|
+
const glob = p.slice(5).toLowerCase() || "*";
|
|
271
|
+
const members = Object.keys(z).filter((n) => fnmatch(n.toLowerCase(), glob) && !n.endsWith("/")).sort();
|
|
272
|
+
if (!members.length) {
|
|
273
|
+
throw new DataError(`no member matching ${JSON.stringify(p.slice(5))} in the zip (${Object.keys(z).slice(0, 10).join(", ")})`, "api");
|
|
274
|
+
}
|
|
275
|
+
const rows = [];
|
|
276
|
+
for (const n of members) {
|
|
277
|
+
if (n.toLowerCase().endsWith(".xlsx"))
|
|
278
|
+
rows.push(...sheetRecords(ep, z[n]));
|
|
279
|
+
else if (n.toLowerCase().endsWith(".xls"))
|
|
280
|
+
rows.push(...xlsRecords(ep, z[n]));
|
|
281
|
+
else
|
|
282
|
+
rows.push(...readCsv(decodeText(z[n])));
|
|
283
|
+
}
|
|
284
|
+
return rows;
|
|
285
|
+
}
|
|
286
|
+
const data = resp.json();
|
|
287
|
+
if (p === "$table") {
|
|
288
|
+
if (!data || !data.length)
|
|
289
|
+
return [];
|
|
290
|
+
const [head, ...rows] = data;
|
|
291
|
+
return rows.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i]])));
|
|
292
|
+
}
|
|
293
|
+
const recs = dig(data, p);
|
|
294
|
+
if (recs === null || recs === undefined)
|
|
295
|
+
return [];
|
|
296
|
+
return Array.isArray(recs) ? recs : [recs];
|
|
297
|
+
}
|
|
298
|
+
export function clean(rows, ep) {
|
|
299
|
+
const missing = new Set(ep.missing ?? [""]);
|
|
300
|
+
return rows.map((r0) => {
|
|
301
|
+
const r = ep.flatten ?? true ? flatten(r0) : r0 && typeof r0 === "object" ? r0 : { value: r0 };
|
|
302
|
+
const o = {};
|
|
303
|
+
for (const [k, v] of Object.entries(r)) {
|
|
304
|
+
o[k] = v === null || v === undefined || (typeof v === "string" && missing.has(v.trim())) ? "" : v;
|
|
305
|
+
}
|
|
306
|
+
return o;
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
/** The endpoint with the spec's file-reading params ("in": "records": sheet, header_row) applied. */
|
|
310
|
+
export function withRecordParams(ep, params) {
|
|
311
|
+
const pdefs = ep.params || {};
|
|
312
|
+
const over = Object.fromEntries(Object.entries(params).filter(([k, v]) => pdefs[k]?.in === "records" && v !== null && v !== undefined && v !== ""));
|
|
313
|
+
return Object.keys(over).length ? { ...ep, ...over } : ep;
|
|
314
|
+
}
|
|
315
|
+
/** All pages of one request. */
|
|
316
|
+
export async function fetchRequest(client, ep0, params, maxRows) {
|
|
317
|
+
const ep = withRecordParams(ep0, params);
|
|
318
|
+
const pg = ep.paging || { type: "none" };
|
|
319
|
+
const kind = pg.type || "none";
|
|
320
|
+
const limit = Math.min(Number(maxRows || ep.max_rows || 100000), Number(ep.max_rows || 1e9));
|
|
321
|
+
const size = Number(pg.size || 1000);
|
|
322
|
+
const floor = Number(pg.min_size || 1);
|
|
323
|
+
const [p, query, body] = split(ep, params);
|
|
324
|
+
const method = ep.method || "GET";
|
|
325
|
+
let rows = [];
|
|
326
|
+
const meta = { pages: 0, total: null, truncated: false };
|
|
327
|
+
let page = Number(pg.start ?? 1);
|
|
328
|
+
let offset = kind === "offset" ? Number(query[pg.offset_param || ""] || 0) || 0 : 0;
|
|
329
|
+
let cursor = null, nextUrl = null, prev = null;
|
|
330
|
+
for (;;) {
|
|
331
|
+
const q = { ...query };
|
|
332
|
+
const b = body !== null ? { ...body } : null;
|
|
333
|
+
const tgt = method === "POST" && (pg.in ?? "body") === "body" && b ? b : q;
|
|
334
|
+
const put = (k, v) => {
|
|
335
|
+
if (k)
|
|
336
|
+
tgt[k] = v;
|
|
337
|
+
};
|
|
338
|
+
const asked = kind === "offset" ? Math.min(size, limit - rows.length) : Math.min(size, limit);
|
|
339
|
+
if (kind === "offset") {
|
|
340
|
+
put(pg.limit_param, asked);
|
|
341
|
+
put(pg.offset_param, offset);
|
|
342
|
+
}
|
|
343
|
+
else if (kind === "page") {
|
|
344
|
+
put(pg.page_param, page);
|
|
345
|
+
put(pg.size_param, Math.max(Math.min(size, limit), floor));
|
|
346
|
+
}
|
|
347
|
+
else if (kind === "cursor") {
|
|
348
|
+
put(pg.size_param, Math.max(Math.min(size, limit), floor));
|
|
349
|
+
if (cursor)
|
|
350
|
+
put(pg.cursor_param, cursor);
|
|
351
|
+
}
|
|
352
|
+
const resp = nextUrl ? await client.request(method, nextUrl, null, { jsonBody: b ?? undefined })
|
|
353
|
+
: await client.request(method, p, q, { jsonBody: b ?? undefined });
|
|
354
|
+
meta.pages++;
|
|
355
|
+
const got = records(ep, resp);
|
|
356
|
+
const sig = JSON.stringify(got);
|
|
357
|
+
if (got.length && (kind === "offset" || kind === "page") && sig === prev) {
|
|
358
|
+
meta.truncated = meta.repeated_page = true;
|
|
359
|
+
break;
|
|
360
|
+
}
|
|
361
|
+
prev = sig;
|
|
362
|
+
rows = rows.concat(got);
|
|
363
|
+
const head = resp.body.subarray(0, 1).toString("latin1");
|
|
364
|
+
if (ep.total && (head === "{" || head === "[")) {
|
|
365
|
+
const t = Number(dig(resp.json(), ep.total));
|
|
366
|
+
if (Number.isFinite(t))
|
|
367
|
+
meta.total = Math.trunc(t);
|
|
368
|
+
}
|
|
369
|
+
if (rows.length >= limit) {
|
|
370
|
+
meta.truncated = (meta.total || 0) > limit || (kind !== "none" && got.length >= asked) || (kind === "none" && got.length > limit);
|
|
371
|
+
rows = rows.slice(0, limit);
|
|
372
|
+
break;
|
|
373
|
+
}
|
|
374
|
+
if (kind === "none" || !got.length)
|
|
375
|
+
break;
|
|
376
|
+
if (kind === "offset") {
|
|
377
|
+
if (got.length < Math.min(size, limit))
|
|
378
|
+
break;
|
|
379
|
+
offset += got.length;
|
|
380
|
+
}
|
|
381
|
+
else if (kind === "page") {
|
|
382
|
+
if (got.length < Math.min(size, limit) || (meta.total !== null && rows.length >= meta.total))
|
|
383
|
+
break;
|
|
384
|
+
page++;
|
|
385
|
+
}
|
|
386
|
+
else if (kind === "cursor") {
|
|
387
|
+
cursor = dig(resp.json(), pg.next);
|
|
388
|
+
if (!cursor)
|
|
389
|
+
break;
|
|
390
|
+
}
|
|
391
|
+
else if (kind === "link") {
|
|
392
|
+
nextUrl = dig(resp.json(), pg.next);
|
|
393
|
+
if (!nextUrl)
|
|
394
|
+
break;
|
|
395
|
+
}
|
|
396
|
+
if (meta.pages >= Number(pg.max_pages ?? 1000)) {
|
|
397
|
+
meta.truncated = true;
|
|
398
|
+
break;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
return [clean(rows, ep), meta];
|
|
402
|
+
}
|
|
403
|
+
/** Rows repeated across page boundaries (same unique_key) are dropped, with a note that as many may be missing. */
|
|
404
|
+
export function dedupePages(rows, ep, meta, name) {
|
|
405
|
+
const col = ep.unique_key;
|
|
406
|
+
if (!col || meta.pages < 2 || !rows.length || rows.some((r) => !(col in r)))
|
|
407
|
+
return [rows, null];
|
|
408
|
+
const seen = new Set();
|
|
409
|
+
const out = rows.filter((r) => {
|
|
410
|
+
const k = pyStr(r[col]);
|
|
411
|
+
if (seen.has(k))
|
|
412
|
+
return false;
|
|
413
|
+
seen.add(k);
|
|
414
|
+
return true;
|
|
415
|
+
});
|
|
416
|
+
if (out.length === rows.length)
|
|
417
|
+
return [rows, null];
|
|
418
|
+
const gap = rows.length - out.length;
|
|
419
|
+
return [out, `${name}: ${gap.toLocaleString("en-US")} row(s) came back twice across page boundaries (same ${col}), so the ` +
|
|
420
|
+
"source's order shifted while paging and about as many rows may be missing; the repeats were dropped " +
|
|
421
|
+
`(${out.length.toLocaleString("en-US")} rows kept` + (meta.total ? ` of ${meta.total.toLocaleString("en-US")}` : "") +
|
|
422
|
+
"). Sort by a unique field (or narrow the request to one page) and fetch again (force)."];
|
|
423
|
+
}
|
|
424
|
+
/** fetchRequest, waiting out a server that stays busy past the client's own retries. */
|
|
425
|
+
async function fetchPatiently(c, ep, params, maxRows, waits = [20, 60]) {
|
|
426
|
+
for (const wait of [...waits, null]) {
|
|
427
|
+
try {
|
|
428
|
+
return await fetchRequest(c, ep, params, maxRows);
|
|
429
|
+
}
|
|
430
|
+
catch (e) {
|
|
431
|
+
if (wait === null || !(e instanceof DataError) || e.kind !== "http" || ![500, 502, 503, 504].includes(e.status ?? 0) ||
|
|
432
|
+
process.env.DATA_ANALYST_FIXTURES)
|
|
433
|
+
throw e;
|
|
434
|
+
await sleep(wait * 1000);
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
throw new Error("unreachable");
|
|
438
|
+
}
|
|
439
|
+
/** Every request runs; the missing (404) and empty ones are named together at the end. */
|
|
440
|
+
export class Misses {
|
|
441
|
+
missing = [];
|
|
442
|
+
empty = [];
|
|
443
|
+
notFound(r, tables, notes) {
|
|
444
|
+
if (r.allow_empty) {
|
|
445
|
+
notes.push(`${r.name}: not found at the source (404); kept as an empty table (allow_empty).`);
|
|
446
|
+
tables[r.name] = [];
|
|
447
|
+
}
|
|
448
|
+
else
|
|
449
|
+
this.missing.push(r.name);
|
|
450
|
+
}
|
|
451
|
+
checkEmpty(r, rows) {
|
|
452
|
+
if (!rows.length && !r.allow_empty) {
|
|
453
|
+
const key = r._key;
|
|
454
|
+
this.empty.push(r.name + (key ? ` (${key.map(([k, v]) => `${k}=${v}`).join(", ")})` : ""));
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
raiseIfAny(missingHint = "", emptyHint = "") {
|
|
458
|
+
const msgs = [];
|
|
459
|
+
if (this.missing.length) {
|
|
460
|
+
msgs.push(`request(s) ${this.missing.map((m) => `'${m}'`).join(", ")} were not found at the source (404): ` +
|
|
461
|
+
`${missingHint || "check the IDs"}. If the item really does not exist yet (e.g. a volume not yet published), add ` +
|
|
462
|
+
'"allow_empty": true to that request and run again; the answer must say it is missing.');
|
|
463
|
+
}
|
|
464
|
+
if (this.empty.length) {
|
|
465
|
+
msgs.push(`request(s) ${this.empty.map((m) => `'${m}'`).join(", ")} returned no rows: ` +
|
|
466
|
+
`${emptyHint || "check the filter values (see source.md)"}. If none is a correct answer (e.g. no private laws in ` +
|
|
467
|
+
'that Congress), add "allow_empty": true to that request and run again.');
|
|
468
|
+
}
|
|
469
|
+
if (msgs.length)
|
|
470
|
+
throw new DataError(msgs.join(" "), this.missing.length ? "not_found" : "bad_request");
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
/** Every request in the spec: [{name: rows}, notes]. */
|
|
474
|
+
export async function fetchSpec(pack, spec, client) {
|
|
475
|
+
const c = client ?? clientFor(pack);
|
|
476
|
+
const tables = {};
|
|
477
|
+
const notes = [];
|
|
478
|
+
const miss = new Misses();
|
|
479
|
+
for (const r of spec.requests) {
|
|
480
|
+
const [ep, params0, probs] = resolve(pack, r);
|
|
481
|
+
if (probs.length)
|
|
482
|
+
throw new DataError(probs.join("; "), "bad_request");
|
|
483
|
+
let params = params0;
|
|
484
|
+
const extra = Object.keys(r.params || {}).filter((k) => !(k in (ep.params || {})) && !(k in (ep.defaults || {})));
|
|
485
|
+
if (extra.length) {
|
|
486
|
+
notes.push(`${r.name}: parameter(s) ${extra.join(", ")} not declared for endpoint '${r.endpoint}' (endpoints.json); ` +
|
|
487
|
+
"the API may ignore them: check the rows");
|
|
488
|
+
}
|
|
489
|
+
if (pack.adapter?.beforeRequest)
|
|
490
|
+
params = pack.adapter.beforeRequest(r, ep, params) || params;
|
|
491
|
+
let rows, meta;
|
|
492
|
+
try {
|
|
493
|
+
[rows, meta] = await fetchPatiently(c, ep, params, r.max_rows);
|
|
494
|
+
}
|
|
495
|
+
catch (e) {
|
|
496
|
+
if (!(e instanceof DataError))
|
|
497
|
+
throw e;
|
|
498
|
+
if (e.kind === "not_found") {
|
|
499
|
+
miss.notFound(r, tables, notes);
|
|
500
|
+
continue;
|
|
501
|
+
}
|
|
502
|
+
throw new DataError(`request '${r.name}' (${r.endpoint}): ${e.message}`, e.kind, e.status, e.retryAfter);
|
|
503
|
+
}
|
|
504
|
+
let dup;
|
|
505
|
+
[rows, dup] = dedupePages(rows, ep, meta, r.name);
|
|
506
|
+
if (dup)
|
|
507
|
+
notes.push(dup);
|
|
508
|
+
if (pack.adapter?.requestNote) {
|
|
509
|
+
const n = pack.adapter.requestNote(r, ep, params);
|
|
510
|
+
if (n)
|
|
511
|
+
notes.push(n);
|
|
512
|
+
}
|
|
513
|
+
if (pack.adapter?.normalize)
|
|
514
|
+
rows = pack.adapter.normalize(r, ep, rows);
|
|
515
|
+
if (meta.repeated_page) {
|
|
516
|
+
notes.push(`${r.name}${INCOMPLETE} stopped at ${rows.length.toLocaleString("en-US")} rows: page ${meta.pages} repeated ` +
|
|
517
|
+
"the page before it (the API ignored the paging parameter), so later rows may be missing. Narrow the request so it fits one page.");
|
|
518
|
+
}
|
|
519
|
+
else if (meta.truncated) {
|
|
520
|
+
const tn = ep.truncated_note || {};
|
|
521
|
+
const k = tn.when_default;
|
|
522
|
+
const own = !!(tn.text && k && params[k] !== undefined && params[k] !== null && params[k] !== "" &&
|
|
523
|
+
((r.params || {})[k] === undefined || (r.params || {})[k] === null || (r.params || {})[k] === ""));
|
|
524
|
+
notes.push(`${r.name}: stopped at ${rows.length.toLocaleString("en-US")} rows (max_rows); the source has more` +
|
|
525
|
+
(meta.total ? ` (${meta.total.toLocaleString("en-US")} in total)` : "") + ". Narrow the filter or raise max_rows." +
|
|
526
|
+
(own ? " " + tn.text : ""));
|
|
527
|
+
}
|
|
528
|
+
miss.checkEmpty(r, rows);
|
|
529
|
+
tables[r.name] = rows;
|
|
530
|
+
}
|
|
531
|
+
miss.raiseIfAny();
|
|
532
|
+
return [tables, notes];
|
|
533
|
+
}
|
|
534
|
+
// ---------------------------------------------------------------------- manual download
|
|
535
|
+
/** One link per page the request would need (up to 25), with the key as {API_KEY}. */
|
|
536
|
+
export function manualItems(pack, spec) {
|
|
537
|
+
const eps = pack.endpoints || {};
|
|
538
|
+
const base = String(eps.base_url || pack.policy.routes?.api?.base || "").replace(/\/+$/, "");
|
|
539
|
+
const a = pack.policy.auth;
|
|
540
|
+
const needsKey = a.required === "required";
|
|
541
|
+
const items = [];
|
|
542
|
+
for (const r of spec.requests || []) {
|
|
543
|
+
const [ep, params0, probs] = resolve(pack, r);
|
|
544
|
+
if (probs.length)
|
|
545
|
+
continue;
|
|
546
|
+
const params = (pack.adapter?.beforeRequest && pack.adapter.beforeRequest(r, ep, params0)) || params0;
|
|
547
|
+
const [p, query, body] = split(ep, params);
|
|
548
|
+
const pg = ep.paging || { type: "none" };
|
|
549
|
+
const size = Number(pg.manual_size || pg.size || 1000);
|
|
550
|
+
const limit = Number(r.max_rows || size);
|
|
551
|
+
const pages = ["none", "cursor", "link"].includes(pg.type) ? 1 : Math.min(25, Math.max(1, Math.ceil(limit / size)));
|
|
552
|
+
const rec = ep.records || "";
|
|
553
|
+
const ext = { $csv: "csv", $xlsx: "xlsx", $xls: "xls" }[rec] ?? (rec.startsWith("$zip:") ? "zip" : "json");
|
|
554
|
+
for (let i = 0; i < pages; i++) {
|
|
555
|
+
const q = { ...query };
|
|
556
|
+
const b = body !== null ? { ...body } : null;
|
|
557
|
+
const tgt = ep.method === "POST" && (pg.in ?? "body") === "body" && b ? b : q;
|
|
558
|
+
if (pg.type === "offset")
|
|
559
|
+
[tgt[pg.limit_param], tgt[pg.offset_param]] = [size, i * size];
|
|
560
|
+
else if (pg.type === "page")
|
|
561
|
+
[tgt[pg.page_param], tgt[pg.size_param]] = [Number(pg.start ?? 1) + i, size];
|
|
562
|
+
let url = /^https?:\/\//.test(p) ? p : `${base}/${p.replace(/^\/+/, "")}`;
|
|
563
|
+
if (Object.keys(q).length)
|
|
564
|
+
url += (url.includes("?") ? "&" : "?") + pyUrlencode(q);
|
|
565
|
+
if (needsKey && a.placement === "query")
|
|
566
|
+
url += (url.includes("?") ? "&" : "?") + `${a.param}={API_KEY}`;
|
|
567
|
+
else if (a.required === "optional" && a.demo_key && a.placement === "query")
|
|
568
|
+
url += (url.includes("?") ? "&" : "?") + `${a.param}=${a.demo_key}`;
|
|
569
|
+
const filename = pages === 1 ? `${r.name}.${ext}` : `${r.name}_${i + 1}.${ext}`;
|
|
570
|
+
let note = pages === 1 ? "" : `page ${i + 1} of up to ${pages}; only needed if the previous page held ${size.toLocaleString("en-US")} rows`;
|
|
571
|
+
if (pages === 1 && ["offset", "page"].includes(pg.type)) {
|
|
572
|
+
note = `holds up to ${size.toLocaleString("en-US")} rows; if it holds exactly that many, more rows exist: narrow the filter`;
|
|
573
|
+
}
|
|
574
|
+
if (["cursor", "link"].includes(pg.type))
|
|
575
|
+
note = "first page only: this API pages with a cursor, so the script route is needed for more rows";
|
|
576
|
+
items.push({ name: `${r.name}#${i + 1}`, request: r.name, url, filename, also_ext: ext === "json" ? [".csv"] : [],
|
|
577
|
+
needs_key: needsKey, method: ep.method || "GET", body: b, note, optional: i > 0 });
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
return items;
|
|
581
|
+
}
|
|
582
|
+
/** Rows from uploaded files for one request (its pages in natural order). */
|
|
583
|
+
export function parseUpload(pack, req, files) {
|
|
584
|
+
const [ep0, params] = resolve(pack, req);
|
|
585
|
+
const ep = withRecordParams(ep0, params);
|
|
586
|
+
const natural = (s) => s.split(/(\d+)/).map((x) => (/^\d+$/.test(x) ? x.padStart(20, "0") : x)).join("");
|
|
587
|
+
let rows = [];
|
|
588
|
+
for (const f of [...files].sort((a, b) => (natural(a) < natural(b) ? -1 : 1))) {
|
|
589
|
+
const data = fs.readFileSync(f);
|
|
590
|
+
const r = { body: data, text: () => decodeText(data), json: () => parsePyJson(data.toString("utf8").replace(/^/, "")) };
|
|
591
|
+
rows = rows.concat(f.toLowerCase().endsWith(".csv") && ep.records !== "$csv" ? readCsv(r.text()) : records(ep, r));
|
|
592
|
+
}
|
|
593
|
+
let out = clean(rows, ep);
|
|
594
|
+
if (pack.adapter?.normalize)
|
|
595
|
+
out = pack.adapter.normalize(req, ep, out);
|
|
596
|
+
return out;
|
|
597
|
+
}
|
|
598
|
+
// ---------------------------------------------------------------------- describing the API to the model
|
|
599
|
+
/** One line per endpoint: the request index of source_brief. */
|
|
600
|
+
export function endpointIndex(pack) {
|
|
601
|
+
return Object.entries(pack.endpoints?.endpoints || {}).sort(([a], [b]) => (a < b ? -1 : 1))
|
|
602
|
+
.map(([name, e]) => ({ name, summary: String(e.description || "").split(/(?<=\.)\s/)[0].slice(0, 200) }));
|
|
603
|
+
}
|
|
604
|
+
/** Full argument schema of named endpoints (what request_schema returns). */
|
|
605
|
+
export function endpointSchema(pack, name) {
|
|
606
|
+
const e = pack.endpoints?.endpoints?.[name];
|
|
607
|
+
if (!e)
|
|
608
|
+
return null;
|
|
609
|
+
const pg = e.paging || { type: "none" };
|
|
610
|
+
return {
|
|
611
|
+
name, method: e.method || "GET", description: e.description || "",
|
|
612
|
+
params: Object.fromEntries(Object.entries(e.params || {}).map(([k, d]) => [k, {
|
|
613
|
+
desc: d.desc || "", ...(d.required ? { required: true } : {}), ...(d.enum ? { enum: d.enum } : {}),
|
|
614
|
+
...(d.catalog ? { catalog: "an alias from the catalog (search with request_schema search) or a dataset id" } : {}),
|
|
615
|
+
...(d.in === "spec" ? { note: "read by the pack, never sent" } : {}),
|
|
616
|
+
}])),
|
|
617
|
+
defaults: e.defaults || {}, paging: pg.type, max_rows: e.max_rows ?? 100000,
|
|
618
|
+
returns: e.records?.startsWith?.("$") ? e.records : "json records",
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
export function searchCatalog(pack, query, limit = 20) {
|
|
622
|
+
const words = (query.toLowerCase().match(/[a-z0-9]+/g) || []).filter((w) => w.length > 1);
|
|
623
|
+
const count = (text, w) => text.split(w).length - 1;
|
|
624
|
+
const hits = [];
|
|
625
|
+
for (const [alias, c] of Object.entries(pack.endpoints?.catalog || {})) {
|
|
626
|
+
const text = `${alias} ${c.id ?? ""} ${c.name ?? ""} ${c.description ?? ""}`.toLowerCase();
|
|
627
|
+
const s = words.reduce((n, w) => n + count(text, w), 0);
|
|
628
|
+
if (s)
|
|
629
|
+
hits.push([s, { alias, ...c }]);
|
|
630
|
+
}
|
|
631
|
+
for (const [n, e] of Object.entries(pack.endpoints?.endpoints || {})) {
|
|
632
|
+
const text = `${n} ${e.description ?? ""}`.toLowerCase();
|
|
633
|
+
const s = words.reduce((k, w) => k + count(text, w), 0);
|
|
634
|
+
if (s)
|
|
635
|
+
hits.push([s, { endpoint: n, description: e.description || "" }]);
|
|
636
|
+
}
|
|
637
|
+
return hits.sort((a, b) => b[0] - a[0]).slice(0, limit).map(([, h]) => h);
|
|
638
|
+
}
|
|
639
|
+
//# sourceMappingURL=generic-rest.js.map
|