data-analyst-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -2
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +716 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +24 -0
- package/dist/report/scan_secrets.js +265 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +247 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +54 -4
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +170 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# U.S. government data → answers and data (method)
|
|
2
|
+
|
|
3
|
+
You are a data analyst. The user brings a question; you deliver the **data** (CSV, data dictionary,
|
|
4
|
+
provenance, in the session workspace) and a **short answer** with the key numbers, each one traced to a
|
|
5
|
+
fetched file. **Accuracy comes before everything else.**
|
|
6
|
+
|
|
7
|
+
## Tools
|
|
8
|
+
| Tool | Job |
|
|
9
|
+
|---|---|
|
|
10
|
+
| `find_sources(question)` | rank the source packs for a question; without a question, list them all |
|
|
11
|
+
| `source_brief(id)` | the source's playbook (coverage, units, traps, citation), **questions_to_ask**, key status, rules, request index; `doc` opens a long reference |
|
|
12
|
+
| `request_schema(id, names \| search)` | full parameters of the requests you will use; `search` finds datasets or series in the catalog |
|
|
13
|
+
| `record_decisions(...)` | the method record in `decisions.json`: how each critical question was settled, the literature searched, references, assumptions, data needs |
|
|
14
|
+
| `fetch(id, spec)` | download into the workspace; returns rows, columns, a preview, the files, the citation and notes |
|
|
15
|
+
| `manual_bundle` / `ingest` | when the server cannot get the data: a one-command download for the user's own computer, then package what they downloaded |
|
|
16
|
+
| `check_key(id)` | whether a key is set up (never shows it) |
|
|
17
|
+
| `compute(inputs, steps, save_as)` | every table the answer draws numbers from, as `exhibits/<name>.csv`, with its steps recorded so `rerun` can replay it |
|
|
18
|
+
| `facts(add, derived)` | register every number the answer cites; returns the exact text to quote |
|
|
19
|
+
| `write_table(path, ...)` | `exhibits/benchmark.csv`, `exhibits/sensitivity.csv`, `published/<name>.csv` (figures read from an agency page) |
|
|
20
|
+
| `cite(url, id, text?)` | snapshot a page a reference cites (pass the text your own fetch tool returned) |
|
|
21
|
+
| `rerun` | replay every exhibit and fact from the fetched data |
|
|
22
|
+
| `audit_answer(answer, rerun)` | the release gate for a quick answer: send the answer only when it prints AUDIT PASSED |
|
|
23
|
+
| `build_report(report)`, `style_check`, `audit_report` | a report or paper, only when asked |
|
|
24
|
+
| `handoff` | the per-source data hand-off (data-only replies) |
|
|
25
|
+
| `read_file(path)` | a small text file from the workspace |
|
|
26
|
+
|
|
27
|
+
`fetch` refuses (`gate_blocked`) until, for that source, `source_brief` was read, `request_schema` returned
|
|
28
|
+
every request it uses, and every one of its questions_to_ask is recorded.
|
|
29
|
+
|
|
30
|
+
## 1. Frame the question, and ask only what is critical
|
|
31
|
+
Write down the **measures** (each number the answer needs), the **universe**, the **period** (fiscal or
|
|
32
|
+
calendar; the latest complete period, found from the data, not from memory), the **geography**, the
|
|
33
|
+
**dollar basis** (nominal, real with which index, per capita, share of GDP) and the **comparisons**.
|
|
34
|
+
|
|
35
|
+
Collect the open choices that would change a headline number: each chosen source's `questions_to_ask`, plus
|
|
36
|
+
nominal vs real (and the deflator), fiscal vs calendar year, geography level, data vintage (latest vs as
|
|
37
|
+
first published), estimate vs margin of error (surveys), answer vs report.
|
|
38
|
+
- Ask in **one** round, at most 4 questions, with your question tool when you have one; each offers exactly
|
|
39
|
+
one "Don't know — decide for me" option (never a separate "Skip"), your recommended option first.
|
|
40
|
+
- A skipped or unanswered question is a "Don't know": decide it by precedent (how the agency itself reports
|
|
41
|
+
it) and record the `resolution`. Exception: a **blocking** question (what the user means: which program,
|
|
42
|
+
place, population or product) has no defensible default. After a skip, ask it once more
|
|
43
|
+
(`followed_up: true`); if still unanswered, use the most literal reading (`rationale`) and say so in the
|
|
44
|
+
answer's opening paragraph.
|
|
45
|
+
- Skip what the request already settles: `how: "not_asked"`, `reason_code: "settled_by_request"`,
|
|
46
|
+
`request`: the user's own words. A one-number lookup with an obvious convention needs no question.
|
|
47
|
+
- `reason_code: "no_user"` only when nobody can answer (a scheduled or headless run). A chat is interactive:
|
|
48
|
+
ask. Each question decided without the user gets a `disclosures` line the answer carries word for word.
|
|
49
|
+
- One question can settle several ids (`covers`). Record every question with `record_decisions`.
|
|
50
|
+
|
|
51
|
+
## 2. Choose the sources
|
|
52
|
+
Split the question into measures and give each its source (`find_sources`; `guide("routing")` lists the
|
|
53
|
+
usual pairings: debt/GDP → treasury + bea; real wages → bls + CPI; spending per person → usaspending +
|
|
54
|
+
census). Each `source_brief` also lists `paired_with`: the questions it usually answers together with
|
|
55
|
+
other sources, and the cross-checks to fetch. Prefer the agency that produces a statistic over a re-publisher (BLS over FRED for CPI detail; FRED
|
|
56
|
+
for long histories and vintages). Cross-check every headline number with a second source or the agency's
|
|
57
|
+
published table when one exists. Read each chosen source's brief, and a `doc` only when the playbook points
|
|
58
|
+
you to it for the task at hand. A source marked `ready: false` is not available on this server yet.
|
|
59
|
+
|
|
60
|
+
## 3. Get the data
|
|
61
|
+
Write the spec as the playbook and `request_schema` show (`{"description", "requests": [{"name",
|
|
62
|
+
"endpoint", "params", "max_rows"}]}`), then `fetch`. What a source serves comes only through `fetch`: find
|
|
63
|
+
its datasets, codes and columns with `request_schema` (`search`) and the playbook, or a small probe request
|
|
64
|
+
(`max_rows`), never by querying the API yourself or from memory.
|
|
65
|
+
- `status: "partial"`: a row cap was hit. Narrow the filter or raise `max_rows` and fetch again with
|
|
66
|
+
`force: true`; if the cap is the API's own limit, say the result is partial. Never present it as complete.
|
|
67
|
+
- A second fetch that depends on the first: `"for_each": {"csv": "data/<id>/<table>.csv", "column": "...",
|
|
68
|
+
"param": "..."}` makes one request per row value; the rows come back as one table.
|
|
69
|
+
- Errors carry `code` and `fix`; follow the fix and never repeat a call unchanged.
|
|
70
|
+
`key_missing` / `key_rejected`: give the user the command or page in `fix` (they store the key
|
|
71
|
+
themselves; never ask for a key in chat), and offer the keyless route it names. `agency_unavailable`: say
|
|
72
|
+
the data could not be fetched now; never substitute numbers from memory. `agency_rejected`: change the
|
|
73
|
+
value as the message says. `not_found`: check IDs and codes.
|
|
74
|
+
|
|
75
|
+
## 2b. Search the literature and methods
|
|
76
|
+
Before choosing definitions and parameters, search for the agency's methodology (handbook, technical notes) and
|
|
77
|
+
for agency or peer-reviewed research behind any parameter the answer needs, with your web search or fetch tool
|
|
78
|
+
when you have one. Snapshot each page you cite with `cite` (pass the text your fetch tool returned), and record
|
|
79
|
+
the searches (`literature_search`: query, date, what was found) and each opened source (`references`: citation,
|
|
80
|
+
url, accessed, `opened: true`, used_for, `quote`: the page's own sentence) with `record_decisions`. With no
|
|
81
|
+
search tool, record `{"unavailable": true, "reason_code": "no_search_tool"}` plus a `disclosures.literature`
|
|
82
|
+
line the answer carries. Never cite what you have not opened.
|
|
83
|
+
|
|
84
|
+
## 4. Analyze
|
|
85
|
+
Read the fetched files (the data dictionary and README say what each column is). Make every table the answer
|
|
86
|
+
draws numbers from with `compute` (filters, derived columns, groups, joins, fiscal years, deflation,
|
|
87
|
+
`align_check` before joining two sources): it saves `exhibits/<name>.csv` with its steps, so `rerun` reproduces
|
|
88
|
+
it. Register every number you will cite with `facts` (its value is read from the exhibit cell; quote its `text`
|
|
89
|
+
exactly). Record each key definition and parameter in `assumptions` (basis empirical > literature > precedent >
|
|
90
|
+
user > judgment, with its evidence; a number borrowed from the literature or an agency states
|
|
91
|
+
`why_not_empirical`), each judgment call's alternative in `exhibits/sensitivity.csv`, each headline against the
|
|
92
|
+
agency's published figure in `exhibits/benchmark.csv` (both with `write_table`), and the data that would
|
|
93
|
+
improve the answer in `data_needs`. Accuracy checklist:
|
|
94
|
+
- **Units and scaling**: name them in every table; convert once (thousands, millions, chained dollars).
|
|
95
|
+
- **Stocks vs flows**: end-of-period or average for levels and rates; sums for flows; never sum
|
|
96
|
+
year-to-date values or stock levels.
|
|
97
|
+
- **Periods**: fiscal years run October to September and are named for the year they end; label partial
|
|
98
|
+
periods; compare the same months; never present an incomplete period as complete.
|
|
99
|
+
- **Revisions and vintages**: say which vintage you used and call preliminary values preliminary.
|
|
100
|
+
- **Surveys**: weighted estimates with margins of error; small cells are unreliable; ACS 1-year and 5-year
|
|
101
|
+
estimates are not interchangeable.
|
|
102
|
+
- **Benchmark**: compare each headline with the agency's published figure and say whether it matches.
|
|
103
|
+
- **Sensitivity**: when a judgment call could change the conclusion, compute the alternative too and say
|
|
104
|
+
whether it does.
|
|
105
|
+
- Each source's own traps are in its playbook. Read them.
|
|
106
|
+
|
|
107
|
+
## 5. Combining sources
|
|
108
|
+
Before any join or ratio across sources: the same period basis (fiscal vs calendar) and frequency, complete
|
|
109
|
+
periods only; the same geography codes (FIPS vs USPS vs names); the same units and dollar basis (one
|
|
110
|
+
deflator, named, with its base period); compatible vintages and coverage (state differences). Cite every source.
|
|
111
|
+
|
|
112
|
+
## 6. The answer
|
|
113
|
+
The answer in 2–4 sentences with the key numbers, the measure named exactly, the period and the data date; a
|
|
114
|
+
small table when there is a series or breakdown (rounded, units in the header); how each "Don't know" was
|
|
115
|
+
decided and any sensitivity that matters; every `disclosures` line, word for word; the benchmark check; the
|
|
116
|
+
data that would sharpen the answer (at most three, those the user can supply first); each source's citation
|
|
117
|
+
(`citation.text`, `page_url`) and retrieval date; every required notice (`citation.notices`) word for word;
|
|
118
|
+
the workspace files. Run `audit_answer` with the reply exactly as you will send it and `rerun: true`, fix every
|
|
119
|
+
FAIL, and send it only when it prints AUDIT PASSED. Data only: send the `handoff` text, each source's caveats,
|
|
120
|
+
and what could not be fetched with its route.
|
|
121
|
+
|
|
122
|
+
## 7. Report or paper (only when asked)
|
|
123
|
+
Read `guide("report_patterns")` and `guide("style")`. Write the report as `build_report`'s report object, with
|
|
124
|
+
every number as `{{fact_name}}` and a `sources` list of the pack ids used; run `style_check`, `build_report`,
|
|
125
|
+
then `audit_report` (`rerun: true`), and rebuild until it prints AUDIT PASSED. The builder applies each source's
|
|
126
|
+
notices, redistribution rules (microdata never leaves the workspace) and vintage blocks.
|
|
127
|
+
|
|
128
|
+
## Integrity rules
|
|
129
|
+
- Never invent, estimate from memory, simulate or "illustrate" a number. If data cannot be fetched, say so
|
|
130
|
+
and give the user the route to get it.
|
|
131
|
+
- Every number in an answer comes from a fetched file or a computation on one.
|
|
132
|
+
- Keys are never printed, asked for in chat, or looked for in environment variables or settings files.
|
|
133
|
+
- Respect each source's terms: its redistribution rule, notices and citation (in its README).
|
|
134
|
+
- Say how current the data are, and cite every source with its retrieval date.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
This content was generated with the help of AI and has not been reviewed by a human expert. AI can make mistakes, so please verify important information before relying on it.
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
<#
|
|
2
|
+
.SYNOPSIS
|
|
3
|
+
Download the files for this analysis into the folder that holds this script. (Windows PowerShell
|
|
4
|
+
counterpart of @STEM@.sh.)
|
|
5
|
+
|
|
6
|
+
.DESCRIPTION
|
|
7
|
+
- The requests are built into this script (written by data-analyst-mcp; do not edit them).
|
|
8
|
+
- Files that need an API key: the script uses the key's environment variable when it is set, otherwise
|
|
9
|
+
asks for the key with hidden input (press Enter to skip those files). The key is never shown or saved.
|
|
10
|
+
- Needs nothing beyond Windows PowerShell 5.1 (built into Windows 10 and 11) or PowerShell 7.
|
|
11
|
+
- At the end it prints the files to upload to the chat.
|
|
12
|
+
|
|
13
|
+
.PARAMETER NoPause
|
|
14
|
+
Do not wait for Enter before closing the window.
|
|
15
|
+
|
|
16
|
+
.EXAMPLE
|
|
17
|
+
powershell -NoProfile -ExecutionPolicy Bypass -File .\@STEM@.ps1
|
|
18
|
+
#>
|
|
19
|
+
[CmdletBinding()]
|
|
20
|
+
param([switch]$NoPause)
|
|
21
|
+
|
|
22
|
+
$ErrorActionPreference = 'Stop'
|
|
23
|
+
$ProgressPreference = 'SilentlyContinue' # Invoke-WebRequest is very slow with the progress bar
|
|
24
|
+
try {
|
|
25
|
+
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
|
26
|
+
} catch { }
|
|
27
|
+
|
|
28
|
+
# Values the requests need. Columns, tab-separated: variable, kind (secret = an API key, asked with
|
|
29
|
+
# hidden input; plain = asked with visible input), where to get it.
|
|
30
|
+
$Values = @'
|
|
31
|
+
@VALUES@
|
|
32
|
+
'@
|
|
33
|
+
|
|
34
|
+
# The requests. Columns, tab-separated ("-" = empty): file, method, optional (1 = may fail), key
|
|
35
|
+
# variable, key placement (query, header, body or none), header or body field name, header prefix,
|
|
36
|
+
# other variables (comma-separated), URL, JSON body.
|
|
37
|
+
$Items = @'
|
|
38
|
+
@ITEMS@
|
|
39
|
+
'@
|
|
40
|
+
|
|
41
|
+
$Here = $PSScriptRoot
|
|
42
|
+
if (-not $Here) { $Here = (Get-Location).Path }
|
|
43
|
+
|
|
44
|
+
# Keep the window open when the script was started with "Run with PowerShell".
|
|
45
|
+
function Finish([int]$Code) {
|
|
46
|
+
$script:Vals = $null
|
|
47
|
+
if (-not $NoPause -and -not [Console]::IsInputRedirected) {
|
|
48
|
+
Write-Host ''
|
|
49
|
+
[void](Read-Host -Prompt 'Press Enter to close')
|
|
50
|
+
}
|
|
51
|
+
exit $Code
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function Lines([string]$Text) { $Text -split "`r?`n" | Where-Object { $_.Trim() } }
|
|
55
|
+
|
|
56
|
+
# ---------------------------------------------------------------------------- keys and other values
|
|
57
|
+
$Vals = @{}
|
|
58
|
+
foreach ($line in (Lines $Values)) {
|
|
59
|
+
$var, $kind, $where = $line -split "`t", 3
|
|
60
|
+
$v = [Environment]::GetEnvironmentVariable($var)
|
|
61
|
+
if ($v) {
|
|
62
|
+
Write-Host "Using `$env:$var."
|
|
63
|
+
} elseif ([Console]::IsInputRedirected) {
|
|
64
|
+
$v = [Console]::In.ReadLine()
|
|
65
|
+
} elseif ($kind -eq 'secret') {
|
|
66
|
+
Write-Host "Some files need your $var ($where). Press Enter to skip them."
|
|
67
|
+
$secure = Read-Host -Prompt "$var (input is hidden)" -AsSecureString
|
|
68
|
+
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
|
69
|
+
try { $v = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
|
|
70
|
+
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
|
|
71
|
+
Remove-Variable secure
|
|
72
|
+
} else {
|
|
73
|
+
$v = Read-Host -Prompt "$var ($where)"
|
|
74
|
+
}
|
|
75
|
+
if ($null -eq $v) { $v = '' }
|
|
76
|
+
if ($kind -eq 'secret') { $v = $v -replace '\s', '' } else { $v = $v.Trim() }
|
|
77
|
+
$Vals[$var] = $v
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function Get-Code($Err) {
|
|
81
|
+
if ($Err.Exception.Response) { return [int]$Err.Exception.Response.StatusCode }
|
|
82
|
+
return 0
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
# ---------------------------------------------------------------------------- download
|
|
86
|
+
$saved = @(); $failed = 0; $skipped = 0
|
|
87
|
+
foreach ($line in (Lines $Items)) {
|
|
88
|
+
$f = @($line -split "`t", 10 | ForEach-Object { if ($_ -eq '-') { '' } else { $_ } })
|
|
89
|
+
$file, $method, $optional, $kvar, $place, $kname, $kprefix, $extras, $url, $body = $f
|
|
90
|
+
$key = ''
|
|
91
|
+
if ($kvar) {
|
|
92
|
+
$key = $Vals[$kvar]
|
|
93
|
+
if (-not $key) { Write-Host "skipped (needs $kvar): $file"; $skipped++; continue }
|
|
94
|
+
}
|
|
95
|
+
$miss = ''
|
|
96
|
+
if ($extras) {
|
|
97
|
+
foreach ($x in $extras.Split(',')) {
|
|
98
|
+
if (-not $Vals[$x]) { $miss = $x; break }
|
|
99
|
+
$url = $url.Replace("{$x}", $Vals[$x])
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if ($miss) { Write-Host "skipped (needs $miss): $file"; $skipped++; continue }
|
|
103
|
+
if ($place -eq 'query') { $url = $url.Replace('{API_KEY}', $key) }
|
|
104
|
+
|
|
105
|
+
$params = @{ Uri = $url; Method = $method; UseBasicParsing = $true; UserAgent = 'data-analyst/1.0'
|
|
106
|
+
OutFile = (Join-Path $Here "$file.part") }
|
|
107
|
+
if ($place -eq 'header') {
|
|
108
|
+
$params.Headers = @{ $kname = "$kprefix$key" }
|
|
109
|
+
$params.MaximumRedirection = 0 # a redirect would carry the key header to another host
|
|
110
|
+
}
|
|
111
|
+
if ($body) {
|
|
112
|
+
if ($place -eq 'body') { $body = $body.Replace('{API_KEY}', ($key -replace '\\', '\\' -replace '"', '\"')) }
|
|
113
|
+
$params.Body = [Text.Encoding]::UTF8.GetBytes($body)
|
|
114
|
+
$params.ContentType = 'application/json'
|
|
115
|
+
}
|
|
116
|
+
$code = 0; $msg = ''; $ok = $false
|
|
117
|
+
for ($try = 1; $try -le 4; $try++) {
|
|
118
|
+
try {
|
|
119
|
+
Invoke-WebRequest @params
|
|
120
|
+
$ok = $true
|
|
121
|
+
break
|
|
122
|
+
} catch {
|
|
123
|
+
$code = Get-Code $_
|
|
124
|
+
$msg = [string]$_.ErrorDetails.Message
|
|
125
|
+
if (-not $msg) { $msg = $_.Exception.Message }
|
|
126
|
+
if ($try -lt 4 -and ($code -eq 0 -or $code -eq 429 -or $code -ge 500)) { Start-Sleep -Seconds (10 * $try); continue }
|
|
127
|
+
break
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
$part = Join-Path $Here "$file.part"
|
|
131
|
+
$dest = Join-Path $Here $file
|
|
132
|
+
if ($ok) {
|
|
133
|
+
Move-Item -LiteralPath $part -Destination $dest -Force
|
|
134
|
+
$saved += $dest
|
|
135
|
+
Write-Host "saved $file"
|
|
136
|
+
} else {
|
|
137
|
+
if (Test-Path -LiteralPath $part) { Remove-Item -LiteralPath $part -Force }
|
|
138
|
+
if ($key) { $msg = $msg.Replace($key, '[KEY]') }
|
|
139
|
+
if ($msg.Length -gt 300) { $msg = $msg.Substring(0, 300) }
|
|
140
|
+
if ($optional -eq '1') {
|
|
141
|
+
Write-Host "not available (optional, HTTP $code): $file"
|
|
142
|
+
} elseif ($code -eq 0) {
|
|
143
|
+
Write-Host "FAILED $($file): could not reach the server ($msg) Check the connection, VPN and proxy." -ForegroundColor Red
|
|
144
|
+
$failed++
|
|
145
|
+
} else {
|
|
146
|
+
Write-Host "FAILED $($file): HTTP $code $msg" -ForegroundColor Red
|
|
147
|
+
$failed++
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
Write-Host ''
|
|
153
|
+
if ($saved.Count -gt 0) {
|
|
154
|
+
Write-Host 'Done. Upload these files to the chat (do not rename or edit them):' -ForegroundColor Green
|
|
155
|
+
foreach ($p in $saved) { Write-Host " $p" }
|
|
156
|
+
}
|
|
157
|
+
if ($skipped -gt 0) { Write-Host "$skipped file(s) skipped: run again and enter the value asked for." }
|
|
158
|
+
if ($failed -gt 0) {
|
|
159
|
+
Write-Host "$failed file(s) failed: send the messages above to the analyst." -ForegroundColor Red
|
|
160
|
+
Finish 1
|
|
161
|
+
}
|
|
162
|
+
Finish 0
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Download the files for this analysis into the folder that holds this script.
|
|
3
|
+
# (macOS, Linux, Git Bash, WSL; on Windows PowerShell use @STEM@.ps1.)
|
|
4
|
+
#
|
|
5
|
+
# Usage: bash @STEM@.sh
|
|
6
|
+
#
|
|
7
|
+
# - The requests are built into this script (written by data-analyst-mcp; do not edit them).
|
|
8
|
+
# - Files that need an API key: the script uses the key's environment variable when it is set,
|
|
9
|
+
# otherwise asks for the key with hidden input (press Enter to skip those files). The key is never
|
|
10
|
+
# shown, saved, or put on a command line.
|
|
11
|
+
# - Needs only curl, which macOS and most Linux systems have.
|
|
12
|
+
# - At the end it prints the files to upload to the chat.
|
|
13
|
+
|
|
14
|
+
die() { echo "ERROR: $*" >&2; exit 1; }
|
|
15
|
+
command -v curl >/dev/null 2>&1 || die "curl is not installed. Install it (for example: sudo apt install curl) and run this script again."
|
|
16
|
+
|
|
17
|
+
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
18
|
+
TMP="$(mktemp -d "${TMPDIR:-/tmp}/data_analyst.XXXXXX")" || die "Cannot create a temporary folder."
|
|
19
|
+
trap 'rm -rf "$TMP"' EXIT
|
|
20
|
+
umask 077
|
|
21
|
+
|
|
22
|
+
# Values the requests need. Columns, tab-separated: variable, kind (secret = an API key, asked with
|
|
23
|
+
# hidden input; plain = asked with visible input), where to get it.
|
|
24
|
+
values() {
|
|
25
|
+
cat <<'DATA_ANALYST_VALUES'
|
|
26
|
+
@VALUES@
|
|
27
|
+
DATA_ANALYST_VALUES
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
# The requests. Columns, tab-separated ("-" = empty): file, method, optional (1 = may fail), key
|
|
31
|
+
# variable, key placement (query, header, body or none), header or body field name, header prefix,
|
|
32
|
+
# other variables (comma-separated), URL, JSON body.
|
|
33
|
+
items() {
|
|
34
|
+
cat <<'DATA_ANALYST_ITEMS'
|
|
35
|
+
@ITEMS@
|
|
36
|
+
DATA_ANALYST_ITEMS
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
# escape for a double-quoted curl config value or a JSON string
|
|
40
|
+
esc() { local s="$1"; s="${s//\\/\\\\}"; s="${s//\"/\\\"}"; printf '%s' "$s"; }
|
|
41
|
+
|
|
42
|
+
# subst TEXT PLACEHOLDER VALUE: every PLACEHOLDER replaced literally (the same in bash 3.2 and 5.x)
|
|
43
|
+
subst() {
|
|
44
|
+
local s="$1" out=""
|
|
45
|
+
[ -n "$2" ] || { printf '%s' "$s"; return; }
|
|
46
|
+
while [[ "$s" == *"$2"* ]]; do out+="${s%%"$2"*}$3"; s="${s#*"$2"}"; done
|
|
47
|
+
printf '%s' "$out$s"
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
# --------------------------------------------------------------------------- keys and other values
|
|
51
|
+
while IFS=$'\t' read -r var kind where <&3; do
|
|
52
|
+
[ -n "$var" ] || continue
|
|
53
|
+
val="${!var:-}"
|
|
54
|
+
if [ -n "$val" ]; then
|
|
55
|
+
echo "Using \$$var."
|
|
56
|
+
else
|
|
57
|
+
if [ "$kind" = secret ]; then
|
|
58
|
+
echo "Some files need your $var ($where). Press Enter to skip them."
|
|
59
|
+
if [ -t 0 ]; then printf '%s (input is hidden): ' "$var"; IFS= read -rs val; echo
|
|
60
|
+
else IFS= read -r val || true; fi
|
|
61
|
+
else
|
|
62
|
+
if [ -t 0 ]; then printf '%s (%s): ' "$var" "$where"; fi
|
|
63
|
+
IFS= read -r val || true
|
|
64
|
+
fi
|
|
65
|
+
fi
|
|
66
|
+
if [ "$kind" = secret ]; then val="$(printf '%s' "$val" | tr -d '[:space:]')"
|
|
67
|
+
else val="$(printf '%s' "$val" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')"; fi
|
|
68
|
+
printf -v "V_$var" '%s' "$val"
|
|
69
|
+
done 3< <(values)
|
|
70
|
+
|
|
71
|
+
# --------------------------------------------------------------------------- download
|
|
72
|
+
SAVED=(); FAILED=0; SKIPPED=0
|
|
73
|
+
while IFS=$'\t' read -r file method optional kvar place kname kprefix extras url body <&3; do
|
|
74
|
+
[ -n "$file" ] && [ "$file" != "-" ] || continue
|
|
75
|
+
key=""
|
|
76
|
+
if [ "$kvar" != "-" ]; then
|
|
77
|
+
v="V_$kvar"; key="${!v:-}"
|
|
78
|
+
if [ -z "$key" ]; then echo "skipped (needs $kvar): $file"; SKIPPED=$((SKIPPED + 1)); continue; fi
|
|
79
|
+
fi
|
|
80
|
+
miss=""
|
|
81
|
+
if [ "$extras" != "-" ]; then
|
|
82
|
+
for x in ${extras//,/ }; do
|
|
83
|
+
v="V_$x"; val="${!v:-}"
|
|
84
|
+
[ -n "$val" ] || { miss="$x"; break; }
|
|
85
|
+
url="$(subst "$url" "{$x}" "$val")"
|
|
86
|
+
done
|
|
87
|
+
fi
|
|
88
|
+
if [ -n "$miss" ]; then echo "skipped (needs $miss): $file"; SKIPPED=$((SKIPPED + 1)); continue; fi
|
|
89
|
+
[ "$place" = query ] && url="$(subst "$url" "{API_KEY}" "$key")"
|
|
90
|
+
|
|
91
|
+
# The URL and any key header reach curl as a config file on stdin (-K -), never as arguments.
|
|
92
|
+
cfg() {
|
|
93
|
+
printf 'url = "%s"\n' "$(esc "$url")"
|
|
94
|
+
[ "$place" = header ] && printf 'header = "%s: %s%s"\n' "$(esc "$kname")" "$(esc "$kprefix")" "$(esc "$key")"
|
|
95
|
+
return 0
|
|
96
|
+
}
|
|
97
|
+
args=(-sS -K - -A "data-analyst/1.0" -o "$HERE/$file.part" -w '%{http_code}')
|
|
98
|
+
# curl -L forwards custom headers (the key) to any host a redirect names: follow redirects only without one
|
|
99
|
+
[ "$place" = header ] || args+=(-L)
|
|
100
|
+
if [ "$body" != "-" ]; then
|
|
101
|
+
[ "$place" = body ] && body="$(subst "$body" "{API_KEY}" "$(esc "$key")")"
|
|
102
|
+
printf '%s' "$body" > "$TMP/body"
|
|
103
|
+
args+=(-H "Content-Type: application/json" --data-binary "@$TMP/body")
|
|
104
|
+
fi
|
|
105
|
+
[ "$method" = GET ] || args+=(-X "$method")
|
|
106
|
+
|
|
107
|
+
tries=0
|
|
108
|
+
while :; do
|
|
109
|
+
tries=$((tries + 1))
|
|
110
|
+
code="$(cfg | curl "${args[@]}")"
|
|
111
|
+
rc=$?
|
|
112
|
+
[ $rc -eq 0 ] && case "$code" in 429|5??) ;; *) break ;; esac
|
|
113
|
+
[ $tries -ge 4 ] && break
|
|
114
|
+
sleep $((tries * 10))
|
|
115
|
+
done
|
|
116
|
+
rm -f "$TMP/body"
|
|
117
|
+
if [ $rc -eq 0 ] && case "$code" in 2??) true ;; *) false ;; esac; then
|
|
118
|
+
mv -f "$HERE/$file.part" "$HERE/$file"
|
|
119
|
+
SAVED+=("$HERE/$file")
|
|
120
|
+
echo "saved $file"
|
|
121
|
+
else
|
|
122
|
+
msg=""
|
|
123
|
+
[ -f "$HERE/$file.part" ] && msg="$(head -c 300 "$HERE/$file.part" | tr -d '\r\n')"
|
|
124
|
+
[ -n "$key" ] && msg="$(subst "$msg" "$key" "[KEY]")"
|
|
125
|
+
rm -f "$HERE/$file.part"
|
|
126
|
+
if [ "$optional" = 1 ]; then
|
|
127
|
+
echo "not available (optional, HTTP ${code:-none}): $file"
|
|
128
|
+
else
|
|
129
|
+
case "$code" in 3??) msg="a redirect: the key is never sent past one; open the URL in a browser instead" ;; esac
|
|
130
|
+
[ $rc -eq 0 ] && echo "FAILED $file: HTTP $code $msg" || echo "FAILED $file: could not reach the server (check the connection, VPN and proxy)"
|
|
131
|
+
FAILED=$((FAILED + 1))
|
|
132
|
+
fi
|
|
133
|
+
fi
|
|
134
|
+
done 3< <(items)
|
|
135
|
+
|
|
136
|
+
echo
|
|
137
|
+
if [ ${#SAVED[@]} -gt 0 ]; then
|
|
138
|
+
echo "Done. Upload these files to the chat (do not rename or edit them):"
|
|
139
|
+
for p in "${SAVED[@]}"; do echo " $p"; done
|
|
140
|
+
fi
|
|
141
|
+
[ $SKIPPED -gt 0 ] && echo "$SKIPPED file(s) skipped: run again and enter the value asked for."
|
|
142
|
+
[ $FAILED -gt 0 ] && { echo "$FAILED file(s) failed: send the messages above to the analyst." >&2; exit 1; }
|
|
143
|
+
exit 0
|