data-analyst-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -2
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +716 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +24 -0
- package/dist/report/scan_secrets.js +265 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +247 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +54 -4
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +170 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,686 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The treasury deep pack's spec, filter, packaging and discovery logic (port of sources/treasury/fetch.py).
|
|
3
|
+
* Output (<workspace>/data/treasury/): <name>.csv per dataset (API field names, values as published, "null" ->
|
|
4
|
+
* empty), fetch_spec.json (provenance) and README.md (data dictionary, values of the code columns, citation).
|
|
5
|
+
*/
|
|
6
|
+
import { createHash } from "node:crypto";
|
|
7
|
+
import fs from "node:fs";
|
|
8
|
+
import os from "node:os";
|
|
9
|
+
import path from "node:path";
|
|
10
|
+
import { DataError } from "../core/http.js";
|
|
11
|
+
import { dataDir } from "../core/package.js";
|
|
12
|
+
import * as pol from "../core/policy.js";
|
|
13
|
+
import { pyJsonDumps, pyStr } from "../core/py.js";
|
|
14
|
+
import { readCsv, unzip, writeCsv } from "../core/tabular.js";
|
|
15
|
+
import { parse } from "csv-parse/sync";
|
|
16
|
+
import { BASE, buildUrl, FiscalDataClient, filterString, fmtInt, MAX_PAGE, normalizeEndpoint, pyRepr, queryParams } from "./_treasury_api.js";
|
|
17
|
+
export const SOURCE = "treasury";
|
|
18
|
+
export const SEARCH_LIMIT = 20;
|
|
19
|
+
export const CATALOG_URL = "https://api.fiscaldata.treasury.gov/services/dtg/metadata/";
|
|
20
|
+
export const CITATION = "U.S. Department of the Treasury, Bureau of the Fiscal Service, Fiscal Data " +
|
|
21
|
+
"(https://fiscaldata.treasury.gov), {datasets}; retrieved {date} through the Fiscal Data API.";
|
|
22
|
+
/** Website dataset pages (Download button: whole table or a date range, no 10,000-row page limit). */
|
|
23
|
+
const DATASET_PAGES = [["/mts/", "monthly-treasury-statement"], ["/dts/", "daily-treasury-statement"],
|
|
24
|
+
["debt_to_penny", "debt-to-the-penny"], ["debt_outstanding", "historical-debt-outstanding"],
|
|
25
|
+
["avg_interest_rates", "average-interest-rates-treasury-securities"],
|
|
26
|
+
["interest_expense", "interest-expense-debt-outstanding"],
|
|
27
|
+
["rates_of_exchange", "treasury-reporting-rates-exchange"],
|
|
28
|
+
["/mspd/", "monthly-statement-public-debt"], ["auctions_query", "treasury-securities-auctions-data"],
|
|
29
|
+
["record_setting_auction", "record-setting-auction-data"], ["/rcm", "revenue-collections-management"],
|
|
30
|
+
["gift_contributions", "gift-contributions-reduce-debt-held-by-public"]];
|
|
31
|
+
const clone = (x) => (x === undefined ? x : JSON.parse(JSON.stringify(x)));
|
|
32
|
+
const isObj = (x) => !!x && typeof x === "object" && !Array.isArray(x);
|
|
33
|
+
/** Python str() of a value read from a row (a missing cell is None). */
|
|
34
|
+
const s = (v) => (v === null || v === undefined ? "None" : pyStr(v));
|
|
35
|
+
const nowZ = () => new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
|
|
36
|
+
// --------------------------------------------------------------------------- spec
|
|
37
|
+
/** The problems load_spec reported, and the spec with normalized endpoints. */
|
|
38
|
+
export function loadSpec(spec0) {
|
|
39
|
+
const spec = clone(spec0);
|
|
40
|
+
const ds = spec?.datasets;
|
|
41
|
+
if (!Array.isArray(ds) || !ds.length)
|
|
42
|
+
throw new DataError('spec.json needs a non-empty "datasets" list', "spec");
|
|
43
|
+
const names = new Set();
|
|
44
|
+
for (const d of ds) {
|
|
45
|
+
if (!isObj(d))
|
|
46
|
+
throw new DataError("each dataset must be an object with name and endpoint", "spec");
|
|
47
|
+
if (!/^[a-z][a-z0-9_]*$/.test(String(d.name ?? ""))) {
|
|
48
|
+
throw new DataError(`dataset name ${pyRepr(d.name)}: lowercase letters, digits and _ only`, "spec");
|
|
49
|
+
}
|
|
50
|
+
if (names.has(d.name))
|
|
51
|
+
throw new DataError(`dataset name ${pyRepr(d.name)} is used twice`, "spec");
|
|
52
|
+
names.add(d.name);
|
|
53
|
+
if (typeof d.endpoint !== "string")
|
|
54
|
+
throw new DataError(`dataset ${pyRepr(d.name)}: "endpoint" is required`, "spec");
|
|
55
|
+
d.endpoint = normalizeEndpoint(d.endpoint);
|
|
56
|
+
}
|
|
57
|
+
return spec;
|
|
58
|
+
}
|
|
59
|
+
export function dateField(d) {
|
|
60
|
+
return d.date_field || "record_date";
|
|
61
|
+
}
|
|
62
|
+
/** The dataset filter plus the spec's as_of cut-off (<date_field> <= as_of). */
|
|
63
|
+
export function effectiveFilter(spec, d) {
|
|
64
|
+
let f = d.filter || {};
|
|
65
|
+
const df = dateField(d);
|
|
66
|
+
if (typeof f === "string")
|
|
67
|
+
return f + (spec.as_of ? `,${df}:lte:${spec.as_of}` : "");
|
|
68
|
+
f = clone(f);
|
|
69
|
+
if (spec.as_of) {
|
|
70
|
+
const cur = f[df];
|
|
71
|
+
if (cur === null || cur === undefined)
|
|
72
|
+
f[df] = { lte: spec.as_of };
|
|
73
|
+
else if (isObj(cur) && !("lte" in cur) && !("lt" in cur) && !("eq" in cur))
|
|
74
|
+
cur.lte = spec.as_of;
|
|
75
|
+
}
|
|
76
|
+
return f;
|
|
77
|
+
}
|
|
78
|
+
// The API mishandles some characters in filter values [live, 2026-10-20]: a value with "&" or "," matches nothing
|
|
79
|
+
// (even URL-encoded), and in an "in" list a value with "," or parentheses is silently dropped. Conditions with such
|
|
80
|
+
// values are applied locally after the download instead (same rows, no silent loss).
|
|
81
|
+
const SAFE_VALUE = /^[A-Za-z0-9 ._/:-]*$/;
|
|
82
|
+
const SAFE_EQ_VALUE = /^[A-Za-z0-9 ._/:()-]*$/;
|
|
83
|
+
/** [api_part, local_part] of a friendly filter. A string filter is already API syntax: sent as is. */
|
|
84
|
+
export function splitFilter(filt) {
|
|
85
|
+
if (!filt || typeof filt === "string" || (isObj(filt) && !Object.keys(filt).length))
|
|
86
|
+
return [filt, {}];
|
|
87
|
+
const api = {}, local = {};
|
|
88
|
+
for (const [field, cond] of Object.entries(filt)) {
|
|
89
|
+
let vals;
|
|
90
|
+
if (isObj(cond))
|
|
91
|
+
vals = Object.entries(cond).flatMap(([op, v]) => (op === "in" && Array.isArray(v) ? v : [v]).map((x) => [op, x]));
|
|
92
|
+
else if (Array.isArray(cond))
|
|
93
|
+
vals = cond.map((x) => ["in", x]);
|
|
94
|
+
else
|
|
95
|
+
vals = [["eq", cond]];
|
|
96
|
+
const safe = vals.every(([op, x]) => (op === "in" ? SAFE_VALUE : SAFE_EQ_VALUE).test(s(x).trim()));
|
|
97
|
+
(safe ? api : local)[field] = cond;
|
|
98
|
+
}
|
|
99
|
+
return [api, local];
|
|
100
|
+
}
|
|
101
|
+
export function defaultSort(d, labels) {
|
|
102
|
+
if (d.sort)
|
|
103
|
+
return d.sort;
|
|
104
|
+
let srt = dateField(d);
|
|
105
|
+
if (labels && "src_line_nbr" in labels && srt !== "src_line_nbr")
|
|
106
|
+
srt += ",src_line_nbr"; // a stable tiebreaker
|
|
107
|
+
return srt;
|
|
108
|
+
}
|
|
109
|
+
export function datasetQueries(spec) {
|
|
110
|
+
return spec.datasets.map((d) => {
|
|
111
|
+
const filt = effectiveFilter(spec, d);
|
|
112
|
+
const [apiF, localF] = splitFilter(filt);
|
|
113
|
+
const sort = defaultSort(d);
|
|
114
|
+
return { name: d.name, endpoint: d.endpoint, fields: d.fields ?? null, date_field: dateField(d), filter: filterString(filt),
|
|
115
|
+
local_filter: localF, sort, url_json: buildUrl(d.endpoint, queryParams(d.fields, apiF, sort)),
|
|
116
|
+
url_csv: buildUrl(d.endpoint, queryParams(d.fields, apiF, sort, MAX_PAGE, 1, "csv")) };
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
export function fingerprint(spec) {
|
|
120
|
+
const q = datasetQueries(spec).map((x) => ({ endpoint: x.endpoint, fields: x.fields, filter: x.filter, sort: x.sort }));
|
|
121
|
+
return createHash("sha256").update(pyJsonDumps(q, true)).digest("hex").slice(0, 16);
|
|
122
|
+
}
|
|
123
|
+
const sha256 = (p) => createHash("sha256").update(fs.readFileSync(p)).digest("hex");
|
|
124
|
+
/** Plain-language unit warning for the README (amount units differ by dataset). A field is in millions when the
|
|
125
|
+
* API's own dataFormats says "$1,000,000" or its name says mil_. */
|
|
126
|
+
export function unitHint(endpoint, labels, types, formats) {
|
|
127
|
+
const cols = Object.keys(labels && Object.keys(labels).length ? labels : types || {});
|
|
128
|
+
const t = types || {};
|
|
129
|
+
const fm = formats || {};
|
|
130
|
+
const ty = (c) => String(t[c] ?? "");
|
|
131
|
+
const money = cols.filter((c) => ty(c).startsWith("CURRENCY") || ((c.endsWith("_amt") || c.endsWith("_bal")) &&
|
|
132
|
+
!(ty(c).startsWith("PERCENTAGE") || ty(c).startsWith("NUMBER"))));
|
|
133
|
+
if (endpoint.includes("/dts/"))
|
|
134
|
+
return "DTS amounts are in MILLIONS of dollars (rounded to the nearest million).";
|
|
135
|
+
const mil = money.filter((c) => c.includes("mil_") || c.endsWith("_mil_amt") || fm[c] === "$1,000,000");
|
|
136
|
+
if (endpoint.includes("rates_of_exchange"))
|
|
137
|
+
return "exchange_rate is FOREIGN currency units per one U.S. dollar.";
|
|
138
|
+
const pct = cols.filter((c) => ty(c).startsWith("PERCENTAGE"));
|
|
139
|
+
const other = money.filter((c) => !mil.includes(c));
|
|
140
|
+
let note = mil.length ? `Fields ${mil.join(", ")} are in MILLIONS of dollars.` : "";
|
|
141
|
+
if (other.length && Object.keys(fm).length)
|
|
142
|
+
note += mil.length ? " Other amounts are in dollars and cents." : "Amounts are in dollars and cents.";
|
|
143
|
+
else if (other.length) {
|
|
144
|
+
note += " Amount units were not confirmed from API metadata: before scaling, check the dataset's data " +
|
|
145
|
+
"dictionary on fiscaldata.treasury.gov (format $1,000,000 = millions; e.g. MSPD table 3 and the " +
|
|
146
|
+
"Schedules of Federal Debt are in millions).";
|
|
147
|
+
}
|
|
148
|
+
if (pct.length)
|
|
149
|
+
note += ` Percentage fields (${pct.join(", ")}) are already in percent (3.870 means 3.87 percent).`;
|
|
150
|
+
return note.trim();
|
|
151
|
+
}
|
|
152
|
+
// --------------------------------------------------------------------------- value summaries
|
|
153
|
+
const CATEGORY_COL = /(_cd|_desc|_type|_catg|_category|_nm|_name|_txt|_term|_security|reopening)$/;
|
|
154
|
+
const NOT_CATEGORY = /(_id|_date|_dt|_amt|_bal|_rate|_pct|_nbr)$/;
|
|
155
|
+
export const MAX_VALUES = 40; // README: list a column's values when it has at most this many distinct ones
|
|
156
|
+
export const MAX_VALUES_CLI = 200; // values query: list values up to this many per column (the most frequent)
|
|
157
|
+
const MAX_CODE_VALUES = 12; // columns with at most this many values form the code combinations
|
|
158
|
+
const blank = (v) => v === "" || v === null || v === undefined || v === "null";
|
|
159
|
+
const cmpStr = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
160
|
+
/** Distinct values (with counts) of the category columns, and the combinations of the low-cardinality code
|
|
161
|
+
* columns with an example description each. */
|
|
162
|
+
export function valueSummary(rows, columns, types, maxValues = MAX_VALUES) {
|
|
163
|
+
const t = types || {};
|
|
164
|
+
const cols = columns.filter((c) => CATEGORY_COL.test(c) && !NOT_CATEGORY.test(c) && String(t[c] || "STRING").toUpperCase() === "STRING");
|
|
165
|
+
const perCol = new Map();
|
|
166
|
+
const codes = [];
|
|
167
|
+
for (const c of cols) {
|
|
168
|
+
const counts = new Map();
|
|
169
|
+
for (const r of rows) {
|
|
170
|
+
const v = blank(r[c]) ? "(empty)" : pyStr(r[c]);
|
|
171
|
+
counts.set(v, (counts.get(v) || 0) + 1);
|
|
172
|
+
}
|
|
173
|
+
const ranked = new Map([...counts].sort((a, b) => b[1] - a[1] || cmpStr(a[0], b[0])));
|
|
174
|
+
if (counts.size <= maxValues)
|
|
175
|
+
perCol.set(c, ranked);
|
|
176
|
+
else if (maxValues >= MAX_VALUES_CLI) {
|
|
177
|
+
perCol.set(c, { distinct: counts.size, values: new Map([...ranked].slice(0, maxValues)),
|
|
178
|
+
more: `+${fmtInt(counts.size - maxValues)} more: narrow the filter (one record_date, or a code column such as a ` +
|
|
179
|
+
"type or category) to list them" });
|
|
180
|
+
}
|
|
181
|
+
else {
|
|
182
|
+
perCol.set(c, { distinct: counts.size, examples: [...counts.keys()].slice(0, 8),
|
|
183
|
+
all_values: "request_schema search \"<endpoint> <filter JSON> values\"" });
|
|
184
|
+
}
|
|
185
|
+
if (counts.size > 1 && counts.size <= MAX_CODE_VALUES && c.endsWith("_cd"))
|
|
186
|
+
codes.push(c);
|
|
187
|
+
}
|
|
188
|
+
const desc = cols.find((c) => c.endsWith("_desc"));
|
|
189
|
+
const combos = new Map();
|
|
190
|
+
if (codes.length) {
|
|
191
|
+
for (const r of rows) {
|
|
192
|
+
const k = codes.map((c) => (blank(r[c]) ? "(empty)" : pyStr(r[c])));
|
|
193
|
+
const key = JSON.stringify(k);
|
|
194
|
+
if (!combos.has(key))
|
|
195
|
+
combos.set(key, { k, rows: 0, examples: [] });
|
|
196
|
+
const e = combos.get(key);
|
|
197
|
+
e.rows++;
|
|
198
|
+
if (desc && !blank(r[desc]) && !e.examples.includes(r[desc]) && e.examples.length < 3)
|
|
199
|
+
e.examples.push(r[desc]);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
const eg = desc ? `e.g. ${desc}` : "e.g.";
|
|
203
|
+
return { rows: rows.length, columns: perCol,
|
|
204
|
+
code_combinations: [...combos.values()].sort((a, b) => b.rows - a.rows).map((v) => ({
|
|
205
|
+
...Object.fromEntries(codes.map((c, i) => [c, v.k[i]])), rows: v.rows, [eg]: v.examples
|
|
206
|
+
})) };
|
|
207
|
+
}
|
|
208
|
+
/** A value summary as JSON (value lists as [value, rows] pairs, so the order by frequency survives). */
|
|
209
|
+
export function summaryJson(vs) {
|
|
210
|
+
const pairs = (m) => [...m];
|
|
211
|
+
const columns = {};
|
|
212
|
+
for (const [c, v] of vs.columns) {
|
|
213
|
+
if (v instanceof Map)
|
|
214
|
+
columns[c] = pairs(v);
|
|
215
|
+
else if ("values" in v)
|
|
216
|
+
columns[c] = { distinct: v.distinct, values: pairs(v.values), more: v.more };
|
|
217
|
+
else
|
|
218
|
+
columns[c] = v;
|
|
219
|
+
}
|
|
220
|
+
return { rows: vs.rows, columns, code_combinations: vs.code_combinations };
|
|
221
|
+
}
|
|
222
|
+
// --------------------------------------------------------------------------- local filters
|
|
223
|
+
/** API filter syntax ("record_date:gte:2020-01-01,type:in:(Bills,Notes)") as a friendly filter. */
|
|
224
|
+
export function parseFilterString(flt) {
|
|
225
|
+
const parts = [];
|
|
226
|
+
let depth = 0, cur = "";
|
|
227
|
+
for (const ch of flt) {
|
|
228
|
+
depth += (ch === "(" ? 1 : 0) - (ch === ")" ? 1 : 0);
|
|
229
|
+
if (ch === "," && depth === 0) {
|
|
230
|
+
parts.push(cur);
|
|
231
|
+
cur = "";
|
|
232
|
+
}
|
|
233
|
+
else
|
|
234
|
+
cur += ch;
|
|
235
|
+
}
|
|
236
|
+
parts.push(cur);
|
|
237
|
+
const out = {};
|
|
238
|
+
for (const part of parts.map((p) => p.trim()).filter(Boolean)) {
|
|
239
|
+
const bits = part.split(":");
|
|
240
|
+
const field = bits[0], op = bits[1] ?? "";
|
|
241
|
+
let v = bits.length > 2 ? bits.slice(2).join(":") : "";
|
|
242
|
+
if (!["eq", "lt", "lte", "gt", "gte", "in"].includes(op)) {
|
|
243
|
+
throw new DataError(`filter condition ${pyRepr(part)}: the operator must be one of eq, lt, lte, gt, gte, in`, "spec");
|
|
244
|
+
}
|
|
245
|
+
if (op === "in")
|
|
246
|
+
v = String(v).trim().replace(/^[()]+|[()]+$/g, "").split(",").map((x) => x.trim());
|
|
247
|
+
(out[field.trim()] ??= {})[op] = v;
|
|
248
|
+
}
|
|
249
|
+
return out;
|
|
250
|
+
}
|
|
251
|
+
/** Python float(): null when the text is not a number. */
|
|
252
|
+
function pyFloat(x) {
|
|
253
|
+
if (typeof x === "number")
|
|
254
|
+
return x;
|
|
255
|
+
if (typeof x === "boolean")
|
|
256
|
+
return x ? 1 : 0;
|
|
257
|
+
if (typeof x !== "string")
|
|
258
|
+
return null;
|
|
259
|
+
const t = x.trim();
|
|
260
|
+
if (/^[+-]?(nan|inf|infinity)$/i.test(t))
|
|
261
|
+
return t.toLowerCase().includes("nan") ? NaN : t.startsWith("-") ? -Infinity : Infinity;
|
|
262
|
+
if (!/^[+-]?((\d(_?\d)*)?\.?\d(_?\d)*|\d(_?\d)*\.)([eE][+-]?\d(_?\d)*)?$/.test(t))
|
|
263
|
+
return null;
|
|
264
|
+
return Number(t.replace(/_/g, ""));
|
|
265
|
+
}
|
|
266
|
+
/** Apply a friendly filter (or an API filter string) locally. String compare works for YYYY-MM-DD dates; numbers
|
|
267
|
+
* compare as floats. */
|
|
268
|
+
export function applyFilter(rows, flt0) {
|
|
269
|
+
const flt = typeof flt0 === "string" ? parseFilterString(flt0) : flt0;
|
|
270
|
+
if (!flt || !Object.keys(flt).length)
|
|
271
|
+
return rows;
|
|
272
|
+
const cmp = (v, op, x) => {
|
|
273
|
+
let a = pyFloat(v), b = pyFloat(x);
|
|
274
|
+
if (a === null || b === null)
|
|
275
|
+
[a, b] = [s(v), s(x)];
|
|
276
|
+
return op === "eq" ? a === b : op === "lt" ? a < b : op === "lte" ? a <= b : op === "gt" ? a > b : op === "gte" ? a >= b
|
|
277
|
+
: (() => { throw new DataError(`filter operator ${pyRepr(op)}: use one of eq, lt, lte, gt, gte, in`, "spec"); })();
|
|
278
|
+
};
|
|
279
|
+
return rows.filter((r) => {
|
|
280
|
+
let ok = true;
|
|
281
|
+
for (const [field, cond] of Object.entries(flt)) {
|
|
282
|
+
const v = r[field];
|
|
283
|
+
const conds = isObj(cond) ? cond : Array.isArray(cond) ? { in: cond } : { eq: cond };
|
|
284
|
+
for (const [op, x] of Object.entries(conds)) {
|
|
285
|
+
const hit = op === "in" ? (Array.isArray(x) ? x : [...String(x)]).map(s).includes(s(v)) : !blank(v) && cmp(v, op, x);
|
|
286
|
+
ok = ok && hit;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
return ok;
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
// --------------------------------------------------------------------------- writing
|
|
293
|
+
function writeTable(p, rows, fields) {
|
|
294
|
+
const clean = rows.map((r) => Object.fromEntries(fields.map((k) => [k, blank(r[k]) ? "" : r[k]])));
|
|
295
|
+
fs.writeFileSync(p, writeCsv(clean, fields));
|
|
296
|
+
}
|
|
297
|
+
function describeCsv(p, dfield) {
|
|
298
|
+
const text = fs.readFileSync(p, "utf8");
|
|
299
|
+
const rows = readCsv(text);
|
|
300
|
+
let first = null, last = null;
|
|
301
|
+
for (const r of rows) {
|
|
302
|
+
const dt = r[dfield] || "";
|
|
303
|
+
if (dt) {
|
|
304
|
+
first = first === null || dt < first ? dt : first;
|
|
305
|
+
last = last === null || dt > last ? dt : last;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
return { rows: rows.length, columns: headerOf(text), first_date: first, last_date: last };
|
|
309
|
+
}
|
|
310
|
+
/** The header row of a CSV text (csv.DictReader fieldnames). */
|
|
311
|
+
function headerOf(text) {
|
|
312
|
+
const grid = parse(text.replace(/^/, ""), { relax_column_count: true, relax_quotes: true,
|
|
313
|
+
skip_empty_lines: true, to_line: 1 });
|
|
314
|
+
return grid[0] || [];
|
|
315
|
+
}
|
|
316
|
+
/** fetch_spec.json + README.md next to the CSV files. */
|
|
317
|
+
export function writePackage(spec, out, results, route) {
|
|
318
|
+
const d = dataDir(out, SOURCE);
|
|
319
|
+
const now = nowZ();
|
|
320
|
+
const qs = Object.fromEntries(datasetQueries(spec).map((q) => [q.name, q]));
|
|
321
|
+
const prov = { source: SOURCE, publisher: "U.S. Treasury Fiscal Data API", base_url: BASE, retrieved: now, route,
|
|
322
|
+
fingerprint: fingerprint(spec), spec, datasets: {} };
|
|
323
|
+
for (const [name, r] of Object.entries(results)) {
|
|
324
|
+
const q = qs[name];
|
|
325
|
+
const p = path.join(d, `${name}.csv`);
|
|
326
|
+
const info = describeCsv(p, q.date_field);
|
|
327
|
+
const labels = r.labels && Object.keys(r.labels).length ? r.labels : Object.fromEntries(info.columns.map((k) => [k, k]));
|
|
328
|
+
prov.datasets[name] = {
|
|
329
|
+
endpoint: q.endpoint, filter: q.filter, local_filter: q.local_filter, sort: r.sort || q.sort,
|
|
330
|
+
date_field: q.date_field, query_url: r.url || q.url_json,
|
|
331
|
+
file: `data/${SOURCE}/${name}.csv`, sha256: sha256(p), ...info, api_total_count: r.total,
|
|
332
|
+
units: unitHint(q.endpoint, labels, r.types || {}, r.formats || {}),
|
|
333
|
+
labels: r.labels || {}, data_types: r.types || {}, data_formats: r.formats || {}
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
prov.tables = Object.entries(prov.datasets).map(([k, v]) => ({ name: k, file: `${k}.csv`, rows: v.rows }));
|
|
337
|
+
fs.writeFileSync(path.join(d, "fetch_spec.json"), JSON.stringify(prov, null, 2));
|
|
338
|
+
const names = [...new Set(Object.values(prov.datasets).map((v) => v.endpoint))].sort(cmpStr).join(", ");
|
|
339
|
+
const citation = CITATION.replace("{datasets}", names).replace("{date}", now.slice(0, 10));
|
|
340
|
+
const how = route === "manual_download" ? "downloaded by hand with the bundled links, then packaged with ingest" : "Fiscal Data API";
|
|
341
|
+
const lines = [`# Treasury Fiscal Data download: ${spec.description ?? ""}`, "",
|
|
342
|
+
`- Retrieved: ${pol.localStamp(now)} (${how})`,
|
|
343
|
+
`- Data cut-off (as_of): ${spec.as_of || "none (latest available at retrieval)"}`,
|
|
344
|
+
`- Source: ${BASE}`, "",
|
|
345
|
+
"Values are exactly as published. The API returns every value as a string; its \"null\" is an empty " +
|
|
346
|
+
"cell here. Units differ by dataset: read the unit line under each file before scaling.", ""];
|
|
347
|
+
for (const [name, x] of Object.entries(prov.datasets)) {
|
|
348
|
+
const hasLocal = Object.keys(x.local_filter || {}).length > 0;
|
|
349
|
+
const local = hasLocal ? ["- Applied locally after the download (the API mishandles `&`, `,` or parentheses in these " +
|
|
350
|
+
`values): \`${pyJsonDumps(x.local_filter, false, false)}\``] : [];
|
|
351
|
+
lines.push(`## ${name}.csv`, "", `- Endpoint: \`${x.endpoint}\``, `- Filter: \`${x.filter || "none"}\``, ...local, `- Rows: ${fmtInt(x.rows)}` + (Number.isInteger(x.api_total_count)
|
|
352
|
+
? ` (API total-count${hasLocal ? " before the local filter" : ""}: ${fmtInt(x.api_total_count)})` : ""), `- ${x.date_field} range: ${x.first_date ?? "None"} to ${x.last_date ?? "None"}`, `- Units: ${x.units || "see the field types below"}`, `- SHA-256: \`${x.sha256}\``, `- Query: ${x.query_url}`, "", "| Field | Label | Type | Format |", "|---|---|---|---|");
|
|
353
|
+
for (const c of x.columns)
|
|
354
|
+
lines.push(`| \`${c}\` | ${x.labels[c] ?? ""} | ${x.data_types[c] ?? ""} | ${x.data_formats[c] ?? ""} |`);
|
|
355
|
+
lines.push("");
|
|
356
|
+
const vs = valueSummary(readCsv(fs.readFileSync(path.join(d, `${name}.csv`), "utf8")), x.columns, x.data_types);
|
|
357
|
+
if (vs.columns.size) {
|
|
358
|
+
lines.push("### Values in this file", "", "Filter on the column a value actually appears in (exact text, case-sensitive).", "", "| Field | Values (rows) |", "|---|---|");
|
|
359
|
+
for (const [c, v] of vs.columns) {
|
|
360
|
+
const txt = v instanceof Map ? [...v].map(([k, n]) => `${k} (${fmtInt(n)})`).join("; ")
|
|
361
|
+
: `${fmtInt(v.distinct)} distinct, e.g. ` + ("examples" in v ? v.examples : [...v.values.keys()]).map(String).join("; ");
|
|
362
|
+
lines.push(`| \`${c}\` | ${txt.replaceAll("|", "/")} |`);
|
|
363
|
+
}
|
|
364
|
+
lines.push("");
|
|
365
|
+
}
|
|
366
|
+
if (vs.code_combinations.length) {
|
|
367
|
+
const keys = Object.keys(vs.code_combinations[0]).filter((k) => k !== "rows");
|
|
368
|
+
lines.push("Code combinations:", "", "| " + keys.join(" | ") + " | rows |", "|" + "---|".repeat(keys.length + 1));
|
|
369
|
+
for (const cmb of vs.code_combinations.slice(0, 30)) {
|
|
370
|
+
lines.push("| " + keys.map((k) => (Array.isArray(cmb[k]) ? cmb[k].map(String).join("; ") : pyStr(cmb[k]))).join(" | ") +
|
|
371
|
+
` | ${fmtInt(cmb.rows)} |`);
|
|
372
|
+
}
|
|
373
|
+
lines.push("");
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
lines.push("## Citation", "", citation, "", "Fiscal Data is published by the U.S. government and is in the public domain; redistribution is " +
|
|
377
|
+
"permitted. Cite the source and the retrieval date, because Treasury revises some series.", "");
|
|
378
|
+
fs.writeFileSync(path.join(d, "README.md"), lines.join("\n"));
|
|
379
|
+
return { prov, citation };
|
|
380
|
+
}
|
|
381
|
+
// --------------------------------------------------------------------------- download
|
|
382
|
+
export function emptyMessage(name, endpoint, filt) {
|
|
383
|
+
return `${name}: the filter ${filterString(filt) || "(none)"} matches no rows in ${endpoint}. Filter values must ` +
|
|
384
|
+
"match exactly (case, spelling, YYYY-MM-DD dates; a malformed date returns no rows, not an error), and " +
|
|
385
|
+
"the value must be in the column you filter on (MTS: data_type_cd D/S/T vs record_type_cd). See which " +
|
|
386
|
+
`column holds which value: request_schema search "${endpoint} {"record_date": "<one date>"} values". ` +
|
|
387
|
+
"If no rows is a valid result, set \"allow_empty\": true on the dataset.";
|
|
388
|
+
}
|
|
389
|
+
export async function download(spec, out, client) {
|
|
390
|
+
const d = dataDir(out, SOURCE);
|
|
391
|
+
fs.mkdirSync(d, { recursive: true });
|
|
392
|
+
const results = {};
|
|
393
|
+
const warnings = [], notes = [];
|
|
394
|
+
for (const ds of spec.datasets) {
|
|
395
|
+
const { name, endpoint: ep } = ds;
|
|
396
|
+
let fields = ds.fields;
|
|
397
|
+
const filt = effectiveFilter(spec, ds);
|
|
398
|
+
const [apiF, localF] = splitFilter(filt); // values the API mishandles are filtered here
|
|
399
|
+
const full = await client.meta(ep, apiF, null); // all columns: labels, types, row count
|
|
400
|
+
const labels = full.labels || {};
|
|
401
|
+
const df = dateField(ds);
|
|
402
|
+
if (!(df in labels)) {
|
|
403
|
+
const dates = Object.entries(full.dataTypes || {}).filter(([, t]) => t === "DATE").map(([k]) => k);
|
|
404
|
+
throw new DataError(`${name}: ${ep} has no ${pyRepr(df)} column; set "date_field" to one of ${pyRepr(dates)} ` +
|
|
405
|
+
"and use it in the filter", "spec");
|
|
406
|
+
}
|
|
407
|
+
const unknown = Object.keys(localF).filter((f) => !(f in labels));
|
|
408
|
+
if (unknown.length)
|
|
409
|
+
throw new DataError(`${name}: filter fields not in ${ep}: ${pyRepr(unknown)}. Fields: ${pyRepr(Object.keys(labels))}`, "spec");
|
|
410
|
+
if (Number(full["total-count"] || 0) === 0 && !ds.allow_empty)
|
|
411
|
+
throw new DataError(emptyMessage(name, ep, filt), "spec");
|
|
412
|
+
if (fields && fields.length) {
|
|
413
|
+
fields = [...fields, ...Object.keys(localF).filter((f) => !fields.includes(f))]; // the local filter needs its columns
|
|
414
|
+
const bad = fields.filter((f) => !(f in labels));
|
|
415
|
+
if (bad.length)
|
|
416
|
+
throw new DataError(`${name}: fields not in ${ep}: ${pyRepr(bad)}. Fields: ${pyRepr(Object.keys(labels))}`, "spec");
|
|
417
|
+
const sub = await client.meta(ep, apiF, null, fields);
|
|
418
|
+
if (Number(sub["total-count"] || 0) !== Number(full["total-count"] || 0)) {
|
|
419
|
+
throw new DataError(`${name}: requesting only ${pyRepr(fields)} returns ${sub["total-count"] ?? "None"} rows instead of ` +
|
|
420
|
+
`${full["total-count"] ?? "None"}: the API grouped and summed rows (it does this when the fields leave ` +
|
|
421
|
+
"out the columns that identify a row, and the sums can mix totals with details). Add the " +
|
|
422
|
+
"row-identity columns (descriptions, src_line_nbr, type codes) or drop \"fields\".", "spec");
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
const sort = defaultSort(ds, labels);
|
|
426
|
+
let [rows, meta] = await client.fetchAll(ep, fields, apiF, sort);
|
|
427
|
+
if (Object.keys(localF).length) {
|
|
428
|
+
const nApi = rows.length;
|
|
429
|
+
rows = applyFilter(rows, localF);
|
|
430
|
+
notes.push(`${name}: filtered locally on ${pyRepr(Object.keys(localF).sort(cmpStr))}: kept ${fmtInt(rows.length)} of ${fmtInt(nApi)} rows`);
|
|
431
|
+
if (!rows.length && !ds.allow_empty)
|
|
432
|
+
throw new DataError(emptyMessage(name, ep, filt), "spec");
|
|
433
|
+
}
|
|
434
|
+
const cols = fields && fields.length ? fields : Object.keys(labels);
|
|
435
|
+
const tmp = path.join(d, `.${name}.csv.part`);
|
|
436
|
+
writeTable(tmp, rows, cols);
|
|
437
|
+
fs.renameSync(tmp, path.join(d, `${name}.csv`));
|
|
438
|
+
if (!rows.length)
|
|
439
|
+
warnings.push(`${name}: the query matched no rows (allowed by "allow_empty").`);
|
|
440
|
+
results[name] = { total: Number(meta["total-count"] || rows.length), labels, types: full.dataTypes, formats: full.dataFormats,
|
|
441
|
+
sort, url: buildUrl(ep, queryParams(fields, apiF, sort)) };
|
|
442
|
+
}
|
|
443
|
+
const { prov, citation } = writePackage(spec, out, results, "api");
|
|
444
|
+
return { prov, citation, warnings, notes };
|
|
445
|
+
}
|
|
446
|
+
/** What is already in data/treasury/ for this spec. */
|
|
447
|
+
export function existingData(out, spec) {
|
|
448
|
+
const p = path.join(dataDir(out, SOURCE), "fetch_spec.json");
|
|
449
|
+
if (!fs.existsSync(p))
|
|
450
|
+
return null;
|
|
451
|
+
let saved;
|
|
452
|
+
try {
|
|
453
|
+
saved = JSON.parse(fs.readFileSync(p, "utf8"));
|
|
454
|
+
}
|
|
455
|
+
catch {
|
|
456
|
+
return null;
|
|
457
|
+
}
|
|
458
|
+
const filesOk = spec.datasets.every((d) => fs.existsSync(path.join(dataDir(out, SOURCE), `${d.name}.csv`)));
|
|
459
|
+
return { retrieved: saved.retrieved, route: saved.route, files_ok: filesOk, spec_matches: saved.fingerprint === fingerprint(spec),
|
|
460
|
+
as_of: saved.spec?.as_of, prov: saved };
|
|
461
|
+
}
|
|
462
|
+
// --------------------------------------------------------------------------- manual route
|
|
463
|
+
export function datasetPage(endpoint) {
|
|
464
|
+
for (const [key, slug] of DATASET_PAGES)
|
|
465
|
+
if (("/" + endpoint).includes(key))
|
|
466
|
+
return `https://fiscaldata.treasury.gov/datasets/${slug}/`;
|
|
467
|
+
return "https://fiscaldata.treasury.gov/datasets/ (search for the table)";
|
|
468
|
+
}
|
|
469
|
+
/** The CSV links of a spec, one per dataset (10,000 rows per page; later pages name_2.csv ...). */
|
|
470
|
+
export function manualItems(spec0) {
|
|
471
|
+
const spec = loadSpec(spec0);
|
|
472
|
+
return datasetQueries(spec).map((q) => ({ name: q.name, filename: `${q.name}.csv`, url: q.url_csv, needs_key: false,
|
|
473
|
+
method: "GET", optional: false,
|
|
474
|
+
note: `Endpoint ${q.endpoint}; filter ${q.filter || "none"}` +
|
|
475
|
+
(Object.keys(q.local_filter).length ? `; the assistant applies the rest of the filter after upload: ${pyJsonDumps(q.local_filter)}` : "") +
|
|
476
|
+
`. At most 10,000 rows per file: if it has exactly 10,000, get the next page with page%5Bnumber%5D=2 and save it as ` +
|
|
477
|
+
`${q.name}_2.csv. Whole table: the Download button on ${datasetPage(q.endpoint)}` }));
|
|
478
|
+
}
|
|
479
|
+
function collectCsvs(paths, tmpdir) {
|
|
480
|
+
const files = [];
|
|
481
|
+
const walk = (p) => {
|
|
482
|
+
for (const n of fs.readdirSync(p)) {
|
|
483
|
+
const f = path.join(p, n);
|
|
484
|
+
if (fs.statSync(f).isDirectory())
|
|
485
|
+
walk(f);
|
|
486
|
+
else if (/\.(csv|zip)$/i.test(n))
|
|
487
|
+
files.push(f);
|
|
488
|
+
}
|
|
489
|
+
};
|
|
490
|
+
for (const p of paths) {
|
|
491
|
+
if (fs.existsSync(p) && fs.statSync(p).isDirectory())
|
|
492
|
+
walk(p);
|
|
493
|
+
else if (fs.existsSync(p))
|
|
494
|
+
files.push(p);
|
|
495
|
+
else
|
|
496
|
+
throw new DataError(`not found: ${p}`, "spec");
|
|
497
|
+
}
|
|
498
|
+
const out = [];
|
|
499
|
+
for (const f of files) {
|
|
500
|
+
if (/\.zip$/i.test(f)) {
|
|
501
|
+
for (const [m, data] of Object.entries(unzip(fs.readFileSync(f)))) {
|
|
502
|
+
if (/\.csv$/i.test(m) && !m.includes("..") && !m.startsWith("/") && !m.startsWith("\\")) {
|
|
503
|
+
const target = path.join(tmpdir, path.basename(m));
|
|
504
|
+
fs.writeFileSync(target, data);
|
|
505
|
+
out.push(target);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
else
|
|
510
|
+
out.push(f);
|
|
511
|
+
}
|
|
512
|
+
return out;
|
|
513
|
+
}
|
|
514
|
+
const reEsc = (x) => x.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
515
|
+
/** Package files the user downloaded by hand: API CSV links (maybe several pages) or a website download (whole
|
|
516
|
+
* table: the spec's filter is applied here). Headers must be API field names. */
|
|
517
|
+
export function ingest(paths, spec, out) {
|
|
518
|
+
const d = dataDir(out, SOURCE);
|
|
519
|
+
fs.mkdirSync(d, { recursive: true });
|
|
520
|
+
const results = {};
|
|
521
|
+
const warnings = [];
|
|
522
|
+
if (!paths.length)
|
|
523
|
+
throw new DataError("ingest needs the downloaded files or their folder", "spec");
|
|
524
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "treasury-ingest-"));
|
|
525
|
+
try {
|
|
526
|
+
const files = collectCsvs(paths, tmp);
|
|
527
|
+
for (const ds of spec.datasets) {
|
|
528
|
+
const want = new Set(ds.fields || []);
|
|
529
|
+
let mine = files.filter((f) => new RegExp(`^${reEsc(ds.name)}(_p?\\d+)?\\.csv$`, "i").test(path.basename(f)));
|
|
530
|
+
if (!mine.length && spec.datasets.length === 1)
|
|
531
|
+
mine = files;
|
|
532
|
+
if (!mine.length) {
|
|
533
|
+
throw new DataError(`no uploaded file for dataset ${pyRepr(ds.name)}; expected ${ds.name}.csv ` +
|
|
534
|
+
`(pages: ${ds.name}_2.csv ...). Got: ${pyRepr(files.map((f) => path.basename(f)))}`, "spec");
|
|
535
|
+
}
|
|
536
|
+
let rows = [];
|
|
537
|
+
let header = null;
|
|
538
|
+
for (const f of [...mine].sort(cmpStr)) {
|
|
539
|
+
const text = fs.readFileSync(f, "utf8");
|
|
540
|
+
const h = headerOf(text);
|
|
541
|
+
if (header && JSON.stringify(h) !== JSON.stringify(header))
|
|
542
|
+
throw new DataError(`${f}: columns differ from the other files of ${ds.name}`, "spec");
|
|
543
|
+
header = h;
|
|
544
|
+
for (const r of readCsv(text)) { // API CSV guards against formula injection: '-123.45 -> -123.45
|
|
545
|
+
rows.push(Object.fromEntries(Object.entries(r).map(([k, v]) => [k, typeof v === "string" && /^'[-+=@]/.test(v) ? v.slice(1) : v])));
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
const hdr = header || [];
|
|
549
|
+
if (hdr.some((c) => c.includes(" "))) {
|
|
550
|
+
throw new DataError(`${ds.name}: the headers are display labels (${pyRepr(hdr.slice(0, 3))} ...), not API field ` +
|
|
551
|
+
"names. Ask the user to use the CSV links from manual_bundle.", "spec");
|
|
552
|
+
}
|
|
553
|
+
const lack = [...want].filter((c) => !hdr.includes(c)).sort(cmpStr);
|
|
554
|
+
if (lack.length)
|
|
555
|
+
warnings.push(`${ds.name}: requested fields missing from the upload: ${pyRepr(lack)}`);
|
|
556
|
+
const n0 = rows.length;
|
|
557
|
+
rows = applyFilter(rows, effectiveFilter(spec, ds));
|
|
558
|
+
if (rows.length < n0) {
|
|
559
|
+
warnings.push(`${ds.name}: applied the spec filter locally: kept ${fmtInt(rows.length)} of ${fmtInt(n0)} rows (a whole-table website download)`);
|
|
560
|
+
}
|
|
561
|
+
if (!rows.length && !ds.allow_empty) {
|
|
562
|
+
throw new DataError(emptyMessage(ds.name, ds.endpoint, effectiveFilter(spec, ds)) + ` (${fmtInt(n0)} rows were uploaded before filtering.)`, "spec");
|
|
563
|
+
}
|
|
564
|
+
const seen = new Set();
|
|
565
|
+
const uniq = [];
|
|
566
|
+
for (const r of rows) { // a page uploaded twice would double count
|
|
567
|
+
const key = JSON.stringify(hdr.map((c) => r[c] ?? null));
|
|
568
|
+
if (!seen.has(key)) {
|
|
569
|
+
seen.add(key);
|
|
570
|
+
uniq.push(r);
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
if (uniq.length !== rows.length)
|
|
574
|
+
warnings.push(`${ds.name}: dropped ${rows.length - uniq.length} duplicate rows (overlapping pages)`);
|
|
575
|
+
const sortCols = defaultSort(ds, Object.fromEntries(hdr.map((c) => [c, c]))).split(",");
|
|
576
|
+
const key = (r) => sortCols.map((c) => (c in r ? s(r[c]) : ""));
|
|
577
|
+
uniq.sort((a, b) => {
|
|
578
|
+
const ka = key(a), kb = key(b);
|
|
579
|
+
for (let i = 0; i < ka.length; i++)
|
|
580
|
+
if (ka[i] !== kb[i])
|
|
581
|
+
return cmpStr(ka[i], kb[i]);
|
|
582
|
+
return 0;
|
|
583
|
+
});
|
|
584
|
+
writeTable(path.join(d, `${ds.name}.csv`), uniq, hdr);
|
|
585
|
+
if (n0 && n0 % MAX_PAGE === 0) {
|
|
586
|
+
warnings.push(`${ds.name}: ${fmtInt(n0)} rows uploaded, a multiple of the 10,000-row page size; a page ` +
|
|
587
|
+
"may be missing. Ask the user whether there is one more page.");
|
|
588
|
+
}
|
|
589
|
+
results[ds.name] = { total: null, labels: {}, types: {}, formats: {} };
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
finally {
|
|
593
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
594
|
+
}
|
|
595
|
+
const { prov, citation } = writePackage(spec, out, results, "manual_download");
|
|
596
|
+
return { prov, citation, warnings, notes: [] };
|
|
597
|
+
}
|
|
598
|
+
// --------------------------------------------------------------------------- discovery
|
|
599
|
+
export async function fieldsInfo(endpoint, filters, dfield = "record_date") {
|
|
600
|
+
const [apiF, localF] = splitFilter(filters);
|
|
601
|
+
const c = new FiscalDataClient();
|
|
602
|
+
let m = await c.meta(endpoint, apiF, dfield);
|
|
603
|
+
let types = m.dataTypes || {};
|
|
604
|
+
const dates = Object.entries(types).filter(([, t]) => t === "DATE").map(([k]) => k);
|
|
605
|
+
if (!m.first_row && dates.length && !dates.includes(dfield)) { // e.g. I bonds: earning_period_start
|
|
606
|
+
m = await c.meta(endpoint, apiF, dates[0]);
|
|
607
|
+
types = m.dataTypes || types;
|
|
608
|
+
}
|
|
609
|
+
const note = Object.keys(localF).length ? { note: `total_count ignores ${pyRepr(Object.keys(localF).sort(cmpStr))}: the API ` +
|
|
610
|
+
"mishandles those values (&, comma or parentheses); fetch applies that part locally. Use values for exact counts." } : {};
|
|
611
|
+
const ep = normalizeEndpoint(endpoint);
|
|
612
|
+
return { endpoint: ep, ...note, total_count: m["total-count"] ?? null,
|
|
613
|
+
units: unitHint(ep, m.labels || {}, types, m.dataFormats || {}),
|
|
614
|
+
date_columns: Object.entries(types).filter(([, t]) => t === "DATE").map(([k]) => k),
|
|
615
|
+
fields: Object.entries(m.labels || {}).map(([k, v]) => ({ field: k, label: v, type: types[k] ?? null,
|
|
616
|
+
format: (m.dataFormats || {})[k] ?? null })),
|
|
617
|
+
first_row: m.first_row, last_row: m.last_row };
|
|
618
|
+
}
|
|
619
|
+
/** The rows under the filter (refuses above `limit`), summarized: which values each code column holds. */
|
|
620
|
+
export async function valuesInfo(endpoint, filters, limit = 20000) {
|
|
621
|
+
const client = new FiscalDataClient();
|
|
622
|
+
const [apiF, localF] = splitFilter(filters);
|
|
623
|
+
const m = await client.meta(endpoint, apiF, null);
|
|
624
|
+
const total = Number(m["total-count"] || 0);
|
|
625
|
+
if (total > limit) {
|
|
626
|
+
throw new DataError(`${fmtInt(total)} rows match; values reads at most ${fmtInt(limit)}. Narrow the filter, e.g. ` +
|
|
627
|
+
"{\"record_date\": \"<one date>\"}", "spec");
|
|
628
|
+
}
|
|
629
|
+
let [rows] = await client.fetchAll(endpoint, null, apiF, null);
|
|
630
|
+
rows = applyFilter(rows, localF);
|
|
631
|
+
return { endpoint: normalizeEndpoint(endpoint), filter: filters ?? null,
|
|
632
|
+
...summaryJson(valueSummary(rows, Object.keys(m.labels || {}), m.dataTypes || {}, MAX_VALUES_CLI)) };
|
|
633
|
+
}
|
|
634
|
+
let catalogCache = null;
|
|
635
|
+
export function loadCatalog(packDir) {
|
|
636
|
+
if (!catalogCache)
|
|
637
|
+
catalogCache = pol.readJson(packDir, "catalog.json").endpoints;
|
|
638
|
+
return catalogCache;
|
|
639
|
+
}
|
|
640
|
+
/** catalog.json: endpoint registry merged with the live catalog snapshot and the curated guidance. Stale entries
|
|
641
|
+
* (HTTP 404) are listed last. */
|
|
642
|
+
export function searchBundled(packDir, words) {
|
|
643
|
+
const hits = loadCatalog(packDir).filter((e) => {
|
|
644
|
+
const g = e.guidance || {};
|
|
645
|
+
const text = [e.dataset, e.table, e.description, e.endpoint, g.question, g.rows_to_use].map((x) => (x === undefined || x === null ? "None" : String(x))).join(" ").toLowerCase();
|
|
646
|
+
return words.every((w) => text.includes(w));
|
|
647
|
+
});
|
|
648
|
+
const k = (e) => ["status" in e ? 1 : 0, e.guidance ? 0 : 1, e.endpoint];
|
|
649
|
+
return hits.sort((a, b) => {
|
|
650
|
+
const x = k(a), y = k(b);
|
|
651
|
+
return x[0] - y[0] || x[1] - y[1] || cmpStr(x[2], y[2]);
|
|
652
|
+
});
|
|
653
|
+
}
|
|
654
|
+
/** Search the live dataset catalog (all words must match); fall back to the bundled catalog.json. */
|
|
655
|
+
export async function searchCatalog(packDir, term) {
|
|
656
|
+
const words = term.toLowerCase().split(/\s+/).filter(Boolean);
|
|
657
|
+
try {
|
|
658
|
+
const raw = await new FiscalDataClient(60, 1).getRaw(CATALOG_URL);
|
|
659
|
+
const cat = JSON.parse(raw.toString("utf8"));
|
|
660
|
+
const guided = Object.fromEntries(loadCatalog(packDir).filter((e) => e.guidance).map((e) => [e.endpoint, e.guidance]));
|
|
661
|
+
const hits = [];
|
|
662
|
+
for (const ds of Array.isArray(cat) ? cat : cat.data || []) {
|
|
663
|
+
for (const api of ds.apis || []) {
|
|
664
|
+
if (!api.endpoint_txt)
|
|
665
|
+
continue;
|
|
666
|
+
const text = [ds.title, ds.short_description, api.table_name, api.table_description, api.endpoint_txt]
|
|
667
|
+
.map((x) => (x === undefined || x === null ? "None" : String(x))).join(" ").toLowerCase();
|
|
668
|
+
if (words.every((w) => text.includes(w))) {
|
|
669
|
+
const ep = String(api.endpoint_txt).split("/fiscal_service/").pop().replace(/^\/+|\/+$/g, "");
|
|
670
|
+
hits.push({ dataset: ds.title, table: api.table_name, endpoint: ep, frequency: api.update_frequency,
|
|
671
|
+
earliest: api.earliest_date, latest: api.latest_date, rows: api.row_count,
|
|
672
|
+
page: `https://fiscaldata.treasury.gov/datasets/${String(ds.dataset_path || "").replace(/^\/+|\/+$/g, "")}/`,
|
|
673
|
+
description: String(api.table_description || "").slice(0, 240), ...(ep in guided ? { guidance: guided[ep] } : {}) });
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
hits.sort((a, b) => (a.guidance ? 0 : 1) - (b.guidance ? 0 : 1));
|
|
678
|
+
return [hits, "live catalog"];
|
|
679
|
+
}
|
|
680
|
+
catch (e) {
|
|
681
|
+
if (!(e instanceof DataError) && !(e instanceof SyntaxError) && !(e instanceof TypeError))
|
|
682
|
+
throw e;
|
|
683
|
+
}
|
|
684
|
+
return [searchBundled(packDir, words), "bundled catalog.json (snapshot; confirm with the endpoint's fields)"];
|
|
685
|
+
}
|
|
686
|
+
//# sourceMappingURL=_treasury_lib.js.map
|