data-analyst-mcp 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +262 -3
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +710 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +21 -0
- package/dist/report/scan_secrets.js +234 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +246 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +56 -6
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +165 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,1274 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The bea deep pack's fetch (port of sources/bea/fetch.py): spec -> requests, the API route,
|
|
3
|
+
* BEA's public bulk files (NIPA flat files, Regional zips, GDP-by-industry workbooks), and one tidy CSV.
|
|
4
|
+
*
|
|
5
|
+
* Output (<workspace>/data/bea/):
|
|
6
|
+
* bea_data.csv tidy long format, one row per series x area x period (COLUMNS); BEA's non-numeric cells
|
|
7
|
+
* ((D), (NA), (NM), (L) ...) leave value empty and keep the code in flag
|
|
8
|
+
* notes.csv request_id, table, note_ref, note_text (BEA's table notes)
|
|
9
|
+
* request.json the spec, the exact API calls (key removed) or bulk files, route, vintage, checksums
|
|
10
|
+
* README.md what is in the file, units, special values, citation and source line
|
|
11
|
+
* raw/ the API responses (key removed)
|
|
12
|
+
* Bulk files are cached in <cache_root>/bea/bulk and revalidated by Last-Modified.
|
|
13
|
+
*/
|
|
14
|
+
import { createHash } from "node:crypto";
|
|
15
|
+
import fs from "node:fs";
|
|
16
|
+
import path from "node:path";
|
|
17
|
+
import { trail } from "../core/context.js";
|
|
18
|
+
import { cacheRoot, mode } from "../core/env.js";
|
|
19
|
+
import { Client, DataError } from "../core/http.js";
|
|
20
|
+
import { dataDir } from "../core/package.js";
|
|
21
|
+
import * as pol from "../core/policy.js";
|
|
22
|
+
import { pyFloatRepr, pyJsonDumps, pyQuote, pyStr } from "../core/py.js";
|
|
23
|
+
import { unzip, writeCsv, xlsxSheets } from "../core/tabular.js";
|
|
24
|
+
import { BeaClient, BeaError, DATASETS, GUID, UA, datasetProblems, findApiKey, firstError, policy, redact, scrubResponse, valueKey } from "./_bea_api.js";
|
|
25
|
+
import { byCodePoint, csvRows, decodeSigLatin1, htmlUnescape, pyFloat, pyS, truthy } from "./_bea_util.js";
|
|
26
|
+
export const SOURCE = "bea";
|
|
27
|
+
export const COLUMNS = ["request_id", "dataset", "table", "line", "series_code", "description", "geo_fips", "geo_name",
|
|
28
|
+
"geo_level", "industry", "time_period", "date", "year", "subperiod", "frequency", "value", "flag", "unit", "metric",
|
|
29
|
+
"unit_mult", "note_ref", "dims"];
|
|
30
|
+
export const NOTE_COLUMNS = ["request_id", "table", "note_ref", "note_text"];
|
|
31
|
+
export const NIPA_TXT = "https://apps.bea.gov/national/Release/TXT/";
|
|
32
|
+
export const MONTHLY_TITLE = /\bmonth/i;
|
|
33
|
+
export const REGIONAL_ZIP = "https://apps.bea.gov/regional/zip/";
|
|
34
|
+
export const INDUSTRY_XLS = "https://apps.bea.gov/industry/Release/XLS/GDPxInd/";
|
|
35
|
+
export const WORKBOOKS = { TVA: "ValueAdded.xlsx", TGO: "GrossOutput.xlsx", TII: "IntermediateInputs.xlsx" };
|
|
36
|
+
export const WORKBOOK_TABLE = /^T(VA|GO|II)\d+$/i;
|
|
37
|
+
const GUID_RE = new RegExp(GUID, "g");
|
|
38
|
+
export const ZIP_ALIASES = { SARPI: ["SARPP"], SAIRPD: ["SARPP"], MAIRPD: ["MARPP"], SAOVA: ["SAOR"],
|
|
39
|
+
SAORSA: ["SAOR"] };
|
|
40
|
+
export const NIPA_LIKE = ["NIPA", "NIUnderlyingDetail", "FixedAssets"];
|
|
41
|
+
const GEO_KEYWORDS = ["STATE", "COUNTY", "MSA", "MIC", "CSA", "PORT", "DIV", "TERR"];
|
|
42
|
+
export const STATE_FIPS = { AL: "01", AK: "02", AZ: "04", AR: "05", CA: "06", CO: "08", CT: "09",
|
|
43
|
+
DE: "10", DC: "11", FL: "12", GA: "13", HI: "15", ID: "16", IL: "17", IN: "18", IA: "19", KS: "20", KY: "21", LA: "22",
|
|
44
|
+
ME: "23", MD: "24", MA: "25", MI: "26", MN: "27", MS: "28", MO: "29", MT: "30", NE: "31", NV: "32", NH: "33", NJ: "34",
|
|
45
|
+
NM: "35", NY: "36", NC: "37", ND: "38", OH: "39", OK: "40", OR: "41", PA: "42", RI: "44", SC: "45", SD: "46", TN: "47",
|
|
46
|
+
TX: "48", UT: "49", VT: "50", VA: "51", WA: "53", WV: "54", WI: "55", WY: "56" };
|
|
47
|
+
export const SPEC_KEYS = new Set(["id", "dataset", "table", "frequency", "years", "line_code", "geo_fips", "params", "description"]);
|
|
48
|
+
const CITATION = "U.S. Bureau of Economic Analysis, @TABLES@, @ROUTE@ (accessed @DATE@). BEA data are in the public domain; " +
|
|
49
|
+
"cite BEA as the source.";
|
|
50
|
+
export const API_NOTICE = "This product uses the Bureau of Economic Analysis (BEA) Data API but is not endorsed or certified by BEA.";
|
|
51
|
+
const FLAG_TEXT = { "(D)": "suppressed to avoid disclosure of confidential information",
|
|
52
|
+
"(NA)": "not available", "(NM)": "not meaningful", "(L)": "less than $50,000 or less than 10 jobs (Regional) - the " +
|
|
53
|
+
"estimate is included in higher-level totals", "(S)": "suppressed: does not meet publication standards",
|
|
54
|
+
"(X)": "not applicable", "---": "not available / not applicable", "(T)": "suppressed (see the table notes)",
|
|
55
|
+
"(*)": "MNE: a nonzero value that rounds to zero", "n.s.": "MNE: not shown", "n.a.": "not available" };
|
|
56
|
+
/** A spec the pack cannot read (Python's ValueError): reported as fix_spec. */
|
|
57
|
+
export class SpecError extends DataError {
|
|
58
|
+
constructor(msg) {
|
|
59
|
+
super(msg, "spec");
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
// --------------------------------------------------------------------------- spec -> requests
|
|
63
|
+
/** '2019-2025' / [2019, 2020] / 'ALL' / 'LAST5' -> the API's Year value. */
|
|
64
|
+
export function yearsParam(years, dataset) {
|
|
65
|
+
if (years === null || years === undefined || ["ALL", "X", "*"].includes(pyS(years).toUpperCase())) {
|
|
66
|
+
return NIPA_LIKE.includes(dataset) ? "X" : "ALL";
|
|
67
|
+
}
|
|
68
|
+
if (Array.isArray(years)) {
|
|
69
|
+
return years.map((y) => {
|
|
70
|
+
const n = typeof y === "number" ? Math.trunc(y) : /^\s*[+-]?\d+\s*$/.test(pyS(y)) ? parseInt(pyS(y), 10) : NaN;
|
|
71
|
+
if (!Number.isFinite(n))
|
|
72
|
+
throw new SpecError(`invalid literal for int() with base 10: ${JSON.stringify(pyS(y))}`);
|
|
73
|
+
return String(n);
|
|
74
|
+
}).join(",");
|
|
75
|
+
}
|
|
76
|
+
const s = pyS(years).trim();
|
|
77
|
+
if (s.toUpperCase().startsWith("LAST"))
|
|
78
|
+
return s.toUpperCase();
|
|
79
|
+
const out = [];
|
|
80
|
+
for (let part of s.split(",")) {
|
|
81
|
+
part = part.trim();
|
|
82
|
+
const m = /^(\d{4})\s*[-:–]\s*(\d{4})$/.exec(part);
|
|
83
|
+
if (m)
|
|
84
|
+
for (let y = Number(m[1]); y <= Number(m[2]); y++)
|
|
85
|
+
out.push(String(y));
|
|
86
|
+
else if (/^\d{4}$/.test(part))
|
|
87
|
+
out.push(part);
|
|
88
|
+
else if (part)
|
|
89
|
+
throw new SpecError(`years: cannot read '${part}'; use '2019-2025', a list of years, 'ALL' or 'LAST5'`);
|
|
90
|
+
}
|
|
91
|
+
return out.join(",");
|
|
92
|
+
}
|
|
93
|
+
/** The years an API Year value stands for (null = all; LASTn handled by the caller). */
|
|
94
|
+
export function yearSet(v) {
|
|
95
|
+
const u = v.toUpperCase();
|
|
96
|
+
if (u === "X" || u === "ALL" || u.startsWith("LAST"))
|
|
97
|
+
return null;
|
|
98
|
+
return new Set(v.split(",").filter(Boolean).map((y) => parseInt(y, 10)));
|
|
99
|
+
}
|
|
100
|
+
/** Spec geo_fips -> the API's GeoFips: keywords and postal codes upper-cased, codes padded. */
|
|
101
|
+
export function normGeo(geo) {
|
|
102
|
+
const items = Array.isArray(geo) ? geo : pyS(geo).split(",");
|
|
103
|
+
const out = [];
|
|
104
|
+
for (const g of items) {
|
|
105
|
+
let s = pyS(g).trim();
|
|
106
|
+
if (/^\d{1,2}$/.test(s))
|
|
107
|
+
s = s.padStart(2, "0") + "000";
|
|
108
|
+
else if (/^\d{3,5}$/.test(s))
|
|
109
|
+
s = s.padStart(5, "0");
|
|
110
|
+
else
|
|
111
|
+
s = s.toUpperCase();
|
|
112
|
+
if (s)
|
|
113
|
+
out.push(s);
|
|
114
|
+
}
|
|
115
|
+
return out.join(",");
|
|
116
|
+
}
|
|
117
|
+
const get = (o, k, d) => (k in o ? o[k] : d);
|
|
118
|
+
/** Each request -> {id, dataset, table, frequency, years, line_codes, geo, calls: [api params], bulk_only}. */
|
|
119
|
+
export function normalizeRequests(spec, notes = []) {
|
|
120
|
+
const reqs = truthy(spec?.requests) ? spec.requests : spec?.dataset ? [spec] : [];
|
|
121
|
+
if (!reqs.length || !Array.isArray(reqs))
|
|
122
|
+
throw new SpecError("spec.json needs a 'requests' list");
|
|
123
|
+
const out = [];
|
|
124
|
+
const seen = new Set();
|
|
125
|
+
reqs.forEach((r, i0) => {
|
|
126
|
+
const i = i0 + 1;
|
|
127
|
+
const ds = truthy(r.dataset) ? pyS(r.dataset) : "NIPA";
|
|
128
|
+
const raw = {};
|
|
129
|
+
for (const [k, v] of Object.entries(r.params || {}))
|
|
130
|
+
raw[k] = pyS(v);
|
|
131
|
+
const table = truthy(r.table) ? pyS(r.table) : raw.TableName || raw.TableID || "";
|
|
132
|
+
const freq = (truthy(r.frequency) ? pyS(r.frequency) : raw.Frequency || "").toUpperCase();
|
|
133
|
+
const yv = raw.Year || yearsParam(r.years, ds);
|
|
134
|
+
const apiYear = ds !== "Regional" && yv.toUpperCase().startsWith("LAST") ? (NIPA_LIKE.includes(ds) ? "X" : "ALL") : yv;
|
|
135
|
+
const base0 = truthy(r.id) ? pyS(r.id) : `${ds}_${table}_${freq}`.replace(/^_+|_+$/g, "");
|
|
136
|
+
let rid = base0.replace(/[^\p{L}\p{N}_]+/gu, "_").replace(/^_+|_+$/g, "").toLowerCase();
|
|
137
|
+
if (seen.has(rid))
|
|
138
|
+
rid = `${rid}_${i}`;
|
|
139
|
+
seen.add(rid);
|
|
140
|
+
const workbook = ds === "GDPbyIndustry" && WORKBOOK_TABLE.test(table || "");
|
|
141
|
+
const ignored = new Set(Object.keys(r).filter((k) => !SPEC_KEYS.has(k)));
|
|
142
|
+
if ("geo_fips" in r && ds !== "Regional")
|
|
143
|
+
ignored.add("geo_fips");
|
|
144
|
+
if ("line_code" in r && ![...NIPA_LIKE, "Regional"].includes(ds))
|
|
145
|
+
ignored.add("line_code");
|
|
146
|
+
const friendly = NIPA_LIKE.includes(ds) || ds === "Regional" || workbook;
|
|
147
|
+
if (!friendly)
|
|
148
|
+
for (const k of ["frequency", "years"])
|
|
149
|
+
if (k in r)
|
|
150
|
+
ignored.add(k);
|
|
151
|
+
if (ignored.size) {
|
|
152
|
+
notes.push(`request ${rid}: ${[...ignored].sort(byCodePoint).join(", ")} not used for ${ds}` +
|
|
153
|
+
(!friendly ? ' (pass API parameters in "params")' : "") + "; see request_schema for the spec keys");
|
|
154
|
+
}
|
|
155
|
+
let base = {};
|
|
156
|
+
let lineCodes = [];
|
|
157
|
+
let geo = null;
|
|
158
|
+
if (NIPA_LIKE.includes(ds)) {
|
|
159
|
+
base = { TableName: table, Year: apiYear };
|
|
160
|
+
if (ds !== "FixedAssets")
|
|
161
|
+
base.Frequency = freq || "A";
|
|
162
|
+
const lc = r.line_code;
|
|
163
|
+
if (lc !== null && lc !== undefined && pyS(lc).toUpperCase() !== "ALL")
|
|
164
|
+
lineCodes = Array.isArray(lc) ? lc.map(pyS) : [pyS(lc)];
|
|
165
|
+
}
|
|
166
|
+
else if (workbook) {
|
|
167
|
+
base = { Table: table.toUpperCase(), Frequency: freq || "A", Year: yv };
|
|
168
|
+
}
|
|
169
|
+
else if (ds === "Regional") {
|
|
170
|
+
const lc = get(r, "line_code", get(raw, "LineCode", "1"));
|
|
171
|
+
lineCodes = Array.isArray(lc) ? lc.map(pyS) : [pyS(lc)];
|
|
172
|
+
geo = normGeo(get(r, "geo_fips", get(raw, "GeoFips", "STATE")));
|
|
173
|
+
if ("GeoFips" in raw)
|
|
174
|
+
raw.GeoFips = normGeo(raw.GeoFips);
|
|
175
|
+
base = { TableName: table, GeoFips: geo, Year: yv };
|
|
176
|
+
}
|
|
177
|
+
Object.assign(base, raw);
|
|
178
|
+
let calls;
|
|
179
|
+
if (ds === "Regional") {
|
|
180
|
+
delete base.LineCode;
|
|
181
|
+
calls = lineCodes.map((lc) => ({ ...base, LineCode: lc }));
|
|
182
|
+
}
|
|
183
|
+
else
|
|
184
|
+
calls = [base];
|
|
185
|
+
out.push({ id: rid, dataset: ds, table, frequency: freq || base.Frequency || "", years: yv, line_codes: lineCodes, geo,
|
|
186
|
+
calls, bulk_only: ds === "GDPbyIndustry" && WORKBOOK_TABLE.test(table || "") });
|
|
187
|
+
});
|
|
188
|
+
return out;
|
|
189
|
+
}
|
|
190
|
+
/** Requests BEA would refuse (every refusal counts against 30 errors a minute): check them first. */
|
|
191
|
+
export function specProblems(reqs) {
|
|
192
|
+
const probs = [];
|
|
193
|
+
for (const q of reqs) {
|
|
194
|
+
if (q.bulk_only)
|
|
195
|
+
continue;
|
|
196
|
+
for (const call of q.calls)
|
|
197
|
+
probs.push(...datasetProblems(q.dataset, call).map((p) => `request ${q.id}: ${p}`));
|
|
198
|
+
}
|
|
199
|
+
return [...new Set(probs)];
|
|
200
|
+
}
|
|
201
|
+
const stopKinds = (e) => e instanceof BeaError && ["auth", "rate_limit", "budget"].includes(e.bea);
|
|
202
|
+
/** Check table names (and Regional line codes) against BEA's cached metadata before GetData. */
|
|
203
|
+
export async function validateWithMetadata(client, reqs) {
|
|
204
|
+
const probs = [];
|
|
205
|
+
const names = new Map();
|
|
206
|
+
for (const q of reqs) {
|
|
207
|
+
const ds = q.dataset;
|
|
208
|
+
if (q.bulk_only || ![...NIPA_LIKE, "Regional"].includes(ds) || !q.table)
|
|
209
|
+
continue;
|
|
210
|
+
if (!names.has(ds)) {
|
|
211
|
+
try {
|
|
212
|
+
const s = new Set((await client.parameterValues(ds, "TableName")).map((v) => valueKey(v).toUpperCase()));
|
|
213
|
+
names.set(ds, s.size ? s : null);
|
|
214
|
+
}
|
|
215
|
+
catch (e) {
|
|
216
|
+
if (!(e instanceof DataError))
|
|
217
|
+
throw e;
|
|
218
|
+
if (stopKinds(e))
|
|
219
|
+
throw e;
|
|
220
|
+
names.set(ds, null);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
const known = names.get(ds);
|
|
224
|
+
const t = q.table.toUpperCase();
|
|
225
|
+
if (known && !known.has(t) && !((ds === "Regional" && known.has(t + "N")) || known.has(t.replace(/N+$/, "")))) {
|
|
226
|
+
const near = [...known].filter((k) => k.slice(0, 4) === t.slice(0, 4)).sort(byCodePoint).slice(0, 8);
|
|
227
|
+
probs.push(`request ${q.id}: ${ds} has no table ${q.table}` + (near.length ? ` (similar: ${near.join(", ")})` : "") +
|
|
228
|
+
`; list them with request_schema search "${ds} <words>"`);
|
|
229
|
+
continue;
|
|
230
|
+
}
|
|
231
|
+
if (ds === "Regional" && q.line_codes.length && !q.line_codes.map((x) => x.toUpperCase()).includes("ALL")) {
|
|
232
|
+
let lines = new Set();
|
|
233
|
+
try {
|
|
234
|
+
lines = new Set((await client.parameterValuesFiltered("Regional", "LineCode", { TableName: q.table })).map(valueKey));
|
|
235
|
+
}
|
|
236
|
+
catch (e) {
|
|
237
|
+
if (!(e instanceof DataError))
|
|
238
|
+
throw e;
|
|
239
|
+
if (stopKinds(e))
|
|
240
|
+
throw e;
|
|
241
|
+
}
|
|
242
|
+
const bad = q.line_codes.filter((lc) => lines.size && !lines.has(lc));
|
|
243
|
+
if (bad.length) {
|
|
244
|
+
probs.push(`request ${q.id}: ${q.table} has no line ${bad.join(", ")} (request_schema search "values Regional LineCode ` +
|
|
245
|
+
`TableName=${q.table}")`);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
return probs;
|
|
250
|
+
}
|
|
251
|
+
export function fingerprint(reqs) {
|
|
252
|
+
const parts = [...reqs.map((r) => r.calls), reqs.map((r) => r.dataset)];
|
|
253
|
+
const filters = reqs.filter((r) => NIPA_LIKE.includes(r.dataset) && r.line_codes.length).map((r) => [r.id, r.line_codes]);
|
|
254
|
+
if (filters.length)
|
|
255
|
+
parts.push(filters);
|
|
256
|
+
return createHash("sha256").update(pyJsonDumps(parts, true)).digest("hex").slice(0, 16);
|
|
257
|
+
}
|
|
258
|
+
// --------------------------------------------------------------------------- shared helpers
|
|
259
|
+
/** [number, ""] for a numeric DataValue ("1,337" -> 1337); [null, the published code] for BEA's special values. */
|
|
260
|
+
export function parseValue(v) {
|
|
261
|
+
const s = pyStr(v ?? "").trim();
|
|
262
|
+
const n = pyFloat(s.replaceAll(",", ""));
|
|
263
|
+
return n === null ? [null, s] : [n, ""];
|
|
264
|
+
}
|
|
265
|
+
const valueText = (n) => (n === null ? "" : pyFloatRepr(n));
|
|
266
|
+
/** '2024' / '2024Q1' / '2024:Q1' / '2024M01' -> [time_period, year, subperiod, frequency]. */
|
|
267
|
+
export function parsePeriod(tp, freq = "") {
|
|
268
|
+
let s = pyStr(tp).trim().replaceAll(":", "");
|
|
269
|
+
const m = /^(\d{4})(Q[1-4]|M\d{1,2})?$/.exec(s);
|
|
270
|
+
if (!m)
|
|
271
|
+
return [s, null, "", freq ?? ""];
|
|
272
|
+
let sub = m[2] || "";
|
|
273
|
+
if (sub.startsWith("M") && sub.length === 2) {
|
|
274
|
+
sub = `M${String(parseInt(sub.slice(1), 10)).padStart(2, "0")}`;
|
|
275
|
+
s = m[1] + sub;
|
|
276
|
+
}
|
|
277
|
+
const f = sub.startsWith("Q") ? "Q" : sub.startsWith("M") ? "M" : freq || "A";
|
|
278
|
+
return [s, parseInt(m[1], 10), sub, f];
|
|
279
|
+
}
|
|
280
|
+
/** nation, region, state, county, or the requested area type (msa, port ...). */
|
|
281
|
+
export function geoLevel(fips, requested) {
|
|
282
|
+
const req = (requested || "").toUpperCase();
|
|
283
|
+
const f = pyStr(fips).trim().replace(/^"+|"+$/g, "");
|
|
284
|
+
if (f === "00000" || f === "00")
|
|
285
|
+
return "nation";
|
|
286
|
+
if (["MSA", "MIC", "CSA", "PORT", "DIV"].includes(req))
|
|
287
|
+
return req.toLowerCase();
|
|
288
|
+
if (/^9[1-8]000$/.test(f))
|
|
289
|
+
return "region";
|
|
290
|
+
if (/^\d{2}000$/.test(f))
|
|
291
|
+
return "state";
|
|
292
|
+
if (/^\d{5}$/.test(f))
|
|
293
|
+
return "county";
|
|
294
|
+
return "";
|
|
295
|
+
}
|
|
296
|
+
/** A GDP-by-industry table's combined unit narrowed by the section the line sits in. */
|
|
297
|
+
export function sectionUnit(unit, section) {
|
|
298
|
+
const sec = (section || "").toLowerCase();
|
|
299
|
+
if ((unit || "").toLowerCase().includes(" and ")) {
|
|
300
|
+
if (sec.startsWith("percentage point"))
|
|
301
|
+
return "Percentage points";
|
|
302
|
+
if (sec.startsWith("percent"))
|
|
303
|
+
return "Percent";
|
|
304
|
+
}
|
|
305
|
+
return unit;
|
|
306
|
+
}
|
|
307
|
+
/** BEA's unit for a NIPA level is just "Level": name it from the metric and the scale. */
|
|
308
|
+
export function plainUnit(unit, metric, unitMult) {
|
|
309
|
+
const u = (unit || "").trim().toLowerCase();
|
|
310
|
+
if (u !== "level" && u !== "level change")
|
|
311
|
+
return unit;
|
|
312
|
+
const scale = { "3": "thousands of ", "6": "millions of ", "9": "billions of " }[pyStr(unitMult).trim()] ?? "";
|
|
313
|
+
const m = (metric || "").toLowerCase();
|
|
314
|
+
const rate = m.includes("period rate") ? "not at annual rates (period rate)" : "quarterly and monthly values at annual rates";
|
|
315
|
+
const change = u === "level change" ? "change in " : "";
|
|
316
|
+
if (m.includes("contribution"))
|
|
317
|
+
return "percentage points (contribution to the percent change; quarterly at an annual rate)";
|
|
318
|
+
if (m.includes("share"))
|
|
319
|
+
return "percent (a share of the total)";
|
|
320
|
+
if (m.includes("ratio"))
|
|
321
|
+
return "a ratio: percent or dollars per person, as the line description says";
|
|
322
|
+
if (m.includes("ipd"))
|
|
323
|
+
return "implicit price deflator, reference year = 100";
|
|
324
|
+
if (m.includes("index"))
|
|
325
|
+
return "index, reference year = 100";
|
|
326
|
+
if (m.includes("chained"))
|
|
327
|
+
return `${change}${scale}chained dollars (reference year in the table title); ${rate}`;
|
|
328
|
+
if (m.includes("dollar"))
|
|
329
|
+
return `${change}${scale}current dollars; ${rate}`;
|
|
330
|
+
if (m.includes("person"))
|
|
331
|
+
return `${scale}persons`;
|
|
332
|
+
return metric ? `${scale}${metric.toLowerCase()} (see the line description)` : unit;
|
|
333
|
+
}
|
|
334
|
+
export function unitMultFromText(unit) {
|
|
335
|
+
const u = unit.toLowerCase();
|
|
336
|
+
for (const [word, p] of [["trillion", 12], ["billion", 9], ["million", 6], ["thousand", 3]])
|
|
337
|
+
if (u.includes(word))
|
|
338
|
+
return p;
|
|
339
|
+
return 0;
|
|
340
|
+
}
|
|
341
|
+
/** 'Hawaii *' -> 'Hawaii' (BEA marks areas with a footnote by a trailing star). */
|
|
342
|
+
export function cleanGeoName(name) {
|
|
343
|
+
return pyStr(name || "").replace(/\s*\*+\s*$/, "").trim();
|
|
344
|
+
}
|
|
345
|
+
const sha256File = (p) => createHash("sha256").update(fs.readFileSync(p)).digest("hex");
|
|
346
|
+
const pad2 = (n) => String(n).padStart(2, "0");
|
|
347
|
+
function localIso(d = new Date()) {
|
|
348
|
+
return `${d.getFullYear()}-${pad2(d.getMonth() + 1)}-${pad2(d.getDate())}T${pad2(d.getHours())}:${pad2(d.getMinutes())}:${pad2(d.getSeconds())}`;
|
|
349
|
+
}
|
|
350
|
+
const today = () => localIso().slice(0, 10);
|
|
351
|
+
const utcIso = () => new Date().toISOString().replace(/\.\d{3}Z$/, "+00:00");
|
|
352
|
+
const MONTHS = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"];
|
|
353
|
+
const longDate = (d = new Date()) => `${MONTHS[d.getMonth()]} ${d.getDate()}, ${d.getFullYear()}`;
|
|
354
|
+
const fmtInt = (n) => n.toLocaleString("en-US");
|
|
355
|
+
// --------------------------------------------------------------------------- API rows -> tidy rows
|
|
356
|
+
const USED = new Set(["TableName", "SeriesCode", "LineNumber", "LineDescription", "TimePeriod", "METRIC_NAME", "CL_UNIT",
|
|
357
|
+
"UNIT_MULT", "DataValue", "NoteRef", "Code", "GeoFips", "GeoName", "TableID", "Frequency", "Year", "Quarter", "Industry",
|
|
358
|
+
"IndustrYDescription", "IndustryDescription", "TimeSeriesId", "TimeSeriesDescription", "AreaOrCountry", "Indicator"]);
|
|
359
|
+
const or = (...xs) => xs.find((x) => truthy(x)) ?? xs[xs.length - 1];
|
|
360
|
+
export function tidyApiRows(req, call, rows) {
|
|
361
|
+
const ds = req.dataset;
|
|
362
|
+
const out = [];
|
|
363
|
+
for (const r of rows) {
|
|
364
|
+
const [value, flag] = parseValue(r.DataValue);
|
|
365
|
+
let tp = r.TimePeriod;
|
|
366
|
+
if (!truthy(tp) && truthy(r.Year)) {
|
|
367
|
+
let q = pyStr(r.Quarter || "").trim().toUpperCase().replace(/^Q+/, "");
|
|
368
|
+
q = { I: "1", II: "2", III: "3", IV: "4" }[q] ?? q;
|
|
369
|
+
tp = ["1", "2", "3", "4"].includes(q) ? `${pyStr(r.Year)}Q${q}` : pyStr(r.Year);
|
|
370
|
+
}
|
|
371
|
+
const [tp2, year, sub, freq] = parsePeriod(truthy(tp) ? tp : "", or(req.frequency, get(r, "Frequency", "")));
|
|
372
|
+
let table = or(r.TableName, r.TableID, req.table);
|
|
373
|
+
let line = get(r, "LineNumber", "");
|
|
374
|
+
if (truthy(r.Code) && pyStr(r.Code).includes("-")) {
|
|
375
|
+
const c = pyStr(r.Code);
|
|
376
|
+
const k = c.lastIndexOf("-");
|
|
377
|
+
[table, line] = [c.slice(0, k), c.slice(k + 1)];
|
|
378
|
+
}
|
|
379
|
+
else if (ds === "Regional")
|
|
380
|
+
line = get(call, "LineCode", "");
|
|
381
|
+
const dims = {};
|
|
382
|
+
for (const [k, v] of Object.entries(r))
|
|
383
|
+
if (!USED.has(k) && v !== null && v !== undefined && v !== "")
|
|
384
|
+
dims[k] = v;
|
|
385
|
+
const reqGeo = req.geo ?? null;
|
|
386
|
+
const lvl = truthy(r.GeoFips)
|
|
387
|
+
? geoLevel(r.GeoFips, pyStr(req.table).toUpperCase().startsWith("MA") && !GEO_KEYWORDS.includes((reqGeo || "").toUpperCase())
|
|
388
|
+
? "MSA" : reqGeo)
|
|
389
|
+
: [...NIPA_LIKE, "GDPbyIndustry", "UnderlyingGDPbyIndustry"].includes(ds) ? "nation" : "";
|
|
390
|
+
out.push({
|
|
391
|
+
request_id: req.id, dataset: ds, table, line,
|
|
392
|
+
series_code: or(r.SeriesCode, r.TimeSeriesId, ""),
|
|
393
|
+
description: or(r.LineDescription, r.TimeSeriesDescription, r.IndustrYDescription, r.IndustryDescription, r.Indicator, ""),
|
|
394
|
+
geo_fips: get(r, "GeoFips", ""), geo_name: cleanGeoName(or(r.GeoName, r.AreaOrCountry, "")), geo_level: lvl,
|
|
395
|
+
industry: get(r, "Industry", ""), time_period: tp2, year: year ?? "", subperiod: sub, frequency: freq,
|
|
396
|
+
value: valueText(value), flag, unit: get(r, "CL_UNIT", ""), metric: get(r, "METRIC_NAME", ""),
|
|
397
|
+
unit_mult: get(r, "UNIT_MULT", ""), note_ref: get(r, "NoteRef", ""),
|
|
398
|
+
dims: Object.keys(dims).length ? pyJsonDumps(dims, false, false) : "",
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
return out;
|
|
402
|
+
}
|
|
403
|
+
export function tidyNotes(req, notes) {
|
|
404
|
+
return (notes || []).filter((n) => !!n && typeof n === "object" && !Array.isArray(n)).map((n) => ({
|
|
405
|
+
request_id: req.id, table: req.table, note_ref: get(n, "NoteRef", ""), note_text: redact(pyStr(get(n, "NoteText", "")))
|
|
406
|
+
}));
|
|
407
|
+
}
|
|
408
|
+
// --------------------------------------------------------------------------- bulk files
|
|
409
|
+
let BULK = null;
|
|
410
|
+
function bulkClient() {
|
|
411
|
+
return (BULK ??= new Client("", { name: "bea-bulk", rps: 4, burst: 4, retries: 2, timeout: 300, deadline: 1800, userAgent: UA }));
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Where bulk files are kept between runs: <cache_root>/bea/bulk (DATA_ANALYST_CACHE sets the root). On a hosted
|
|
415
|
+
* server without an operator cache, inside the session's workspace instead (nothing is shared across users).
|
|
416
|
+
*/
|
|
417
|
+
export function cacheDir(workspace) {
|
|
418
|
+
pol.checkCache(policy(), "persistent");
|
|
419
|
+
const root = mode() === "remote" && !process.env.DATA_ANALYST_CACHE && workspace ? path.join(path.resolve(workspace), ".cache")
|
|
420
|
+
: cacheRoot();
|
|
421
|
+
const d = path.join(root, "bea", "bulk");
|
|
422
|
+
fs.mkdirSync(d, { recursive: true });
|
|
423
|
+
return d;
|
|
424
|
+
}
|
|
425
|
+
async function httpGet(url, dest) {
|
|
426
|
+
let r;
|
|
427
|
+
try {
|
|
428
|
+
r = await bulkClient().get(url);
|
|
429
|
+
}
|
|
430
|
+
catch (e) {
|
|
431
|
+
if (!(e instanceof DataError))
|
|
432
|
+
throw e;
|
|
433
|
+
throw new BeaError(`${url}: ${e.message}`, e.kind === "network" ? "network" : ["not_found", "fixture_missing"].includes(e.kind)
|
|
434
|
+
? "api" : "http");
|
|
435
|
+
}
|
|
436
|
+
const tmp = dest + ".part";
|
|
437
|
+
fs.writeFileSync(tmp, r.body);
|
|
438
|
+
fs.renameSync(tmp, dest);
|
|
439
|
+
return { url, last_modified: r.headers["last-modified"] ?? "", status: r.status };
|
|
440
|
+
}
|
|
441
|
+
async function head(url) {
|
|
442
|
+
if (process.env.DATA_ANALYST_FIXTURES)
|
|
443
|
+
return {};
|
|
444
|
+
try {
|
|
445
|
+
const r = await bulkClient().request("HEAD", url);
|
|
446
|
+
return { status: r.status, last_modified: r.headers["last-modified"] ?? "", length: Number(r.headers["content-length"] || 0),
|
|
447
|
+
type: r.headers["content-type"] ?? "" };
|
|
448
|
+
}
|
|
449
|
+
catch (e) {
|
|
450
|
+
if (e instanceof DataError)
|
|
451
|
+
return {};
|
|
452
|
+
throw e;
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
/** Download a bulk file into the cache, reusing a copy whose Last-Modified has not changed. */
|
|
456
|
+
export async function fetchSource(url, srcDir, manifest) {
|
|
457
|
+
fs.mkdirSync(srcDir, { recursive: true });
|
|
458
|
+
const name = url.slice(url.lastIndexOf("/") + 1);
|
|
459
|
+
const dest = path.join(srcDir, name);
|
|
460
|
+
const metaPath = dest + ".meta.json";
|
|
461
|
+
if (fs.existsSync(dest) && fs.existsSync(metaPath)) {
|
|
462
|
+
const old = JSON.parse(fs.readFileSync(metaPath, "utf8"));
|
|
463
|
+
const lm = (await head(url)).last_modified;
|
|
464
|
+
if (!lm || lm === old.last_modified) {
|
|
465
|
+
manifest[name] = old;
|
|
466
|
+
return dest;
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
const meta = await httpGet(url, dest);
|
|
470
|
+
const fd = fs.openSync(dest, "r");
|
|
471
|
+
const magic = Buffer.alloc(4);
|
|
472
|
+
fs.readSync(fd, magic, 0, 4, 0);
|
|
473
|
+
fs.closeSync(fd);
|
|
474
|
+
if (name.endsWith(".zip") && magic.subarray(0, 2).toString("latin1") !== "PK") {
|
|
475
|
+
fs.rmSync(dest);
|
|
476
|
+
throw new BeaError(`${url} is not a zip file (BEA answers a missing bulk file with an HTML page).`, "api");
|
|
477
|
+
}
|
|
478
|
+
Object.assign(meta, { sha256: sha256File(dest), bytes: fs.statSync(dest).size, retrieved: localIso(), path: dest });
|
|
479
|
+
fs.writeFileSync(metaPath, JSON.stringify(meta, null, 1));
|
|
480
|
+
manifest[name] = meta;
|
|
481
|
+
return dest;
|
|
482
|
+
}
|
|
483
|
+
export function readTextMember(p) {
|
|
484
|
+
return decodeSigLatin1(fs.readFileSync(p));
|
|
485
|
+
}
|
|
486
|
+
const isDigits = (s) => /^\d+$/.test(s);
|
|
487
|
+
const cmpRowLine = (a, b) => {
|
|
488
|
+
const la = isDigits(pyStr(a.line)) ? parseInt(a.line, 10) : 0, lb = isDigits(pyStr(b.line)) ? parseInt(b.line, 10) : 0;
|
|
489
|
+
return la - lb || byCodePoint(a.time_period, b.time_period);
|
|
490
|
+
};
|
|
491
|
+
function keepLastN(rows, n) {
|
|
492
|
+
const keep = new Set([...new Set(rows.map((r) => r.year))].sort((a, b) => a - b).slice(-n));
|
|
493
|
+
return rows.filter((r) => keep.has(r.year));
|
|
494
|
+
}
|
|
495
|
+
/** Rebuild a NIPA table from the flat files: SeriesRegister maps TableId:LineNo -> SeriesCode. */
|
|
496
|
+
export async function nipaFromFlat(req, srcDir, manifest, files = {}) {
|
|
497
|
+
const f = req.frequency || "A";
|
|
498
|
+
const reg = files["SeriesRegister.txt"] || await fetchSource(NIPA_TXT + "SeriesRegister.txt", srcDir, manifest);
|
|
499
|
+
let tablesReg = files["TablesRegister.txt"] || null;
|
|
500
|
+
if (!tablesReg) {
|
|
501
|
+
try {
|
|
502
|
+
tablesReg = await fetchSource(NIPA_TXT + "TablesRegister.txt", srcDir, manifest);
|
|
503
|
+
}
|
|
504
|
+
catch (e) {
|
|
505
|
+
if (!(e instanceof BeaError) && !e?.code)
|
|
506
|
+
throw e;
|
|
507
|
+
tablesReg = null;
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
const table = req.table.toUpperCase();
|
|
511
|
+
const titles = {};
|
|
512
|
+
if (tablesReg)
|
|
513
|
+
for (const row of csvRows(readTextMember(tablesReg)))
|
|
514
|
+
if (row.length > 1)
|
|
515
|
+
titles[row[0].toUpperCase()] = row[1];
|
|
516
|
+
const lines = new Map();
|
|
517
|
+
const homes = new Map();
|
|
518
|
+
for (const row of csvRows(readTextMember(reg))) {
|
|
519
|
+
if (!row.length || row[0].startsWith("%") || row.length < 6)
|
|
520
|
+
continue;
|
|
521
|
+
const [code, label, metric, calc, scale, locs] = row;
|
|
522
|
+
homes.set(code, new Set(locs.split("|").filter((x) => x.includes(":")).map((x) => x.split(":")[0].toUpperCase())));
|
|
523
|
+
for (const loc of locs.split("|")) {
|
|
524
|
+
if (loc.includes(":") && loc.split(":")[0].toUpperCase() === table) {
|
|
525
|
+
if (!lines.has(code))
|
|
526
|
+
lines.set(code, []);
|
|
527
|
+
lines.get(code).push({ line: loc.split(":")[1], label, metric, calc,
|
|
528
|
+
unit_mult: /^\d+$/.test(scale.replace(/^-+/, "")) ? String(-parseInt(scale, 10) || 0) : "" });
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
if (!lines.size) {
|
|
533
|
+
throw new BeaError(`NIPA table ${table} is not in SeriesRegister.txt (check the name with request_schema search).`, "api");
|
|
534
|
+
}
|
|
535
|
+
if (f === "M" && table.startsWith("T") && Object.hasOwn(titles, table) && !MONTHLY_TITLE.test(titles[table])) {
|
|
536
|
+
const alt = [...new Set([...lines.keys()].flatMap((c) => [...(homes.get(c) || [])])
|
|
537
|
+
.filter((t) => MONTHLY_TITLE.test(titles[t] ?? "")))].sort(byCodePoint);
|
|
538
|
+
throw new BeaError(`NIPA ${table} (${titles[table]}) is published annually and quarterly, not monthly. ` +
|
|
539
|
+
(alt.length ? `Its series' monthly values are in ${alt.join(", ")}: request that table with frequency M (check its line ` +
|
|
540
|
+
`numbers with request_schema search "lines ${alt[0]}").` : "Use frequency A or Q."), "api");
|
|
541
|
+
}
|
|
542
|
+
const dat = files[`NipaData${f}.txt`] || await fetchSource(NIPA_TXT + `NipaData${f}.txt`, srcDir, manifest);
|
|
543
|
+
const ys = yearSet(req.years);
|
|
544
|
+
const lastn = req.years.toUpperCase().startsWith("LAST") ? parseInt(req.years.slice(4), 10) : null;
|
|
545
|
+
let out = [];
|
|
546
|
+
const seenYears = new Set();
|
|
547
|
+
for (const row of csvRows(readTextMember(dat))) {
|
|
548
|
+
if (!row.length || row[0].startsWith("%") || !lines.has(row[0]) || row.length < 3)
|
|
549
|
+
continue;
|
|
550
|
+
const [tp, year, sub, freq] = parsePeriod(row[1], f);
|
|
551
|
+
seenYears.add(year);
|
|
552
|
+
if (ys !== null && (year === null || !ys.has(year)))
|
|
553
|
+
continue;
|
|
554
|
+
const [value, flag] = parseValue(row[2]);
|
|
555
|
+
for (const info of lines.get(row[0])) {
|
|
556
|
+
out.push({ request_id: req.id, dataset: "NIPA", table, line: info.line, series_code: row[0], description: info.label,
|
|
557
|
+
geo_fips: "", geo_name: "United States", geo_level: "nation", industry: "", time_period: tp, year, subperiod: sub,
|
|
558
|
+
frequency: freq, value: valueText(value), flag, unit: info.calc, metric: info.metric, unit_mult: info.unit_mult,
|
|
559
|
+
note_ref: "", dims: "" });
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
if (!out.length) {
|
|
563
|
+
const have = [...seenYears].filter((y) => y !== null).sort((a, b) => a - b);
|
|
564
|
+
throw new BeaError(`No rows for NIPA ${table}, frequency ${f}, years ${req.years}: ` +
|
|
565
|
+
(have.length ? `NipaData${f}.txt holds ${table} for ${have[0]}-${have[have.length - 1]}.`
|
|
566
|
+
: `${table} is not published at frequency ${f} (try A, Q or M; request_schema search "lines ${table}").`), "api");
|
|
567
|
+
}
|
|
568
|
+
if (lastn)
|
|
569
|
+
out = keepLastN(out, lastn);
|
|
570
|
+
out.sort(cmpRowLine);
|
|
571
|
+
const title = titles[table] ?? "";
|
|
572
|
+
const notes = [{ request_id: req.id, table, note_ref: table, note_text: (title ? title + ". " : "") + "Rebuilt from the NIPA " +
|
|
573
|
+
"flat files (SeriesRegister.txt TableId:LineNo); line descriptions are BEA's series labels, which can differ slightly " +
|
|
574
|
+
"from the table's indented stub." }];
|
|
575
|
+
return [out, notes];
|
|
576
|
+
}
|
|
577
|
+
function regionalCsvRows(text) {
|
|
578
|
+
const rows = csvRows(text);
|
|
579
|
+
const header = rows[0].map((h) => h.trim());
|
|
580
|
+
const data = [], foot = [];
|
|
581
|
+
for (const r of rows.slice(1)) {
|
|
582
|
+
if (r.length >= header.length - 1 && r.length && /^\s*\d{5}\s*$/.test(r[0].trim().replace(/^"+|"+$/g, "") || "x")) {
|
|
583
|
+
data.push(r.map((c) => c.trim()));
|
|
584
|
+
}
|
|
585
|
+
else if (r.length && r.some((c) => c.trim()))
|
|
586
|
+
foot.push(r.map((c) => c.trim()).join(" "));
|
|
587
|
+
}
|
|
588
|
+
return [header, data, foot];
|
|
589
|
+
}
|
|
590
|
+
/** BEA's footnotes for a Regional table, from <TABLE>__Footnotes.html in the zip: [[ref, text]]. */
|
|
591
|
+
function regionalFootnotes(z, names) {
|
|
592
|
+
const want = new Set([...names].map((nm) => `${nm.toLowerCase()}__footnotes.html`));
|
|
593
|
+
const member = Object.keys(z).find((n) => want.has(n.toLowerCase()));
|
|
594
|
+
if (!member)
|
|
595
|
+
return [];
|
|
596
|
+
const text = decodeSigLatin1Strict(z[member]);
|
|
597
|
+
const out = [];
|
|
598
|
+
for (const m of text.matchAll(/<li[^>]*>([\s\S]*?)<\/li>/gi)) {
|
|
599
|
+
const s = htmlUnescape(m[1].replace(/<[^>]+>/g, " ")).replaceAll(" ", " ").replace(/\s+/g, " ").trim();
|
|
600
|
+
if (!s)
|
|
601
|
+
continue;
|
|
602
|
+
const mm = /^(\d+)\.\s*(.*)$|^(\([A-Z]+\))\s*(.*)$/.exec(s);
|
|
603
|
+
out.push(mm ? [mm[1] ?? mm[3], mm[1] !== undefined ? mm[2] : mm[4]] : ["", s]);
|
|
604
|
+
}
|
|
605
|
+
return out;
|
|
606
|
+
}
|
|
607
|
+
/** bytes.decode("utf-8") else latin-1 (no BOM strip: the footnote page is read as Python read it). */
|
|
608
|
+
function decodeSigLatin1Strict(b) {
|
|
609
|
+
try {
|
|
610
|
+
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(b);
|
|
611
|
+
}
|
|
612
|
+
catch {
|
|
613
|
+
return Buffer.from(b).toString("latin1");
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
/** Geographies regionalFromZip can select: STATE, COUNTY, a postal code (its counties), 5-digit codes, any MA* area. */
|
|
617
|
+
export function bulkGeoOk(geo, table) {
|
|
618
|
+
const g = geo.toUpperCase();
|
|
619
|
+
const codes = g.split(",").filter(Boolean);
|
|
620
|
+
const t = table.toUpperCase();
|
|
621
|
+
return t.startsWith("MA") || g === "STATE" || g === "COUNTY" || (Object.hasOwn(STATE_FIPS, g) && t.startsWith("CA")) ||
|
|
622
|
+
(codes.length > 0 && codes.every((c) => /^\d{5}$/.test(c)));
|
|
623
|
+
}
|
|
624
|
+
export function zipCandidates(table) {
|
|
625
|
+
let t = table.toUpperCase();
|
|
626
|
+
const fam = t.replace(/\d.*$/, "");
|
|
627
|
+
const ca = t.startsWith("CA");
|
|
628
|
+
if (!ca)
|
|
629
|
+
t = t.replace(/(?<=\d)N$/, "");
|
|
630
|
+
const c = [...(ca ? [t] : []), fam, ...(ZIP_ALIASES[t] || []), ...(ZIP_ALIASES[fam] || [])];
|
|
631
|
+
if (!ca)
|
|
632
|
+
c.push(t);
|
|
633
|
+
return [...new Set(c)];
|
|
634
|
+
}
|
|
635
|
+
/** Can BEA's public bulk files serve this request? [true, files] or [false, why not]. */
|
|
636
|
+
export function bulkCoverage(req) {
|
|
637
|
+
const ds = req.dataset, t = req.table.toUpperCase(), f = req.frequency;
|
|
638
|
+
if (ds === "NIPA") {
|
|
639
|
+
if (!["A", "Q", "M"].includes(f))
|
|
640
|
+
return [false, `frequency '${f}' not in the NIPA flat files`];
|
|
641
|
+
return [true, `NIPA flat files NipaData${f}.txt + SeriesRegister.txt`];
|
|
642
|
+
}
|
|
643
|
+
if (ds === "Regional") {
|
|
644
|
+
const geo = pyStr(req.geo || "").toUpperCase();
|
|
645
|
+
if (["MSA", "MIC", "CSA", "PORT", "DIV"].includes(geo) && !t.startsWith("MA")) {
|
|
646
|
+
return [false, `${geo} areas are not in the bulk zip files; the API is needed. Note: since February 2026 BEA no longer ` +
|
|
647
|
+
"produces GDP or personal income for metro areas in the county tables (CAINC*, CAGDP*); sum the component counties " +
|
|
648
|
+
"instead (docs/datasets.md)"];
|
|
649
|
+
}
|
|
650
|
+
if (!/^(SA|SQ|CA|MA)[A-Z]/.test(t))
|
|
651
|
+
return [false, `no bulk file known for Regional table ${t}`];
|
|
652
|
+
if (!bulkGeoOk(geo || "STATE", t))
|
|
653
|
+
return [false, `GeoFips ${geo} cannot be selected from the bulk zip files; the API is needed`];
|
|
654
|
+
return [true, "Regional bulk zip " + zipCandidates(t).join(", ")];
|
|
655
|
+
}
|
|
656
|
+
if (ds === "GDPbyIndustry" && req.bulk_only)
|
|
657
|
+
return [true, `GDP-by-industry workbook ${WORKBOOKS[t.slice(0, 3)]} (sheet ${t}-${f || "A"})`];
|
|
658
|
+
if (ds === "GDPbyIndustry") {
|
|
659
|
+
return [false, "GDPbyIndustry with API TableIDs needs a key; the same national tables are in BEA's public workbooks under " +
|
|
660
|
+
'codes TVA101-TVA113, TGO101-TGO107, TII1xx: use "table": "TVA105" etc. (docs/datasets.md), which needs no key'];
|
|
661
|
+
}
|
|
662
|
+
return [false, `${ds} has no public bulk file this server can read; the API is needed`];
|
|
663
|
+
}
|
|
664
|
+
export async function regionalFromZip(req, srcDir, manifest, files = {}) {
|
|
665
|
+
const table = req.table.toUpperCase();
|
|
666
|
+
const names = new Set([table, table.replace(/(?<=\d)N$/, "")]);
|
|
667
|
+
let chosen = null;
|
|
668
|
+
const errors = [];
|
|
669
|
+
for (const cand of zipCandidates(table)) {
|
|
670
|
+
let local = files[`${cand}.zip`];
|
|
671
|
+
if (!local) {
|
|
672
|
+
try {
|
|
673
|
+
local = await fetchSource(REGIONAL_ZIP + `${cand}.zip`, srcDir, manifest);
|
|
674
|
+
}
|
|
675
|
+
catch (e) {
|
|
676
|
+
if (e instanceof BeaError) {
|
|
677
|
+
if (e.bea === "network")
|
|
678
|
+
throw e;
|
|
679
|
+
errors.push(`${cand}.zip: ${e.message.slice(0, 80)}`);
|
|
680
|
+
continue;
|
|
681
|
+
}
|
|
682
|
+
if (e?.code) {
|
|
683
|
+
errors.push(`${cand}.zip: ${String(e.message).slice(0, 80)}`);
|
|
684
|
+
continue;
|
|
685
|
+
}
|
|
686
|
+
throw e;
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
const z = unzip(fs.readFileSync(local));
|
|
690
|
+
const members = Object.keys(z).filter((n) => n.toLowerCase().endsWith(".csv") &&
|
|
691
|
+
[...names].some((nm) => new RegExp(`^${nm.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}_`, "i").test(n)));
|
|
692
|
+
if (members.length) {
|
|
693
|
+
chosen = [z, members];
|
|
694
|
+
break;
|
|
695
|
+
}
|
|
696
|
+
errors.push(`${cand}.zip has no ${table} file`);
|
|
697
|
+
}
|
|
698
|
+
let footnotes = [];
|
|
699
|
+
let texts;
|
|
700
|
+
if (!chosen && files[`${table}.csv`])
|
|
701
|
+
texts = [readTextMember(files[`${table}.csv`])];
|
|
702
|
+
else if (chosen) {
|
|
703
|
+
const [z, members] = chosen;
|
|
704
|
+
const allf = members.filter((m) => m.toUpperCase().includes("ALL_AREAS"));
|
|
705
|
+
texts = (allf.length ? allf : members).map((m) => Buffer.from(z[m]).toString("latin1"));
|
|
706
|
+
footnotes = regionalFootnotes(z, names);
|
|
707
|
+
}
|
|
708
|
+
else {
|
|
709
|
+
throw new BeaError(`Regional table ${table} was not found in BEA's bulk files (${errors.join("; ")}). Use the API route (it ` +
|
|
710
|
+
"needs a key) or check the table name.", "api");
|
|
711
|
+
}
|
|
712
|
+
const ys = yearSet(req.years);
|
|
713
|
+
const lastn = req.years.toUpperCase().startsWith("LAST") ? parseInt(req.years.slice(4), 10) : null;
|
|
714
|
+
const lcs = new Set((req.line_codes?.length ? req.line_codes : ["ALL"]).map((x) => pyStr(x).toUpperCase()));
|
|
715
|
+
const geo = normGeo(req.geo || "STATE");
|
|
716
|
+
if (!bulkGeoOk(geo, table)) {
|
|
717
|
+
throw new BeaError(`GeoFips ${geo} cannot be selected from the bulk files for ${table}; use STATE, COUNTY, a state postal ` +
|
|
718
|
+
"code, a list of FIPS codes, or the API route.", "api");
|
|
719
|
+
}
|
|
720
|
+
const geoList = /^\d/.test(geo) ? new Set(geo.split(",")) : null;
|
|
721
|
+
const postal = Object.hasOwn(STATE_FIPS, geo) ? STATE_FIPS[geo] : undefined;
|
|
722
|
+
let out = [];
|
|
723
|
+
const notes = [];
|
|
724
|
+
const seen = new Set();
|
|
725
|
+
const haveYears = new Set();
|
|
726
|
+
for (const text of texts) {
|
|
727
|
+
const [header, data, foot] = regionalCsvRows(text);
|
|
728
|
+
const periods = header.slice(8);
|
|
729
|
+
for (const r of data) {
|
|
730
|
+
const fips = r[0].trim().replace(/^"+|"+$/g, "").trim();
|
|
731
|
+
const lvl = geoLevel(fips, table.startsWith("MA") ? "MSA" : "");
|
|
732
|
+
if (geoList !== null) {
|
|
733
|
+
if (!geoList.has(fips))
|
|
734
|
+
continue;
|
|
735
|
+
}
|
|
736
|
+
else if (postal) {
|
|
737
|
+
if (lvl !== "county" || !fips.startsWith(postal))
|
|
738
|
+
continue;
|
|
739
|
+
}
|
|
740
|
+
else if (geo === "STATE" && !["state", "nation"].includes(lvl))
|
|
741
|
+
continue;
|
|
742
|
+
else if (geo === "COUNTY" && !["county", "state", "nation"].includes(lvl))
|
|
743
|
+
continue;
|
|
744
|
+
const line = r[4].trim();
|
|
745
|
+
if (!lcs.has("ALL") && !lcs.has(line))
|
|
746
|
+
continue;
|
|
747
|
+
const unit = r[7].trim();
|
|
748
|
+
periods.forEach((p, k) => {
|
|
749
|
+
const [tp, year, sub, freq] = parsePeriod(p, p.includes(":") ? "Q" : "A");
|
|
750
|
+
haveYears.add(year);
|
|
751
|
+
if (ys !== null && (year === null || !ys.has(year)))
|
|
752
|
+
return;
|
|
753
|
+
const key = `${fips}\u0000${line}\u0000${tp}`;
|
|
754
|
+
if (seen.has(key))
|
|
755
|
+
return;
|
|
756
|
+
seen.add(key);
|
|
757
|
+
const [value, flag] = parseValue(8 + k < r.length ? r[8 + k] : "");
|
|
758
|
+
const ind = r[5].trim().replace(/^"+|"+$/g, "");
|
|
759
|
+
out.push({ request_id: req.id, dataset: "Regional", table: r[3].trim() || table, line, series_code: `${table}-${line}`,
|
|
760
|
+
description: r[6].trim().replace(/\s+\d+\/\s*$/, ""), geo_fips: fips, geo_name: cleanGeoName(r[1]), geo_level: lvl,
|
|
761
|
+
industry: ind === "..." ? "" : ind, time_period: tp, year, subperiod: sub, frequency: freq, value: valueText(value),
|
|
762
|
+
flag, unit, metric: "", unit_mult: String(unitMultFromText(unit)), note_ref: "", dims: "" });
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
for (const line of foot) {
|
|
766
|
+
if (footnotes.length && /see the included footnote file/i.test(line))
|
|
767
|
+
continue;
|
|
768
|
+
notes.push({ request_id: req.id, table, note_ref: "", note_text: line });
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
for (const [ref, text] of footnotes) {
|
|
772
|
+
if (!notes.some((n) => n.note_text === text))
|
|
773
|
+
notes.push({ request_id: req.id, table, note_ref: ref, note_text: text });
|
|
774
|
+
}
|
|
775
|
+
if (lastn)
|
|
776
|
+
out = keepLastN(out, lastn);
|
|
777
|
+
if (!out.length) {
|
|
778
|
+
const have = [...haveYears].filter((y) => y !== null).sort((a, b) => a - b);
|
|
779
|
+
if (have.length && ys !== null && ![...ys].some((y) => have.includes(y))) {
|
|
780
|
+
throw new BeaError(`No rows for ${table}, years ${req.years}: the bulk file holds ${have[0]}-${have[have.length - 1]} (BEA ` +
|
|
781
|
+
`has not published later years of ${table} yet; RPPs lag state income by about a year).`, "bad_request");
|
|
782
|
+
}
|
|
783
|
+
throw new BeaError(`No rows for ${table} with lines ${pyListRepr([...lcs].sort(byCodePoint))}, geography ${geo}, years ` +
|
|
784
|
+
`${req.years} in the bulk file. Check the line code (request_schema search "values Regional LineCode TableName=${table}" ` +
|
|
785
|
+
"needs a key; the bulk zip's definition.xml lists the lines).", "api");
|
|
786
|
+
}
|
|
787
|
+
return [out, notes];
|
|
788
|
+
}
|
|
789
|
+
const pyListRepr = (xs) => "[" + xs.map((x) => `'${x}'`).join(", ") + "]";
|
|
790
|
+
/** '[Millions of dollars] Seasonally adjusted at annual rates' -> 'Millions of dollars; seasonally adjusted at annual rates'. */
|
|
791
|
+
export function workbookUnit(text) {
|
|
792
|
+
const m = /^\[([^\]]*)\]\s*(.*)$/.exec(text.trim());
|
|
793
|
+
if (!m)
|
|
794
|
+
return text.trim();
|
|
795
|
+
const rest = m[2].trim();
|
|
796
|
+
return m[1].trim() + (rest ? `; ${rest.slice(0, 1).toLowerCase()}${rest.slice(1)}` : "");
|
|
797
|
+
}
|
|
798
|
+
/** A GDP-by-industry table (TVA105 ...) from BEA's public workbook: sheet '<table>-<A|Q>'. */
|
|
799
|
+
export async function industryFromXlsx(req, srcDir, manifest, files = {}) {
|
|
800
|
+
const table = req.table.toUpperCase(), f = (req.frequency || "A").toUpperCase();
|
|
801
|
+
const wbname = WORKBOOKS[table.slice(0, 3)];
|
|
802
|
+
const local = files[wbname] || await fetchSource(INDUSTRY_XLS + wbname, srcDir, manifest);
|
|
803
|
+
const sheet = `${table}-${f}`;
|
|
804
|
+
const sheets = xlsxSheets(fs.readFileSync(local), new Set([sheet, "Contents"]));
|
|
805
|
+
if (!(sheet in sheets))
|
|
806
|
+
throw new BeaError(`${wbname} has no sheet ${sheet}; list the tables with request_schema search "GDPbyIndustry"`, "api");
|
|
807
|
+
const rows = sheets[sheet];
|
|
808
|
+
const hdrI = rows.findIndex((r) => r.length && pyStr(r[0]).trim() === "Line");
|
|
809
|
+
if (hdrI < 0)
|
|
810
|
+
throw new BeaError(`${wbname} sheet ${sheet} has no header row ("Line")`, "api");
|
|
811
|
+
const header = rows[hdrI];
|
|
812
|
+
const meta = rows.slice(0, hdrI).filter((r) => r.length && truthy(r[0])).map((r) => pyStr(r[0]));
|
|
813
|
+
const unit = meta.filter((m) => m.startsWith("[")).map(workbookUnit)[0] ?? "";
|
|
814
|
+
const ys = yearSet(req.years);
|
|
815
|
+
const lastn = req.years.toUpperCase().startsWith("LAST") ? parseInt(req.years.slice(4), 10) : null;
|
|
816
|
+
let out = [];
|
|
817
|
+
const notes = [{ request_id: req.id, table, note_ref: table, note_text: meta.join(" | ") }];
|
|
818
|
+
let section = "";
|
|
819
|
+
for (const r of rows.slice(hdrI + 1)) {
|
|
820
|
+
if (r.length && r[0] === null && r.length > 1 && pyStr(r[1] || "").trim().endsWith(":")) {
|
|
821
|
+
section = pyStr(r[1]).trim().replace(/:+$/, "").trim();
|
|
822
|
+
continue;
|
|
823
|
+
}
|
|
824
|
+
if (!r.length || r[0] === null)
|
|
825
|
+
continue;
|
|
826
|
+
const line = pyStr(r[0]).trim();
|
|
827
|
+
if (!isDigits(line)) {
|
|
828
|
+
notes.push({ request_id: req.id, table, note_ref: "", note_text: line });
|
|
829
|
+
continue;
|
|
830
|
+
}
|
|
831
|
+
const rawDesc = pyStr(r[1] || "");
|
|
832
|
+
const indent = rawDesc.length - rawDesc.replace(/^ +/, "").length;
|
|
833
|
+
const desc = rawDesc.trim().replace(/\\\d+\\$/, "").trim();
|
|
834
|
+
for (let k = 3; k < header.length; k++) {
|
|
835
|
+
if (header[k] === null)
|
|
836
|
+
continue;
|
|
837
|
+
const [tp, year, sub, freq] = parsePeriod(pyStr(header[k]).trim(), f);
|
|
838
|
+
if (ys !== null && (year === null || !ys.has(year)))
|
|
839
|
+
continue;
|
|
840
|
+
const [value, flag] = parseValue(k < r.length ? r[k] : "");
|
|
841
|
+
out.push({ request_id: req.id, dataset: "GDPbyIndustry", table, line, series_code: `${table}-${line}`, description: desc,
|
|
842
|
+
geo_fips: "00000", geo_name: "United States", geo_level: "nation", industry: desc, time_period: tp, year, subperiod: sub,
|
|
843
|
+
frequency: freq, value: valueText(value), flag, unit: sectionUnit(unit, section), metric: "",
|
|
844
|
+
unit_mult: String(unitMultFromText(unit)), note_ref: "", dims: pyJsonDumps({ indent, section }) });
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
if (lastn)
|
|
848
|
+
out = keepLastN(out, lastn);
|
|
849
|
+
if (!out.length)
|
|
850
|
+
throw new BeaError(`No rows for ${table} ${f} in years ${req.years}`, "api");
|
|
851
|
+
return [out, notes];
|
|
852
|
+
}
|
|
853
|
+
export function periodDate(tp) {
|
|
854
|
+
const m = /^(\d{4})(?:Q([1-4])|M(\d{2}))?$/.exec(pyStr(tp));
|
|
855
|
+
if (!m)
|
|
856
|
+
return "";
|
|
857
|
+
const month = m[2] ? (Number(m[2]) - 1) * 3 + 1 : m[3] ? Number(m[3]) : 1;
|
|
858
|
+
return `${m[1]}-${pad2(month)}-01`;
|
|
859
|
+
}
|
|
860
|
+
// --------------------------------------------------------------------------- package
|
|
861
|
+
/** What the API cannot do, applied on every route: requested lines, the last n years, and loud empty results. */
|
|
862
|
+
export function finalizeRows(reqs, rows) {
|
|
863
|
+
const by = new Map();
|
|
864
|
+
for (const r of rows) {
|
|
865
|
+
if (!by.has(r.request_id))
|
|
866
|
+
by.set(r.request_id, []);
|
|
867
|
+
by.get(r.request_id).push(r);
|
|
868
|
+
}
|
|
869
|
+
const out = [];
|
|
870
|
+
for (const q of reqs) {
|
|
871
|
+
let rs = by.get(q.id) || [];
|
|
872
|
+
if (NIPA_LIKE.includes(q.dataset) && q.line_codes?.length && rs.length) {
|
|
873
|
+
const want = new Set(q.line_codes);
|
|
874
|
+
const have = new Set(rs.map((r) => pyStr(r.line)));
|
|
875
|
+
const miss = [...want].filter((x) => !have.has(x));
|
|
876
|
+
if (miss.length) {
|
|
877
|
+
const lines = [...have].sort((a, b) => (isDigits(a) ? parseInt(a, 10) : 0) - (isDigits(b) ? parseInt(b, 10) : 0));
|
|
878
|
+
throw new BeaError(`Request ${q.id}: ${q.table} has no line ${miss.sort(byCodePoint).join(", ")}. Its lines: ` +
|
|
879
|
+
`${lines.join(", ")} (request_schema search "lines ${q.table}").`, "api");
|
|
880
|
+
}
|
|
881
|
+
rs = rs.filter((r) => want.has(pyStr(r.line)));
|
|
882
|
+
}
|
|
883
|
+
if (pyStr(q.years).toUpperCase().startsWith("LAST") && rs.length) {
|
|
884
|
+
const n = parseInt(pyStr(q.years).slice(4), 10);
|
|
885
|
+
const keep = new Set([...new Set(rs.map((r) => r.year).filter((y) => typeof y === "number"))].sort((a, b) => a - b).slice(-n));
|
|
886
|
+
rs = rs.filter((r) => keep.has(r.year));
|
|
887
|
+
}
|
|
888
|
+
if (!rs.length) {
|
|
889
|
+
throw new BeaError(`Request ${q.id} (${q.dataset} ${q.table}, years ${q.years}) returned no rows: check the years, frequency, ` +
|
|
890
|
+
"lines and areas.", "api");
|
|
891
|
+
}
|
|
892
|
+
if (q.dataset === "Regional") {
|
|
893
|
+
const asked = new Set(pyStr(q.geo || "").split(",").filter((g) => /^\d{5}$/.test(g)));
|
|
894
|
+
const got = new Set(rs.map((r) => r.geo_fips));
|
|
895
|
+
const missing = [...asked].filter((g) => !got.has(g));
|
|
896
|
+
if (missing.length) {
|
|
897
|
+
throw new BeaError(`Request ${q.id}: no ${q.table} rows for GeoFips ${missing.sort(byCodePoint).join(", ")} (states are ` +
|
|
898
|
+
"SS000, counties SSCCC; see docs/datasets.md section 4).", "api");
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
out.push(...rs);
|
|
902
|
+
}
|
|
903
|
+
return out;
|
|
904
|
+
}
|
|
905
|
+
/** {request id: years the request names explicitly that no row has}. */
|
|
906
|
+
export function missingYears(reqs, rows) {
|
|
907
|
+
const have = new Map();
|
|
908
|
+
for (const r of rows) {
|
|
909
|
+
if (!have.has(r.request_id))
|
|
910
|
+
have.set(r.request_id, new Set());
|
|
911
|
+
have.get(r.request_id).add(r.year);
|
|
912
|
+
}
|
|
913
|
+
const out = {};
|
|
914
|
+
for (const q of reqs) {
|
|
915
|
+
const ys = yearSet(pyStr(truthy(q.years) ? q.years : "X"));
|
|
916
|
+
const miss = ys && ys.size && have.has(q.id) ? [...ys].filter((y) => !have.get(q.id).has(y)).sort((a, b) => a - b) : [];
|
|
917
|
+
if (miss.length)
|
|
918
|
+
out[q.id] = miss;
|
|
919
|
+
}
|
|
920
|
+
return out;
|
|
921
|
+
}
|
|
922
|
+
export const ROUTE_TEXT = { api: "retrieved through the BEA Data API", bulk: "downloaded from BEA's public bulk files",
|
|
923
|
+
manual_download: "downloaded by the user and uploaded" };
|
|
924
|
+
export function packageData(out, reqs, spec, rows0, notes, route, callsLog, manifest, acquired, extraNotes = []) {
|
|
925
|
+
const rows = finalizeRows(reqs, rows0);
|
|
926
|
+
const ddir = dataDir(out, SOURCE);
|
|
927
|
+
fs.mkdirSync(ddir, { recursive: true });
|
|
928
|
+
for (const r of rows)
|
|
929
|
+
r.date = periodDate(r.time_period ?? "");
|
|
930
|
+
const csvPath = path.join(ddir, "bea_data.csv");
|
|
931
|
+
fs.writeFileSync(csvPath + ".part", writeCsv(rows, COLUMNS));
|
|
932
|
+
fs.renameSync(csvPath + ".part", csvPath);
|
|
933
|
+
fs.writeFileSync(path.join(ddir, "notes.csv"), writeCsv(notes, NOTE_COLUMNS));
|
|
934
|
+
const vintage = [...new Set(notes.map((n) => pyStr(n.note_text)).filter((t) => /last (?:updated|revised)|release/i.test(t)))].sort(byCodePoint);
|
|
935
|
+
vintage.push(...Object.entries(manifest).filter(([, v]) => v.last_modified).map(([k, v]) => `${k}: Last-Modified ${v.last_modified}`));
|
|
936
|
+
const flags = new Map();
|
|
937
|
+
for (const r of rows)
|
|
938
|
+
if (r.flag)
|
|
939
|
+
flags.set(r.flag, (flags.get(r.flag) || 0) + 1);
|
|
940
|
+
const byReq = new Map();
|
|
941
|
+
for (const r of rows)
|
|
942
|
+
byReq.set(r.request_id, (byReq.get(r.request_id) || 0) + 1);
|
|
943
|
+
const gaps = missingYears(reqs, rows);
|
|
944
|
+
const gapNotes = Object.entries(gaps).map(([k, v]) => `${k}: no rows for year(s) ${v.join(", ")}, which the spec asks for (BEA ` +
|
|
945
|
+
"has not published them, or the table does not hold them); the data end earlier: say so");
|
|
946
|
+
const retrieved = utcIso();
|
|
947
|
+
const tables = [{ name: "bea_data", file: "bea_data.csv", rows: rows.length, columns: COLUMNS },
|
|
948
|
+
{ name: "notes", file: "notes.csv", rows: notes.length, columns: NOTE_COLUMNS }];
|
|
949
|
+
const tableNames = [...new Set(reqs.map((q) => `${q.dataset} ${q.table}`))].sort(byCodePoint);
|
|
950
|
+
const cite = CITATION.replace("@TABLES@", tableNames.join("; ")).replace("@ROUTE@", ROUTE_TEXT[route] ?? route)
|
|
951
|
+
.replace("@DATE@", longDate());
|
|
952
|
+
const P = policy();
|
|
953
|
+
const rec = { source: SOURCE, description: spec?.description ?? "", fingerprint: fingerprint(reqs), route, acquired,
|
|
954
|
+
downloaded: today(), retrieved, rows: rows.length, rows_by_request: Object.fromEntries(byReq), missing_years: gaps,
|
|
955
|
+
notes: [...extraNotes, ...gapNotes], flags: Object.fromEntries(flags), vintage, spec, requests: reqs, api_calls: callsLog,
|
|
956
|
+
bulk_files: manifest, csv_sha256: sha256File(csvPath), tables, citation: cite, http_requests: trail().slice(),
|
|
957
|
+
policy: { redistribution: P.redistribution.mode, cache: P.cache.mode } };
|
|
958
|
+
fs.writeFileSync(path.join(ddir, "request.json"), redact(JSON.stringify(rec, null, 2)));
|
|
959
|
+
const unitKeys = new Map();
|
|
960
|
+
for (const r of rows) {
|
|
961
|
+
const t = [pyStr(r.table), plainUnit(pyStr(r.unit ?? ""), pyStr(r.metric ?? ""), r.unit_mult ?? ""),
|
|
962
|
+
pyStr(r.unit_mult)];
|
|
963
|
+
unitKeys.set(t.join("\u0000"), t);
|
|
964
|
+
}
|
|
965
|
+
const units = [...unitKeys.values()].sort((a, b) => byCodePoint(a[0], b[0]) || byCodePoint(a[1], b[1]) || byCodePoint(a[2], b[2]));
|
|
966
|
+
const unitTables = [...new Set(units.map(([t]) => t))].slice(0, 40);
|
|
967
|
+
const lines = [
|
|
968
|
+
`# BEA data: ${spec?.description ?? ""}`, "",
|
|
969
|
+
`- Downloaded: ${today()} (${acquired})`,
|
|
970
|
+
`- Route: ${route}`,
|
|
971
|
+
`- Rows: ${fmtInt(rows.length)} (` + [...byReq].map(([k, v]) => `${k}: ${fmtInt(v)}`).join(", ") + ")",
|
|
972
|
+
...Object.entries(gaps).map(([k, v]) => `- **Years asked for but not in the data**: \`${k}\`: ${v.join(", ")} (not published ` +
|
|
973
|
+
"yet, or not in this table): say so wherever the period is given"),
|
|
974
|
+
"- File: `bea_data.csv`, tidy long format (one row per series, area and period)",
|
|
975
|
+
"- Notes from BEA: `notes.csv`; exact request and checksums: `request.json`", "",
|
|
976
|
+
"## Requests",
|
|
977
|
+
...reqs.map((q) => `- \`${q.id}\`: ${q.dataset} ${q.table}` + (q.frequency ? `, frequency ${q.frequency}` : "") +
|
|
978
|
+
`, years ${pyStr(q.years).toUpperCase() === "X" ? "ALL" : q.years}` + (q.line_codes.length ? `, lines ${q.line_codes.join(",")}` : "") +
|
|
979
|
+
(q.geo ? `, areas ${q.geo}` : "")), "",
|
|
980
|
+
"## Columns",
|
|
981
|
+
"- `value` is numeric as BEA publishes it; multiply by 10**`unit_mult` for base units (for example unit_mult 6 = millions " +
|
|
982
|
+
"of dollars): amount = value x 10**unit_mult.",
|
|
983
|
+
"- `unit` is BEA's kind of figure (Level, Percent change, annual rate, Index ...) and `metric` the measure (Current " +
|
|
984
|
+
"Dollars, Chained Dollars, Persons, Fisher Quantity Index; Ratio is a percent or dollars per person, as the line " +
|
|
985
|
+
"description says). Check both before combining series.",
|
|
986
|
+
"- `flag` holds BEA's non-numeric codes ((D), (NA), (NM), (L) ...); `value` is then empty. Never treat a flagged cell as zero.",
|
|
987
|
+
"- `time_period`: 2024, 2024Q1 or 2024M01; `date` is the first day of that period (for dashboards); `year`, `subperiod` " +
|
|
988
|
+
"and `frequency` split it.",
|
|
989
|
+
"- Chained-dollar series use 2017 as the reference year (since BEA's 2023 comprehensive update); the NIPA bulk files say " +
|
|
990
|
+
"only \"Chained Dollars\", so state the year from the table header when you cite it.",
|
|
991
|
+
"- GDP-by-industry rows keep the table's indentation in `dims` ({\"indent\": n}); line numbers differ between tables " +
|
|
992
|
+
"(TVA106 has an extra line), so match industries across tables by description and order.",
|
|
993
|
+
"", "## Bulk files",
|
|
994
|
+
...(Object.keys(manifest).length ? Object.entries(manifest).map(([k, v]) => `- ${k}: ${v.url ?? ""} (Last-Modified ` +
|
|
995
|
+
`${v.last_modified ?? ""}; cached at ${v.path ?? ""})`) : ["- none (API or upload)"]), "",
|
|
996
|
+
"## Units by table",
|
|
997
|
+
...unitTables.map((t) => `- ${t}: ` + units.filter(([t2]) => t2 === t).map(([, u, m]) => `${u || "(see notes)"}` +
|
|
998
|
+
(m !== "" && m !== "0" ? ` (unit_mult ${m})` : "")).join("; ")),
|
|
999
|
+
"", "## Special values found",
|
|
1000
|
+
...(flags.size ? [...flags].map(([k, v]) => `- \`${k}\` x ${fmtInt(v)}: ${Object.hasOwn(FLAG_TEXT, k) ? FLAG_TEXT[k] : "see the table notes"}`) : ["- none"]),
|
|
1001
|
+
"", "## Vintage",
|
|
1002
|
+
...(vintage.length ? vintage.map((v) => `- ${v}`) : ["- BEA serves only the latest estimates; the download date above is the vintage."]),
|
|
1003
|
+
"Estimates are revised: annual updates each September and comprehensive updates every five years or so. Rerunning later " +
|
|
1004
|
+
"can change the numbers; the checksum in `request.json` identifies this vintage.", "",
|
|
1005
|
+
"## Citation", cite, "",
|
|
1006
|
+
"Suggested source line: \"Analyst calculations using data from the Bureau of Economic Analysis.\" (or \"<Organization>, " +
|
|
1007
|
+
"using data from the Bureau of Economic Analysis\")",
|
|
1008
|
+
"", "## Use and redistribution", "",
|
|
1009
|
+
`Rule: **${P.redistribution.mode}**. ${P.redistribution.why ?? ""}`,
|
|
1010
|
+
];
|
|
1011
|
+
if (route === "api")
|
|
1012
|
+
lines.push("", `Required notice for products built with the API: "${API_NOTICE}"`);
|
|
1013
|
+
fs.writeFileSync(path.join(ddir, "README.md"), lines.join("\n") + "\n");
|
|
1014
|
+
const raw = path.join(ddir, "raw");
|
|
1015
|
+
const rawFiles = callsLog.map((c) => path.join(ddir, c.file)).filter((p) => fs.existsSync(p) && p.startsWith(raw));
|
|
1016
|
+
return { data_dir: ddir, rows: rows.length, rows_by_request: Object.fromEntries(byReq), tables, citation: cite, retrieved,
|
|
1017
|
+
vintage, notes: [...extraNotes, ...gapNotes], files: rawFiles, route };
|
|
1018
|
+
}
|
|
1019
|
+
// --------------------------------------------------------------------------- routes
|
|
1020
|
+
export async function runApi(client, reqs, out) {
|
|
1021
|
+
const rawDir = path.join(dataDir(out, SOURCE), "raw");
|
|
1022
|
+
fs.mkdirSync(rawDir, { recursive: true });
|
|
1023
|
+
const rows = [], notes = [], log = [];
|
|
1024
|
+
for (const q of reqs) {
|
|
1025
|
+
for (let k = 0; k < q.calls.length; k++) {
|
|
1026
|
+
const call = q.calls[k];
|
|
1027
|
+
const ds = q.dataset;
|
|
1028
|
+
const [data, nts, info] = await client.getData(ds, call);
|
|
1029
|
+
rows.push(...tidyApiRows(q, call, data));
|
|
1030
|
+
notes.push(...tidyNotes(q, nts));
|
|
1031
|
+
const name = `${q.id}${q.calls.length > 1 ? "_" + (k + 1) : ""}.json`;
|
|
1032
|
+
fs.writeFileSync(path.join(rawDir, name), redact(JSON.stringify(scrubResponse({ request: { ...call, DataSetName: ds }, Data: data,
|
|
1033
|
+
...info }), null, 1)));
|
|
1034
|
+
log.push({ request_id: q.id, dataset: ds, params: call, rows: data.length, file: `raw/${name}` });
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
return [rows, notes, log];
|
|
1038
|
+
}
|
|
1039
|
+
export async function runBulk(reqs, srcDir, files) {
|
|
1040
|
+
const rows = [], notes = [];
|
|
1041
|
+
const manifest = {};
|
|
1042
|
+
for (const q of reqs) {
|
|
1043
|
+
const [ok, why] = bulkCoverage(q);
|
|
1044
|
+
if (!ok)
|
|
1045
|
+
throw new BeaError(`Request ${q.id}: ${why}`, "api");
|
|
1046
|
+
let r;
|
|
1047
|
+
if (q.dataset === "NIPA")
|
|
1048
|
+
r = await nipaFromFlat(q, srcDir, manifest, files);
|
|
1049
|
+
else if (q.dataset === "Regional")
|
|
1050
|
+
r = await regionalFromZip(q, srcDir, manifest, files);
|
|
1051
|
+
else if (q.dataset === "GDPbyIndustry" && q.bulk_only)
|
|
1052
|
+
r = await industryFromXlsx(q, srcDir, manifest, files);
|
|
1053
|
+
else
|
|
1054
|
+
throw new BeaError(`${q.dataset} has no bulk route`, "api");
|
|
1055
|
+
rows.push(...r[0]);
|
|
1056
|
+
notes.push(...r[1]);
|
|
1057
|
+
}
|
|
1058
|
+
return [rows, notes, manifest];
|
|
1059
|
+
}
|
|
1060
|
+
export function existingData(out, reqs) {
|
|
1061
|
+
const ddir = dataDir(out, SOURCE);
|
|
1062
|
+
const rp = path.join(ddir, "request.json"), cp = path.join(ddir, "bea_data.csv");
|
|
1063
|
+
if (!fs.existsSync(rp))
|
|
1064
|
+
return null;
|
|
1065
|
+
let saved;
|
|
1066
|
+
try {
|
|
1067
|
+
saved = JSON.parse(fs.readFileSync(rp, "utf8"));
|
|
1068
|
+
}
|
|
1069
|
+
catch {
|
|
1070
|
+
return null;
|
|
1071
|
+
}
|
|
1072
|
+
const ids = (saved.requests || []).map((q) => q.id);
|
|
1073
|
+
const sameIds = ids.length === reqs.length && ids.every((x, i) => x === reqs[i].id);
|
|
1074
|
+
return { path: cp, csv_exists: fs.existsSync(cp), route: saved.route, downloaded: saved.downloaded, rows: saved.rows,
|
|
1075
|
+
spec_matches: saved.fingerprint === fingerprint(reqs) && sameIds, saved };
|
|
1076
|
+
}
|
|
1077
|
+
/** The GetData URL for one call. keyExpr null leaves UserID out. */
|
|
1078
|
+
export function apiUrl(dataset, call, keyExpr = null) {
|
|
1079
|
+
const q = Object.entries(call).map(([k, v]) => `${k}=${pyQuote(pyStr(v), ",")}`).join("&");
|
|
1080
|
+
const user = keyExpr === null ? "" : `UserID=${keyExpr}&`;
|
|
1081
|
+
return `https://apps.bea.gov/api/data?${user}method=GetData&DataSetName=${dataset}&${q}&ResultFormat=JSON`;
|
|
1082
|
+
}
|
|
1083
|
+
export function bulkUrls(q) {
|
|
1084
|
+
if (q.dataset === "GDPbyIndustry")
|
|
1085
|
+
return [INDUSTRY_XLS + WORKBOOKS[q.table.toUpperCase().slice(0, 3)]];
|
|
1086
|
+
if (q.dataset === "NIPA") {
|
|
1087
|
+
return [NIPA_TXT + `NipaData${q.frequency || "A"}.txt`, NIPA_TXT + "SeriesRegister.txt", NIPA_TXT + "TablesRegister.txt"];
|
|
1088
|
+
}
|
|
1089
|
+
return [REGIONAL_ZIP + `${zipCandidates(q.table)[0]}.zip`];
|
|
1090
|
+
}
|
|
1091
|
+
export const apiFileName = (q, k) => `${q.id}${q.calls.length > 1 ? "_" + (k + 1) : ""}.json`;
|
|
1092
|
+
/** The files a user downloads by hand (core manual.ts items): public bulk files, and API replies with the key. */
|
|
1093
|
+
export function manualItems(reqs) {
|
|
1094
|
+
const bulk = [];
|
|
1095
|
+
const api = [];
|
|
1096
|
+
for (const q of reqs) {
|
|
1097
|
+
if (bulkCoverage(q)[0])
|
|
1098
|
+
bulk.push(...bulkUrls(q));
|
|
1099
|
+
else {
|
|
1100
|
+
q.calls.forEach((call, k) => {
|
|
1101
|
+
const name = apiFileName(q, k);
|
|
1102
|
+
api.push({ name, url: apiUrl(q.dataset, call, "{API_KEY}"), filename: name, needs_key: true, method: "GET",
|
|
1103
|
+
note: `BEA ${q.dataset} GetData for request ${q.id}. BEA repeats the key inside the reply: it is removed when the ` +
|
|
1104
|
+
"file is ingested.", optional: false });
|
|
1105
|
+
});
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
const files = [...new Set(bulk)].map((u) => {
|
|
1109
|
+
const name = u.slice(u.lastIndexOf("/") + 1);
|
|
1110
|
+
return { name, url: u, filename: name, needs_key: false, method: "GET", note: "BEA public bulk file (no key)", optional: false };
|
|
1111
|
+
});
|
|
1112
|
+
return [...files, ...api];
|
|
1113
|
+
}
|
|
1114
|
+
// --------------------------------------------------------------------------- ingest
|
|
1115
|
+
const BULK_NAME = /^(NipaData[AQM]\.txt|SeriesRegister\.txt|TablesRegister\.txt|[A-Z0-9]+\.zip|[A-Z0-9]+\.csv|[A-Za-z]+\.xlsx)$/i;
|
|
1116
|
+
export async function ingestFiles(paths, spec, reqs, out, extraNotes = []) {
|
|
1117
|
+
if (!paths.length)
|
|
1118
|
+
throw new BeaError("ingest needs the downloaded files or their folder.", "other");
|
|
1119
|
+
const files = [];
|
|
1120
|
+
const walk = (p) => {
|
|
1121
|
+
if (fs.statSync(p).isDirectory())
|
|
1122
|
+
for (const n of fs.readdirSync(p))
|
|
1123
|
+
walk(path.join(p, n));
|
|
1124
|
+
else
|
|
1125
|
+
files.push(p);
|
|
1126
|
+
};
|
|
1127
|
+
for (const p of paths) {
|
|
1128
|
+
if (!fs.existsSync(p))
|
|
1129
|
+
throw new BeaError(`Not found: ${p}`, "other", { dataKind: "not_found" });
|
|
1130
|
+
walk(p);
|
|
1131
|
+
}
|
|
1132
|
+
const byName = new Map(files.map((f) => [path.basename(f), f]));
|
|
1133
|
+
const src = path.join(dataDir(out, SOURCE), "source");
|
|
1134
|
+
fs.mkdirSync(src, { recursive: true });
|
|
1135
|
+
const rows = [], notes = [], log = [];
|
|
1136
|
+
const manifest = {};
|
|
1137
|
+
const warn = [];
|
|
1138
|
+
for (const q of reqs) {
|
|
1139
|
+
const [ok] = bulkCoverage(q);
|
|
1140
|
+
const jsons = q.calls.map((_, k) => byName.get(apiFileName(q, k)));
|
|
1141
|
+
if (jsons.every(Boolean)) {
|
|
1142
|
+
for (let k = 0; k < q.calls.length; k++) {
|
|
1143
|
+
const jp = jsons[k], call = q.calls[k];
|
|
1144
|
+
const text = readTextMember(jp);
|
|
1145
|
+
let payload;
|
|
1146
|
+
try {
|
|
1147
|
+
payload = JSON.parse(text);
|
|
1148
|
+
}
|
|
1149
|
+
catch {
|
|
1150
|
+
throw new BeaError(`${path.basename(jp)} is not a BEA JSON reply.`, "other");
|
|
1151
|
+
}
|
|
1152
|
+
if (new RegExp(GUID).test(text)) {
|
|
1153
|
+
warn.push(`${path.basename(jp)} still held the user's key; it was removed from the saved copy.`);
|
|
1154
|
+
}
|
|
1155
|
+
const err = firstError(payload);
|
|
1156
|
+
if (err)
|
|
1157
|
+
throw new BeaError(`${path.basename(jp)} holds a BEA error: ${pyStr(err.APIErrorDescription)}`, "api");
|
|
1158
|
+
let res = payload?.BEAAPI?.Results ?? {};
|
|
1159
|
+
if (Array.isArray(res) && res.length)
|
|
1160
|
+
res = res[0];
|
|
1161
|
+
const data = truthy(res?.Data) ? res.Data : payload?.BEAAPI?.Data ?? [];
|
|
1162
|
+
const list = Array.isArray(data) ? data : [data];
|
|
1163
|
+
rows.push(...tidyApiRows(q, call, list));
|
|
1164
|
+
notes.push(...tidyNotes(q, res?.Notes ?? []));
|
|
1165
|
+
const rawDir = path.join(dataDir(out, SOURCE), "raw");
|
|
1166
|
+
fs.mkdirSync(rawDir, { recursive: true });
|
|
1167
|
+
fs.writeFileSync(path.join(rawDir, path.basename(jp)), redact(JSON.stringify(scrubResponse(payload), null, 1)).replace(GUID_RE, "[REDACTED]"));
|
|
1168
|
+
log.push({ request_id: q.id, dataset: q.dataset, params: call, rows: list.length, file: `raw/${path.basename(jp)}` });
|
|
1169
|
+
}
|
|
1170
|
+
continue;
|
|
1171
|
+
}
|
|
1172
|
+
if (ok) {
|
|
1173
|
+
const local = {};
|
|
1174
|
+
for (const [name, fp] of byName) {
|
|
1175
|
+
if (BULK_NAME.test(name)) {
|
|
1176
|
+
const dest = path.join(src, name);
|
|
1177
|
+
if (path.resolve(fp) !== path.resolve(dest))
|
|
1178
|
+
fs.copyFileSync(fp, dest);
|
|
1179
|
+
local[name] = dest;
|
|
1180
|
+
manifest[name] = { file: name, sha256: sha256File(dest), bytes: fs.statSync(dest).size, acquired: "uploaded by the user" };
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
let r;
|
|
1184
|
+
try {
|
|
1185
|
+
if (q.dataset === "NIPA")
|
|
1186
|
+
r = await nipaFromFlat(q, src, {}, local);
|
|
1187
|
+
else if (q.dataset === "GDPbyIndustry")
|
|
1188
|
+
r = await industryFromXlsx(q, src, {}, local);
|
|
1189
|
+
else
|
|
1190
|
+
r = await regionalFromZip(q, src, {}, local);
|
|
1191
|
+
}
|
|
1192
|
+
catch (e) {
|
|
1193
|
+
if (!(e instanceof DataError) && !e?.code)
|
|
1194
|
+
throw e;
|
|
1195
|
+
throw new BeaError(`Request ${q.id}: the uploaded files do not hold it (${String(e.message).slice(0, 200)}). ` +
|
|
1196
|
+
"Expected " + bulkUrls(q).map((u) => u.slice(u.lastIndexOf("/") + 1)).join(", "), "other");
|
|
1197
|
+
}
|
|
1198
|
+
rows.push(...r[0]);
|
|
1199
|
+
notes.push(...r[1]);
|
|
1200
|
+
continue;
|
|
1201
|
+
}
|
|
1202
|
+
throw new BeaError(`Request ${q.id}: no uploaded file for it. Expected ` + q.calls.map((_, k) => apiFileName(q, k)).join(", ") +
|
|
1203
|
+
" (from the download script).", "other", { dataKind: "not_found" });
|
|
1204
|
+
}
|
|
1205
|
+
return packageData(out, reqs, spec, rows, notes, "manual_download", log, manifest, "downloaded by the user and uploaded", [...extraNotes, ...warn]);
|
|
1206
|
+
}
|
|
1207
|
+
/** Table names: NIPA titles and the GDP-by-industry workbooks (no key); other datasets through the API (key). */
|
|
1208
|
+
export async function findTables(text, dataset, workspace) {
|
|
1209
|
+
const words = text.split(/\s+/).filter(Boolean).map((w) => w.toLowerCase());
|
|
1210
|
+
const hits = [];
|
|
1211
|
+
const problems = [];
|
|
1212
|
+
if (dataset === null || dataset === "NIPA") {
|
|
1213
|
+
try {
|
|
1214
|
+
const reg = await fetchSource(NIPA_TXT + "TablesRegister.txt", cacheDir(workspace), {});
|
|
1215
|
+
for (const row of csvRows(readTextMember(reg))) {
|
|
1216
|
+
if (row.length >= 2 && words.every((w) => (row[0] + " " + row[1]).toLowerCase().includes(w))) {
|
|
1217
|
+
hits.push({ dataset: "NIPA", table: row[0], title: row[1] });
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
1221
|
+
catch (e) {
|
|
1222
|
+
if (!(e instanceof DataError))
|
|
1223
|
+
throw e;
|
|
1224
|
+
problems.push(`NIPA table list unavailable: ${e.message}`);
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
if (dataset === null || dataset === "GDPbyIndustry") {
|
|
1228
|
+
for (const wb of Object.values(WORKBOOKS)) {
|
|
1229
|
+
try {
|
|
1230
|
+
const local = await fetchSource(INDUSTRY_XLS + wb, cacheDir(workspace), {});
|
|
1231
|
+
for (const row of xlsxSheets(fs.readFileSync(local), new Set(["Contents"])).Contents ?? []) {
|
|
1232
|
+
if (row.length > 2 && row[1] && WORKBOOK_TABLE.test(pyStr(row[1])) &&
|
|
1233
|
+
words.every((w) => (pyStr(row[1]) + " " + pyStr(row[2])).toLowerCase().includes(w))) {
|
|
1234
|
+
hits.push({ dataset: "GDPbyIndustry", table: pyStr(row[1]), title: `${pyStr(row[2])} (workbook ${wb}; A and Q sheets; no key)` });
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
catch (e) {
|
|
1239
|
+
if (!(e instanceof DataError))
|
|
1240
|
+
throw e;
|
|
1241
|
+
problems.push(`${wb} unavailable: ${e.message}`);
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
if (dataset !== null && dataset !== "NIPA" && dataset !== "GDPbyIndustry") {
|
|
1246
|
+
const c = new BeaClient();
|
|
1247
|
+
const param = ["UnderlyingGDPbyIndustry", "InputOutput"].includes(dataset) ? "TableID" : "TableName";
|
|
1248
|
+
for (const v of await c.parameterValues(dataset, param)) {
|
|
1249
|
+
const k = valueKey(v);
|
|
1250
|
+
const d = pyStr(v.Desc || v.Description || "");
|
|
1251
|
+
if (words.every((w) => (k + " " + d).toLowerCase().includes(w)))
|
|
1252
|
+
hits.push({ dataset, table: k, title: d });
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
return [hits, problems];
|
|
1256
|
+
}
|
|
1257
|
+
/** The lines of a NIPA table (SeriesRegister.txt, no key): [line, series code, label, metric, calculation]. */
|
|
1258
|
+
export async function listLines(table, workspace) {
|
|
1259
|
+
const reg = await fetchSource(NIPA_TXT + "SeriesRegister.txt", cacheDir(workspace), {});
|
|
1260
|
+
const found = [];
|
|
1261
|
+
for (const row of csvRows(readTextMember(reg))) {
|
|
1262
|
+
if (row.length < 6 || row[0].startsWith("%"))
|
|
1263
|
+
continue;
|
|
1264
|
+
for (const loc of row[5].split("|")) {
|
|
1265
|
+
if (loc.includes(":") && loc.split(":")[0].toUpperCase() === table.toUpperCase()) {
|
|
1266
|
+
found.push([parseInt(loc.split(":")[1], 10), row[0], row[1], row[2], row[3]]);
|
|
1267
|
+
}
|
|
1268
|
+
}
|
|
1269
|
+
}
|
|
1270
|
+
found.sort((a, b) => a[0] - b[0] || byCodePoint(a[1], b[1]) || byCodePoint(a[2], b[2]) || byCodePoint(a[3], b[3]) || byCodePoint(a[4], b[4]));
|
|
1271
|
+
return found.map(([line, series_code, label, metric, calculation]) => ({ line, series_code, label, metric, calculation }));
|
|
1272
|
+
}
|
|
1273
|
+
export { DATASETS, findApiKey };
|
|
1274
|
+
//# sourceMappingURL=_bea_fetch.js.map
|