data-analyst-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -2
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +716 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +24 -0
- package/dist/report/scan_secrets.js +265 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +247 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +54 -4
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +170 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,728 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* actuaries: reading the published tables (the Python adapter's cell cleaning, workbook and CSV reading, tidy(),
|
|
3
|
+
* long layout, ZIP contents and CMS page links). Rows are Maps so that year columns ("1960", "2024") keep the
|
|
4
|
+
* published column order (a JS object would move integer-like keys to the front); asTable() hands them on.
|
|
5
|
+
*
|
|
6
|
+
* Every table is read the same way (tidy()): title lines above the column names and footnote lines below the data
|
|
7
|
+
* go to the fetch notes; the column-name rows are detected (or given with header_row) and joined per column, with
|
|
8
|
+
* merged header cells spread over their columns; a row holding a single label inside the data ("Intermediate:",
|
|
9
|
+
* "Amount in Billions") becomes the `section` column of the rows below it; `row` is the sheet row number. Values
|
|
10
|
+
* are as published with these exceptions, each stated in source.md: cells holding only a dash (not applicable)
|
|
11
|
+
* are empty; CSV "$" and thousands separators are dropped; .xlsx floats are written in their shortest exact form
|
|
12
|
+
* (74.099999999999994 -> 74.1); a cell stored as an Excel percent-format fraction (0.106 shown as 10.6%) is written
|
|
13
|
+
* as displayed (10.6), with a note giving the count. An indented label column gets `indent` (spaces as published),
|
|
14
|
+
* `group` (the nearest row above with no indent) and `parent` (the nearest row above with a smaller indent), and a
|
|
15
|
+
* first column of years ("2033b" with a footnote letter) gets a numeric `year` column. A second header inside the
|
|
16
|
+
* sheet (SSA V.A4: a historical block, then intermediate / low-cost / high-cost columns) renames the columns of the
|
|
17
|
+
* rows below it; a header repeated on a continued page is skipped. header_row overrides the detection.
|
|
18
|
+
*/
|
|
19
|
+
import { DataError } from "../core/http.js";
|
|
20
|
+
import { withColumns } from "../core/package.js";
|
|
21
|
+
import { unzip, xlsxSheets } from "../core/tabular.js";
|
|
22
|
+
/** Python repr() of a str (or str() of a number). */
|
|
23
|
+
export function repr(s) {
|
|
24
|
+
if (s === null || s === undefined)
|
|
25
|
+
return "None";
|
|
26
|
+
if (typeof s === "number")
|
|
27
|
+
return String(s);
|
|
28
|
+
if (typeof s === "boolean")
|
|
29
|
+
return s ? "True" : "False";
|
|
30
|
+
const t = String(s);
|
|
31
|
+
const q = t.includes("'") && !t.includes('"') ? '"' : "'";
|
|
32
|
+
return q + t.replace(/\\/g, "\\\\").replace(new RegExp(q, "g"), "\\" + q).replace(/\n/g, "\\n").replace(/\r/g, "\\r")
|
|
33
|
+
.replace(/\t/g, "\\t") + q;
|
|
34
|
+
}
|
|
35
|
+
/** Python " ".join(s.split()). */
|
|
36
|
+
const words = (s) => s.split(/\s+/).filter(Boolean).join(" ");
|
|
37
|
+
/** Python s.strip() (whitespace, not NUL). */
|
|
38
|
+
const strip = (s) => s.replace(/^\s+|\s+$/g, "");
|
|
39
|
+
const basename = (p) => p.slice(Math.max(p.lastIndexOf("/"), p.lastIndexOf("\\")) + 1);
|
|
40
|
+
// ---------------------------------------------------------------------- cell values
|
|
41
|
+
export const NUM = /^-?(\d+\.?\d*|\.\d+)([eE][-+]?\d+)?$/;
|
|
42
|
+
export const DASH = "\x00dash"; // a published "not applicable" dash: counts as a value while the table is read, written empty
|
|
43
|
+
const DASHES = new Set(["-", "—", "–", "−", "--", "�"]);
|
|
44
|
+
/** A plain decimal text (sign, digits, optional point) normalized as Decimal.normalize() + format "f" writes it. */
|
|
45
|
+
function plain(sign, intPart, frac) {
|
|
46
|
+
const i = intPart.replace(/^0+/, "") || "0";
|
|
47
|
+
const f = frac.replace(/0+$/, "");
|
|
48
|
+
const s = sign + i + (f ? "." + f : "");
|
|
49
|
+
return s === "-0" || s === "" ? "0" : s;
|
|
50
|
+
}
|
|
51
|
+
/** Decimal("%.15g" % x) as [sign, integer digits, fraction digits]. */
|
|
52
|
+
function sig15(x) {
|
|
53
|
+
if (x === 0)
|
|
54
|
+
return [Object.is(x, -0) ? "-" : "", "0", ""];
|
|
55
|
+
const [m, es] = Math.abs(x).toExponential(14).split("e");
|
|
56
|
+
const digits = m.replace(".", "");
|
|
57
|
+
const e = Number(es);
|
|
58
|
+
const sign = x < 0 ? "-" : "";
|
|
59
|
+
if (e >= digits.length - 1)
|
|
60
|
+
return [sign, digits + "0".repeat(e - (digits.length - 1)), ""];
|
|
61
|
+
if (e >= 0)
|
|
62
|
+
return [sign, digits.slice(0, e + 1), digits.slice(e + 1)];
|
|
63
|
+
return [sign, "0", "0".repeat(-e - 1) + digits];
|
|
64
|
+
}
|
|
65
|
+
/** Shortest exact text of a stored float; a percent-format cell as displayed (x100). */
|
|
66
|
+
function xlsxNum(v, pct) {
|
|
67
|
+
if (!NUM.test(v))
|
|
68
|
+
return v;
|
|
69
|
+
let sign, ip, fp;
|
|
70
|
+
if (v.includes(".") || v.toLowerCase().includes("e")) {
|
|
71
|
+
// Excel keeps 15 significant digits (74.099999999999994 is shown as 74.1, 495.00000000000006 as 495)
|
|
72
|
+
const x = parseFloat(v);
|
|
73
|
+
if (!Number.isFinite(x))
|
|
74
|
+
return v;
|
|
75
|
+
[sign, ip, fp] = sig15(x);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
sign = v.startsWith("-") ? "-" : "";
|
|
79
|
+
ip = v.replace(/^-/, "");
|
|
80
|
+
fp = "";
|
|
81
|
+
}
|
|
82
|
+
if (pct) {
|
|
83
|
+
const f2 = (fp + "00").slice(0, 2);
|
|
84
|
+
ip += f2;
|
|
85
|
+
fp = fp.slice(2);
|
|
86
|
+
}
|
|
87
|
+
return plain(sign, ip, fp);
|
|
88
|
+
}
|
|
89
|
+
function csvNum(v) {
|
|
90
|
+
const s = strip(v);
|
|
91
|
+
const t = strip(s.replace(/\$/g, "").replace(/,/g, ""));
|
|
92
|
+
if (t && NUM.test(t) && /\d/.test(t) && (s.includes(",") || s.includes("$")))
|
|
93
|
+
return t;
|
|
94
|
+
return v;
|
|
95
|
+
}
|
|
96
|
+
export function clean(v, pct = false, fromCsv = false) {
|
|
97
|
+
if (v === null || v === undefined)
|
|
98
|
+
return "";
|
|
99
|
+
const s = String(v);
|
|
100
|
+
if (DASHES.has(strip(s)))
|
|
101
|
+
return DASH;
|
|
102
|
+
if (fromCsv)
|
|
103
|
+
return csvNum(s);
|
|
104
|
+
return NUM.test(strip(s)) ? xlsxNum(strip(s), pct) : s;
|
|
105
|
+
}
|
|
106
|
+
// ---------------------------------------------------------------------- workbook reading
|
|
107
|
+
const ENT = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" };
|
|
108
|
+
function unxml(s) {
|
|
109
|
+
return s.replace(/&(#x[0-9a-fA-F]+|#\d+|\w+);/g, (all, e) => e[0] === "#" ? String.fromCodePoint(e[1] === "x" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10)) : ENT[e] ?? all);
|
|
110
|
+
}
|
|
111
|
+
function attrs(tag) {
|
|
112
|
+
const out = {};
|
|
113
|
+
for (const m of tag.matchAll(/([\w:]+)\s*=\s*"([^"]*)"/g))
|
|
114
|
+
out[m[1]] = unxml(m[2]);
|
|
115
|
+
return out;
|
|
116
|
+
}
|
|
117
|
+
function members(data) {
|
|
118
|
+
return unzip(data);
|
|
119
|
+
}
|
|
120
|
+
const text = (z, n) => (z[n] ? new TextDecoder().decode(z[n]) : "");
|
|
121
|
+
export function sheetNames(data) {
|
|
122
|
+
const wb = text(members(data), "xl/workbook.xml");
|
|
123
|
+
return [...wb.matchAll(/<sheet\b[^>]*\bname="([^"]*)"/g)].map((m) => unescapeHtml(m[1]));
|
|
124
|
+
}
|
|
125
|
+
/** [merged ranges as 0-based [r1, c1, r2, c2], cells stored with a percent number format ("r,c")] of one sheet. */
|
|
126
|
+
function sheetMeta(data, sheet) {
|
|
127
|
+
const z = members(data);
|
|
128
|
+
const target = {};
|
|
129
|
+
for (const m of text(z, "xl/_rels/workbook.xml.rels").matchAll(/<Relationship\b[^>]*>/g)) {
|
|
130
|
+
const a = attrs(m[0]);
|
|
131
|
+
target[a.Id] = a.Target;
|
|
132
|
+
}
|
|
133
|
+
let p = null;
|
|
134
|
+
for (const m of text(z, "xl/workbook.xml").matchAll(/<sheet\b[^>]*>/g)) {
|
|
135
|
+
const a = attrs(m[0]);
|
|
136
|
+
if (a.name === sheet) {
|
|
137
|
+
p = (target[a["r:id"]] || "").replace(/^\/+/, "");
|
|
138
|
+
p = p.startsWith("xl/") ? p : "xl/" + p;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
if (p === null)
|
|
142
|
+
return [[], new Set()];
|
|
143
|
+
const pctStyles = new Set();
|
|
144
|
+
if (z["xl/styles.xml"]) {
|
|
145
|
+
const st = text(z, "xl/styles.xml");
|
|
146
|
+
const fmts = new Set([9, 10]);
|
|
147
|
+
for (const m of st.matchAll(/<numFmt\b[^>]*>/g)) {
|
|
148
|
+
const a = attrs(m[0]);
|
|
149
|
+
const code = (a.formatCode || "").replace(/"[^"]*"|\\.|_./g, "");
|
|
150
|
+
if (code.includes("%"))
|
|
151
|
+
fmts.add(parseInt(a.numFmtId, 10));
|
|
152
|
+
}
|
|
153
|
+
const xfs = /<cellXfs\b[^>]*>([\s\S]*?)<\/cellXfs>/.exec(st);
|
|
154
|
+
if (xfs) {
|
|
155
|
+
let i = 0;
|
|
156
|
+
for (const m of xfs[1].matchAll(/<xf\b[^>]*>/g)) {
|
|
157
|
+
if (fmts.has(parseInt(attrs(m[0]).numFmtId || "0", 10)))
|
|
158
|
+
pctStyles.add(i);
|
|
159
|
+
i++;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
const rc = (ref) => {
|
|
164
|
+
const m = /^([A-Z]+)(\d+)/.exec(ref);
|
|
165
|
+
let col = 0;
|
|
166
|
+
for (const ch of m[1])
|
|
167
|
+
col = col * 26 + ch.charCodeAt(0) - 64;
|
|
168
|
+
return [parseInt(m[2], 10) - 1, col - 1];
|
|
169
|
+
};
|
|
170
|
+
const sh = text(z, p);
|
|
171
|
+
const merges = [];
|
|
172
|
+
for (const m of sh.matchAll(/<mergeCell\b[^>]*\bref="([A-Z]+\d+:[A-Z]+\d+)"/g)) {
|
|
173
|
+
const [a, b] = m[1].split(":");
|
|
174
|
+
const [r1, c1] = rc(a), [r2, c2] = rc(b);
|
|
175
|
+
merges.push([r1, c1, r2, c2]);
|
|
176
|
+
}
|
|
177
|
+
const pct = new Set();
|
|
178
|
+
if (pctStyles.size) {
|
|
179
|
+
for (const m of sh.matchAll(/<c\b[^>]*\br="([A-Z]+\d+)"[^>]*\bs="(\d+)"/g))
|
|
180
|
+
if (pctStyles.has(parseInt(m[2], 10)))
|
|
181
|
+
pct.add(rc(m[1]).join(","));
|
|
182
|
+
for (const m of sh.matchAll(/<c\b[^>]*\bs="(\d+)"[^>]*\br="([A-Z]+\d+)"/g))
|
|
183
|
+
if (pctStyles.has(parseInt(m[1], 10)))
|
|
184
|
+
pct.add(rc(m[2]).join(","));
|
|
185
|
+
}
|
|
186
|
+
return [merges, pct];
|
|
187
|
+
}
|
|
188
|
+
function pickSheet(names, want, label) {
|
|
189
|
+
if (want === null || want === undefined || want === "") {
|
|
190
|
+
const rest = names.filter((n) => strip(n).toLowerCase() !== "notes");
|
|
191
|
+
return (rest.length ? rest : names)[0];
|
|
192
|
+
}
|
|
193
|
+
if ((typeof want === "number" && Number.isInteger(want)) || /^\d+$/.test(strip(String(want)))) {
|
|
194
|
+
const i = parseInt(strip(String(want)), 10);
|
|
195
|
+
if (!(i >= 1 && i <= names.length)) {
|
|
196
|
+
throw new DataError(`${label} has ${names.length} sheet(s), not sheet ${want} (sheets: ${names.join(", ")})`, "bad_request");
|
|
197
|
+
}
|
|
198
|
+
return names[i - 1];
|
|
199
|
+
}
|
|
200
|
+
for (const n of names)
|
|
201
|
+
if (n === want || strip(n).toLowerCase() === strip(String(want)).toLowerCase())
|
|
202
|
+
return n;
|
|
203
|
+
throw new DataError(`no sheet ${repr(want)} in ${label} (sheets: ${names.join(", ")})`, "bad_request");
|
|
204
|
+
}
|
|
205
|
+
/** [sheet name, cleaned grid, merges, percent cells converted] of one sheet. */
|
|
206
|
+
export function xlsxGrid(data, sheet) {
|
|
207
|
+
const names = sheetNames(data);
|
|
208
|
+
const name = pickSheet(names, sheet, "the workbook");
|
|
209
|
+
const raw = xlsxSheets(data, new Set([name]))[name];
|
|
210
|
+
const [merges, pct] = sheetMeta(data, name);
|
|
211
|
+
let nPct = 0;
|
|
212
|
+
const grid = raw.map((r, i) => r.map((v, j) => {
|
|
213
|
+
const p = pct.has(`${i},${j}`) && v !== null && v !== undefined && NUM.test(strip(String(v)));
|
|
214
|
+
if (p)
|
|
215
|
+
nPct++;
|
|
216
|
+
return clean(v, p);
|
|
217
|
+
}));
|
|
218
|
+
return [name, grid, merges, nPct];
|
|
219
|
+
}
|
|
220
|
+
/** Python's csv.reader (excel dialect, not strict) over text iterated by lines ending in "\n". */
|
|
221
|
+
export function pyCsvRows(text) {
|
|
222
|
+
const out = [];
|
|
223
|
+
let fields = [];
|
|
224
|
+
let field = "";
|
|
225
|
+
let st = "start_record";
|
|
226
|
+
const save = () => {
|
|
227
|
+
fields.push(field);
|
|
228
|
+
field = "";
|
|
229
|
+
};
|
|
230
|
+
const lines = text.match(/[^\n]*\n|[^\n]+$/g) || [];
|
|
231
|
+
for (const line of lines) {
|
|
232
|
+
for (const ch of [...line, null]) {
|
|
233
|
+
const eol = ch === null;
|
|
234
|
+
switch (st) {
|
|
235
|
+
case "start_record":
|
|
236
|
+
if (eol)
|
|
237
|
+
break;
|
|
238
|
+
if (ch === "\n" || ch === "\r") {
|
|
239
|
+
st = "eat_crnl";
|
|
240
|
+
break;
|
|
241
|
+
}
|
|
242
|
+
st = "start_field";
|
|
243
|
+
// falls through
|
|
244
|
+
case "start_field":
|
|
245
|
+
if (eol || ch === "\n" || ch === "\r") {
|
|
246
|
+
save();
|
|
247
|
+
st = eol ? "start_record" : "eat_crnl";
|
|
248
|
+
}
|
|
249
|
+
else if (ch === '"')
|
|
250
|
+
st = "in_quoted";
|
|
251
|
+
else if (ch === ",")
|
|
252
|
+
save();
|
|
253
|
+
else {
|
|
254
|
+
field += ch;
|
|
255
|
+
st = "in_field";
|
|
256
|
+
}
|
|
257
|
+
break;
|
|
258
|
+
case "in_field":
|
|
259
|
+
if (eol || ch === "\n" || ch === "\r") {
|
|
260
|
+
save();
|
|
261
|
+
st = eol ? "start_record" : "eat_crnl";
|
|
262
|
+
}
|
|
263
|
+
else if (ch === ",") {
|
|
264
|
+
save();
|
|
265
|
+
st = "start_field";
|
|
266
|
+
}
|
|
267
|
+
else
|
|
268
|
+
field += ch;
|
|
269
|
+
break;
|
|
270
|
+
case "in_quoted":
|
|
271
|
+
if (eol)
|
|
272
|
+
break;
|
|
273
|
+
if (ch === '"')
|
|
274
|
+
st = "quote_in_quoted";
|
|
275
|
+
else
|
|
276
|
+
field += ch;
|
|
277
|
+
break;
|
|
278
|
+
case "quote_in_quoted":
|
|
279
|
+
if (ch === '"') {
|
|
280
|
+
field += ch;
|
|
281
|
+
st = "in_quoted";
|
|
282
|
+
}
|
|
283
|
+
else if (ch === ",") {
|
|
284
|
+
save();
|
|
285
|
+
st = "start_field";
|
|
286
|
+
}
|
|
287
|
+
else if (eol || ch === "\n" || ch === "\r") {
|
|
288
|
+
save();
|
|
289
|
+
st = eol ? "start_record" : "eat_crnl";
|
|
290
|
+
}
|
|
291
|
+
else {
|
|
292
|
+
field += ch;
|
|
293
|
+
st = "in_field";
|
|
294
|
+
}
|
|
295
|
+
break;
|
|
296
|
+
case "eat_crnl":
|
|
297
|
+
if (ch === "\n" || ch === "\r")
|
|
298
|
+
break;
|
|
299
|
+
if (eol)
|
|
300
|
+
st = "start_record";
|
|
301
|
+
else
|
|
302
|
+
throw new DataError("new-line character seen in unquoted field - do you need to open the file with newline=''?", "api");
|
|
303
|
+
break;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
if (st === "start_record") {
|
|
307
|
+
out.push(fields);
|
|
308
|
+
fields = [];
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
// end of input inside a field (no line end after the last quoted field, or a quote left open)
|
|
312
|
+
if (field.length || st === "in_quoted") {
|
|
313
|
+
save();
|
|
314
|
+
out.push(fields);
|
|
315
|
+
}
|
|
316
|
+
return out;
|
|
317
|
+
}
|
|
318
|
+
export function csvGrid(data) {
|
|
319
|
+
let t;
|
|
320
|
+
try {
|
|
321
|
+
t = new TextDecoder("utf-8", { fatal: true }).decode(data).replace(/^/, "");
|
|
322
|
+
}
|
|
323
|
+
catch {
|
|
324
|
+
t = new TextDecoder("windows-1252").decode(data);
|
|
325
|
+
}
|
|
326
|
+
return pyCsvRows(t).map((r) => r.map((v) => clean(v, false, true)));
|
|
327
|
+
}
|
|
328
|
+
// ---------------------------------------------------------------------- table structure
|
|
329
|
+
function cells(r) {
|
|
330
|
+
const out = [];
|
|
331
|
+
r.forEach((v, i) => {
|
|
332
|
+
if (v !== null && v !== undefined && strip(String(v)))
|
|
333
|
+
out.push([i, v]);
|
|
334
|
+
});
|
|
335
|
+
return out;
|
|
336
|
+
}
|
|
337
|
+
function isnum(v) {
|
|
338
|
+
return v === DASH || NUM.test(strip(String(v)));
|
|
339
|
+
}
|
|
340
|
+
const YEAR = /^(1[89]|2[01])\d\d$/;
|
|
341
|
+
function yearHeader(r) {
|
|
342
|
+
const ne = cells(r);
|
|
343
|
+
if (!ne.length || (isnum(ne[0][1]) && ne[0][0] === 0))
|
|
344
|
+
return false;
|
|
345
|
+
const nums = ne.filter(([, v]) => isnum(v)).map(([, v]) => strip(v));
|
|
346
|
+
if (nums.length < 3 || !nums.every((v) => YEAR.test(v)))
|
|
347
|
+
return false;
|
|
348
|
+
const ys = nums.map(Number);
|
|
349
|
+
return ys.slice(1).every((b, i) => b > ys[i]);
|
|
350
|
+
}
|
|
351
|
+
const YEAR_ROW = /^(1[89]|2[01])\d\d[a-z]{0,2}$/;
|
|
352
|
+
/** A row of values: a label or year and at least one number, or a year alone (its values all published as dashes). */
|
|
353
|
+
function isData(r) {
|
|
354
|
+
const ne = cells(r);
|
|
355
|
+
if (ne.length && ne[0][0] === 0 && YEAR_ROW.test(strip(String(ne[0][1]))) && ne.slice(1).every(([, v]) => isnum(v)))
|
|
356
|
+
return true;
|
|
357
|
+
return ne.length >= 2 && ne.slice(1).some(([, v]) => isnum(v)) && !yearHeader(r);
|
|
358
|
+
}
|
|
359
|
+
export function label(v) {
|
|
360
|
+
if (v === DASH)
|
|
361
|
+
return "";
|
|
362
|
+
return words(String(v).replace(/ /g, " "));
|
|
363
|
+
}
|
|
364
|
+
const joinCells = (c) => c.map(([, v]) => v).join(" ");
|
|
365
|
+
export function tidy(grid, merges = [], headerRow = null) {
|
|
366
|
+
const n = grid.length;
|
|
367
|
+
let hrs, start, titles, firstSection;
|
|
368
|
+
if (headerRow !== null && headerRow !== undefined && headerRow !== "") {
|
|
369
|
+
hrs = (Array.isArray(headerRow) ? headerRow : [headerRow]).map((h) => Math.trunc(Number(h)) - 1);
|
|
370
|
+
if (!hrs.length || Math.min(...hrs) < 0 || Math.max(...hrs) >= n) {
|
|
371
|
+
const shown = Array.isArray(headerRow) ? `[${headerRow.join(", ")}]` : String(headerRow);
|
|
372
|
+
throw new DataError(`header_row ${shown} is outside the sheet's ${n} rows`, "bad_request");
|
|
373
|
+
}
|
|
374
|
+
start = Math.max(...hrs) + 1;
|
|
375
|
+
titles = [];
|
|
376
|
+
for (let i = 0; i < Math.min(...hrs); i++) {
|
|
377
|
+
const c = cells(grid[i]);
|
|
378
|
+
if (c.length === 1)
|
|
379
|
+
titles.push(label(c[0][1]));
|
|
380
|
+
}
|
|
381
|
+
firstSection = "";
|
|
382
|
+
}
|
|
383
|
+
else {
|
|
384
|
+
const s = grid.findIndex(isData);
|
|
385
|
+
if (s < 0) { // a notes sheet: one text column
|
|
386
|
+
const rows = [];
|
|
387
|
+
grid.forEach((r, i) => {
|
|
388
|
+
const t = label(joinCells(cells(r)));
|
|
389
|
+
if (t)
|
|
390
|
+
rows.push(new Map([["row", i + 1], ["text", t]]));
|
|
391
|
+
});
|
|
392
|
+
return [rows, [], rows.length ? String(rows[0].get("text")) : ""];
|
|
393
|
+
}
|
|
394
|
+
start = s;
|
|
395
|
+
hrs = [];
|
|
396
|
+
titles = [];
|
|
397
|
+
firstSection = "";
|
|
398
|
+
for (let i = 0; i < start; i++) {
|
|
399
|
+
const c = cells(grid[i]);
|
|
400
|
+
const nums = c.some(([, v]) => isnum(v)) && !yearHeader(grid[i]);
|
|
401
|
+
const idx = new Set(c.map(([j]) => j));
|
|
402
|
+
if (c.length === 2 && !nums && idx.size === 2 && idx.has(0) && idx.has(1) && hrs.some((h) => cells(grid[h]).length >= 3)) {
|
|
403
|
+
firstSection = label(joinCells(c)); // "Year 2017 | Amount in Billions"
|
|
404
|
+
}
|
|
405
|
+
else if (c.length >= 2) {
|
|
406
|
+
hrs.push(i);
|
|
407
|
+
firstSection = "";
|
|
408
|
+
}
|
|
409
|
+
else if (c.length === 1) {
|
|
410
|
+
if (hrs.length)
|
|
411
|
+
firstSection = label(c[0][1]);
|
|
412
|
+
else if (c[0][0] > 0 && titles.length)
|
|
413
|
+
hrs.push(i); // a group label over columns ("Projected")
|
|
414
|
+
else
|
|
415
|
+
titles.push(label(c[0][1]));
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
let width = 0;
|
|
420
|
+
for (let i = start; i < n; i++)
|
|
421
|
+
width = Math.max(width, grid[i].length);
|
|
422
|
+
for (const h of hrs)
|
|
423
|
+
width = Math.max(width, grid[h].length);
|
|
424
|
+
const columnNames = (block) => {
|
|
425
|
+
const head = new Map();
|
|
426
|
+
for (const h of block) {
|
|
427
|
+
const r = grid[h].map((v) => (label(v) ? v : ""));
|
|
428
|
+
while (r.length < width)
|
|
429
|
+
r.push("");
|
|
430
|
+
head.set(h, r);
|
|
431
|
+
}
|
|
432
|
+
for (const [r1, c1, r2, c2] of merges) { // spread merged header cells over their columns and rows
|
|
433
|
+
const top = r1 < n && c1 < grid[r1].length ? grid[r1][c1] : "";
|
|
434
|
+
for (const h of block) {
|
|
435
|
+
if (r1 <= h && h <= r2) {
|
|
436
|
+
const hr = head.get(h);
|
|
437
|
+
for (let c = c1; c <= Math.min(c2, width - 1); c++)
|
|
438
|
+
if (!hr[c])
|
|
439
|
+
hr[c] = top;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
if (block.length) { // a label over year columns ("Projected") covers the years to its right
|
|
444
|
+
const last = head.get(block[block.length - 1]);
|
|
445
|
+
for (const h of block.slice(0, -1)) {
|
|
446
|
+
const hr = head.get(h);
|
|
447
|
+
for (let c = 1; c < width; c++) {
|
|
448
|
+
if (!hr[c] && hr[c - 1] && YEAR.test(strip(String(last[c]))) && YEAR.test(strip(String(last[c - 1]))))
|
|
449
|
+
hr[c] = hr[c - 1];
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
const out = [];
|
|
454
|
+
const seen = new Set();
|
|
455
|
+
for (let c = 0; c < width; c++) {
|
|
456
|
+
const parts = [];
|
|
457
|
+
for (const h of block) {
|
|
458
|
+
const p = label(head.get(h)[c]);
|
|
459
|
+
if (p && (!parts.length || parts[parts.length - 1] !== p))
|
|
460
|
+
parts.push(p);
|
|
461
|
+
}
|
|
462
|
+
let nm = parts.join(" ") || `col${c + 1}`;
|
|
463
|
+
nm = !seen.has(nm) ? nm : `${nm}_col${c + 1}`;
|
|
464
|
+
seen.add(nm);
|
|
465
|
+
out.push(nm);
|
|
466
|
+
}
|
|
467
|
+
const txt = new Set();
|
|
468
|
+
for (const h of block)
|
|
469
|
+
txt.add(JSON.stringify(head.get(h).map(label).filter(Boolean)));
|
|
470
|
+
for (const h of block)
|
|
471
|
+
txt.add(JSON.stringify(grid[h].map(label).filter(Boolean)));
|
|
472
|
+
return [out, txt];
|
|
473
|
+
};
|
|
474
|
+
let [names, headerText] = columnNames(hrs);
|
|
475
|
+
const order = [...names];
|
|
476
|
+
const rows = [];
|
|
477
|
+
let pending = [], block = [], section = firstSection;
|
|
478
|
+
const notes = [];
|
|
479
|
+
for (let i = start; i < n; i++) {
|
|
480
|
+
const r = grid[i];
|
|
481
|
+
const c = cells(r);
|
|
482
|
+
if (!c.length)
|
|
483
|
+
continue;
|
|
484
|
+
if (headerText.has(JSON.stringify(c.map(([, v]) => label(v)))))
|
|
485
|
+
continue; // a header repeated on a continued page
|
|
486
|
+
const blankRow = c.length === 1 && c[0][0] === 0 && rows.length > 0 && String(c[0][1]).replace(/ /g, " ").startsWith(" ");
|
|
487
|
+
if (!isData(r) && !blankRow && (c.length === 1 || !c.some(([, v]) => isnum(v)))) {
|
|
488
|
+
if (c.length >= 3)
|
|
489
|
+
block.push(i); // a new header inside the sheet (a second block of columns)
|
|
490
|
+
else
|
|
491
|
+
pending.push(label(joinCells(c)));
|
|
492
|
+
continue;
|
|
493
|
+
}
|
|
494
|
+
if (block.length) {
|
|
495
|
+
let txt;
|
|
496
|
+
[names, txt] = columnNames(block);
|
|
497
|
+
for (const t of txt)
|
|
498
|
+
headerText.add(t);
|
|
499
|
+
for (const nm of names)
|
|
500
|
+
if (!order.includes(nm))
|
|
501
|
+
order.push(nm);
|
|
502
|
+
block = [];
|
|
503
|
+
}
|
|
504
|
+
if (pending.length) {
|
|
505
|
+
section = pending[pending.length - 1];
|
|
506
|
+
notes.push(...pending.slice(0, -1));
|
|
507
|
+
pending = [];
|
|
508
|
+
}
|
|
509
|
+
const rec = new Map();
|
|
510
|
+
for (let j = 0; j < width; j++)
|
|
511
|
+
rec.set(names[j], j < r.length ? r[j] : "");
|
|
512
|
+
rows.push([i, section, rec]);
|
|
513
|
+
}
|
|
514
|
+
notes.push(...pending);
|
|
515
|
+
names = order;
|
|
516
|
+
const full = rows.map(([i, sec, rec]) => [i, sec, new Map(names.map((nm) => [nm, rec.has(nm) ? rec.get(nm) : ""]))]);
|
|
517
|
+
let used = names.filter((nm) => full.some(([, , rec]) => label(rec.get(nm))));
|
|
518
|
+
if (!used.length)
|
|
519
|
+
used = names;
|
|
520
|
+
const hasSection = full.some(([, s]) => s);
|
|
521
|
+
const first = used.length ? used[0] : null;
|
|
522
|
+
let lab = null;
|
|
523
|
+
for (const cand of used.slice(0, 2)) {
|
|
524
|
+
if (full.some(([, , rec]) => String(rec.get(cand)).replace(/ /g, " ").startsWith(" ") && label(rec.get(cand)))) {
|
|
525
|
+
lab = cand;
|
|
526
|
+
break;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
const vals = first ? full.map(([, , rec]) => strip(String(rec.get(first)))).filter(Boolean) : [];
|
|
530
|
+
// a first column of years with footnote letters ("2033b"): add a plain `year` column
|
|
531
|
+
const yearcol = vals.length && vals.every((v) => /^(1[89]|2[01])\d\d[a-z ,]*$/.test(v)) && vals.some((v) => v.length > 4) ? first : null;
|
|
532
|
+
const out = [];
|
|
533
|
+
let stack = [];
|
|
534
|
+
let group = "";
|
|
535
|
+
for (const [i, sec, rec] of full) {
|
|
536
|
+
const o = new Map([["row", i + 1]]);
|
|
537
|
+
if (hasSection)
|
|
538
|
+
o.set("section", strip(sec.replace(/:+$/, "")));
|
|
539
|
+
for (const nm of used) {
|
|
540
|
+
const v = rec.get(nm);
|
|
541
|
+
if (nm === lab) {
|
|
542
|
+
const raw = v === DASH ? "" : String(v).replace(/ /g, " ");
|
|
543
|
+
const ind = raw.length - raw.replace(/^ +/, "").length;
|
|
544
|
+
const lb = label(raw);
|
|
545
|
+
const scope = JSON.stringify([sec, lab !== used[0] ? rec.get(used[0]) : ""]);
|
|
546
|
+
stack = stack.filter(([s, k]) => s === scope && k < ind);
|
|
547
|
+
const parent = stack.length ? stack[stack.length - 1][2] : "";
|
|
548
|
+
stack.push([scope, ind, lb]);
|
|
549
|
+
if (ind === 0 && lab === used[0])
|
|
550
|
+
group = lb;
|
|
551
|
+
o.set(nm, lb);
|
|
552
|
+
o.set("indent", ind);
|
|
553
|
+
if (lab === used[0])
|
|
554
|
+
o.set("group", ind ? group : lb);
|
|
555
|
+
o.set("parent", parent);
|
|
556
|
+
}
|
|
557
|
+
else
|
|
558
|
+
o.set(nm, typeof v === "string" ? label(v) : v);
|
|
559
|
+
if (nm === yearcol)
|
|
560
|
+
o.set("year", label(v).slice(0, 4));
|
|
561
|
+
}
|
|
562
|
+
out.push(o);
|
|
563
|
+
}
|
|
564
|
+
return [out, [...titles, ...notes], titles.length ? titles[0] : ""];
|
|
565
|
+
}
|
|
566
|
+
const YEAR_COL = /^(?:([A-Za-z][A-Za-z ]*?) )?((?:1[89]|2[01])\d\d)(?: ([A-Za-z][A-Za-z ]*))?$/;
|
|
567
|
+
/** [year, label] of a year column ("2024", "2025 Projected", "Projected 2025"), else null. */
|
|
568
|
+
function yearCol(name) {
|
|
569
|
+
const m = YEAR_COL.exec(name);
|
|
570
|
+
if (!m)
|
|
571
|
+
return null;
|
|
572
|
+
const lb = [m[1], m[3]].filter(Boolean).join(" ");
|
|
573
|
+
return lb.split(/\s+/).filter(Boolean).length <= 2 ? [m[2], lb] : null;
|
|
574
|
+
}
|
|
575
|
+
/** Melt year columns ("2024", "2025 Projected", "Projected 2025") into year / value rows. */
|
|
576
|
+
export function longLayout(rows) {
|
|
577
|
+
if (!rows.length)
|
|
578
|
+
return [rows, true];
|
|
579
|
+
const ycols = new Map();
|
|
580
|
+
for (const k of rows[0].keys()) {
|
|
581
|
+
const y = yearCol(k);
|
|
582
|
+
if (y)
|
|
583
|
+
ycols.set(k, y);
|
|
584
|
+
}
|
|
585
|
+
if (!ycols.size)
|
|
586
|
+
return [rows, false];
|
|
587
|
+
const ids = [...rows[0].keys()].filter((k) => !ycols.has(k));
|
|
588
|
+
const labelled = [...ycols.values()].some(([, lb]) => lb);
|
|
589
|
+
const out = [];
|
|
590
|
+
for (const r of rows) {
|
|
591
|
+
for (const [k, [yr, lb]] of ycols) {
|
|
592
|
+
const o = new Map(ids.map((i) => [i, r.get(i)]));
|
|
593
|
+
o.set("year", yr);
|
|
594
|
+
if (labelled)
|
|
595
|
+
o.set("year_label", lb);
|
|
596
|
+
o.set("value", r.get(k));
|
|
597
|
+
out.push(o);
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
return [out, true];
|
|
601
|
+
}
|
|
602
|
+
// ---------------------------------------------------------------------- ZIP contents
|
|
603
|
+
const stem = (n) => {
|
|
604
|
+
const b = basename(n);
|
|
605
|
+
const i = b.lastIndexOf(".");
|
|
606
|
+
return (i > 0 ? b.slice(0, i) : b).toLowerCase();
|
|
607
|
+
};
|
|
608
|
+
/** [member, kind] of the tables in a ZIP: .xlsx members, and .csv members that have no .xlsx twin. */
|
|
609
|
+
export function zipTables(data) {
|
|
610
|
+
const z = members(data);
|
|
611
|
+
const nms = Object.keys(z).filter((n) => !n.endsWith("/"));
|
|
612
|
+
const stems = new Set(nms.filter((n) => n.toLowerCase().endsWith(".xlsx")).map(stem));
|
|
613
|
+
const key = (s) => [s.split("/").length - 1, s.toLowerCase()];
|
|
614
|
+
const sorted = [...nms].sort((a, b) => {
|
|
615
|
+
const [ca, la] = key(a), [cb, lb] = key(b);
|
|
616
|
+
return ca !== cb ? ca - cb : la < lb ? -1 : la > lb ? 1 : 0;
|
|
617
|
+
});
|
|
618
|
+
const out = [];
|
|
619
|
+
for (const n of sorted) {
|
|
620
|
+
const low = n.toLowerCase();
|
|
621
|
+
if (low.endsWith(".xlsx"))
|
|
622
|
+
out.push([n, "xlsx"]);
|
|
623
|
+
else if (low.endsWith(".csv") && !stems.has(stem(n)))
|
|
624
|
+
out.push([n, "csv"]);
|
|
625
|
+
}
|
|
626
|
+
return out;
|
|
627
|
+
}
|
|
628
|
+
export function norm(s) {
|
|
629
|
+
return words(basename(String(s)).replace(/\.(xlsx|csv)$/i, "").toLowerCase());
|
|
630
|
+
}
|
|
631
|
+
export function pickMember(tables, want, lbl) {
|
|
632
|
+
if (!tables.length)
|
|
633
|
+
throw new DataError(`${lbl} holds no .xlsx or .csv table`, "api");
|
|
634
|
+
if (want === null || want === undefined || want === "") {
|
|
635
|
+
if (tables.length === 1)
|
|
636
|
+
return tables[0];
|
|
637
|
+
throw new DataError(`${lbl} holds ${tables.length} tables: name one with "table" (` +
|
|
638
|
+
tables.slice(0, 40).map(([m]) => norm(m)).join("; ") + ")", "bad_request");
|
|
639
|
+
}
|
|
640
|
+
const q = norm(want);
|
|
641
|
+
const exact = tables.filter((t) => norm(t[0]) === q);
|
|
642
|
+
if (exact.length)
|
|
643
|
+
return exact[0];
|
|
644
|
+
const pre = tables.filter((t) => norm(t[0]).startsWith(q) && (norm(t[0]).length === q.length || norm(t[0])[q.length] === " "));
|
|
645
|
+
if (pre.length === 1)
|
|
646
|
+
return pre[0];
|
|
647
|
+
const sub = tables.filter((t) => norm(t[0]).includes(q));
|
|
648
|
+
if (sub.length === 1)
|
|
649
|
+
return sub[0];
|
|
650
|
+
const hits = pre.length ? pre : sub;
|
|
651
|
+
if (hits.length) {
|
|
652
|
+
throw new DataError(`table ${repr(want)} matches ${hits.length} files in ${lbl}: ` +
|
|
653
|
+
hits.slice(0, 20).map(([m]) => norm(m)).join("; ") + ". Give more of the name.", "bad_request");
|
|
654
|
+
}
|
|
655
|
+
throw new DataError(`no table ${repr(want)} in ${lbl} (tables: ` + tables.slice(0, 40).map(([m]) => norm(m)).join("; ") + ")", "bad_request");
|
|
656
|
+
}
|
|
657
|
+
export function readMember(data, member, kind, sheet) {
|
|
658
|
+
const raw = members(data)[member];
|
|
659
|
+
if (kind === "csv") {
|
|
660
|
+
if (!(sheet === null || sheet === undefined || sheet === "" || sheet === 1 || sheet === "1" || sheet === true)) {
|
|
661
|
+
throw new DataError(`${basename(member)} is a CSV file: it has no sheets`, "bad_request");
|
|
662
|
+
}
|
|
663
|
+
return ["", csvGrid(raw), [], 0];
|
|
664
|
+
}
|
|
665
|
+
return xlsxGrid(raw, sheet);
|
|
666
|
+
}
|
|
667
|
+
export function listZip(data) {
|
|
668
|
+
const rows = [];
|
|
669
|
+
const z = members(data);
|
|
670
|
+
const firstLabel = (g) => {
|
|
671
|
+
for (const r of g)
|
|
672
|
+
for (const [, v] of cells(r))
|
|
673
|
+
return label(v);
|
|
674
|
+
return "";
|
|
675
|
+
};
|
|
676
|
+
for (const [member, kind] of zipTables(data)) {
|
|
677
|
+
const raw = z[member];
|
|
678
|
+
if (kind === "csv") {
|
|
679
|
+
const g = csvGrid(raw);
|
|
680
|
+
rows.push(new Map([["table", norm(member)], ["sheet", ""], ["title", firstLabel(g)], ["format", "csv"],
|
|
681
|
+
["member", member], ["grid_rows", g.length]]));
|
|
682
|
+
continue;
|
|
683
|
+
}
|
|
684
|
+
const sheets = xlsxSheets(raw);
|
|
685
|
+
for (const name of sheetNames(raw)) {
|
|
686
|
+
const g = sheets[name];
|
|
687
|
+
if (!g)
|
|
688
|
+
continue;
|
|
689
|
+
const cl = g.map((r) => r.map((v) => clean(v)));
|
|
690
|
+
rows.push(new Map([["table", norm(member)], ["sheet", name], ["title", firstLabel(cl)], ["format", "xlsx"],
|
|
691
|
+
["member", member], ["grid_rows", g.length]]));
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
return rows;
|
|
695
|
+
}
|
|
696
|
+
// ---------------------------------------------------------------------- CMS pages
|
|
697
|
+
const HTML_ENT = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", ndash: "–",
|
|
698
|
+
mdash: "—", lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”", hellip: "…", copy: "©",
|
|
699
|
+
reg: "®", trade: "™", middot: "·", bull: "•", eacute: "é", shy: "", sect: "§" };
|
|
700
|
+
/** Python html.unescape for numeric references and the common named ones. */
|
|
701
|
+
export function unescapeHtml(s) {
|
|
702
|
+
return s.replace(/&(#[xX][0-9a-fA-F]+|#\d+|[A-Za-z][A-Za-z0-9]*);/g, (all, e) => {
|
|
703
|
+
if (e[0] === "#") {
|
|
704
|
+
const cp = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
|
|
705
|
+
return cp > 0 && cp <= 0x10ffff && !(cp >= 0xd800 && cp <= 0xdfff) ? String.fromCodePoint(cp) : "�";
|
|
706
|
+
}
|
|
707
|
+
return HTML_ENT[e] ?? all;
|
|
708
|
+
});
|
|
709
|
+
}
|
|
710
|
+
export function pageLinks(html) {
|
|
711
|
+
const out = [];
|
|
712
|
+
for (const m of html.matchAll(/<a\b[^>]*\bhref="([^"]+)"[^>]*>(.*?)<\/a>/gis)) {
|
|
713
|
+
out.push([unescapeHtml(m[1]), label(unescapeHtml(m[2].replace(/<[^>]+>/g, " ")))]);
|
|
714
|
+
}
|
|
715
|
+
return out;
|
|
716
|
+
}
|
|
717
|
+
// ---------------------------------------------------------------------- output
|
|
718
|
+
/** Rows as package.write takes them: objects, with the columns in the rows' own key order (withColumns). */
|
|
719
|
+
export function asTable(rows) {
|
|
720
|
+
const cols = [];
|
|
721
|
+
const seen = new Set();
|
|
722
|
+
for (const r of rows)
|
|
723
|
+
for (const k of r.keys())
|
|
724
|
+
if (!seen.has(k))
|
|
725
|
+
(seen.add(k), cols.push(k));
|
|
726
|
+
return withColumns(rows.map((r) => Object.fromEntries(r)), cols);
|
|
727
|
+
}
|
|
728
|
+
//# sourceMappingURL=_actuaries_read.js.map
|