data-analyst-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -2
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +716 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +24 -0
- package/dist/report/scan_secrets.js +265 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +247 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +54 -4
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +170 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,869 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* usda-fna Program Data: FNA's XLSX workbooks (SNAP, WIC, child nutrition) read into tidy rows (the Python
|
|
3
|
+
* adapter's "Program Data (XLSX)" section, unchanged in meaning).
|
|
4
|
+
*
|
|
5
|
+
* The listing pages (/pd/supplemental-nutrition-assistance-program-snap, /pd/wic-program, /pd/child-nutrition-tables)
|
|
6
|
+
* link one workbook per table, named <stem>-<release>.xlsx (e.g. snap-persons-10.xlsx); the release number changes,
|
|
7
|
+
* so the adapter reads the current name from the listing page. Layouts (verified 2026-10-30):
|
|
8
|
+
* annual national annual summary: "Fiscal Year" header, one row per year (1969-)
|
|
9
|
+
* monthly national monthly: ANNUAL SUMMARY (FY rows) then MONTHLY DATA ("FY 2023" headings, "Oct 2022" rows)
|
|
10
|
+
* state_fy one measure by state, five fiscal years ("FY 2021" ... columns); WIC indents its tribal parts
|
|
11
|
+
* state_latest one measure by state for three months (a year ago, last month, this month) and percent changes
|
|
12
|
+
* snap_zip snap-zip-fy69tocurrent-NN.zip: FYyy.xlsx (FY 2020-) by region sheet, state blocks of months
|
|
13
|
+
* wic_agency wicagencies<FY>ytd.xlsx: sixteen sheets (one per measure), agencies x months
|
|
14
|
+
* Gates (a mismatch drops the data and says so, never guesses): the title row must match the table, every column used
|
|
15
|
+
* must carry the label endpoints.json gives it, a value under an unlabeled visible column fails the table (hidden
|
|
16
|
+
* columns are skipped), months must fall in their fiscal year in order, and month headers of state_latest tables
|
|
17
|
+
* must be a year ago / last month / this month. Agency sums are compared with the printed totals (a note, not a fix).
|
|
18
|
+
* The workbook reader here keeps what core's xlsxSheets drops: hidden columns and sheet row numbers (empty rows kept).
|
|
19
|
+
*/
|
|
20
|
+
import { unzipSync } from "fflate";
|
|
21
|
+
import { INCOMPLETE } from "../core/generic-rest.js";
|
|
22
|
+
import { pyFloatRepr } from "../core/py.js";
|
|
23
|
+
// ---------------------------------------------------------------------- Python text forms
|
|
24
|
+
/** Python's whitespace (str.isspace / re \s), not JS's (which adds U+FEFF and lacks U+001C-U+001F, U+0085). */
|
|
25
|
+
const WS = "\\t-\\r \\x1c-\\x1f\\x85\\xa0\\u1680\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000";
|
|
26
|
+
const WS_RUN = new RegExp(`[${WS}]+`, "g");
|
|
27
|
+
const WS_EDGE = new RegExp(`^[${WS}]+|[${WS}]+$`, "g");
|
|
28
|
+
export function pyStrip(s) {
|
|
29
|
+
return s.replace(WS_EDGE, "");
|
|
30
|
+
}
|
|
31
|
+
/** repr() of a str, int, float, None, bool or list (what Python prints for {x!r} and {list}). */
|
|
32
|
+
export function pyRepr(v) {
|
|
33
|
+
if (v === null || v === undefined)
|
|
34
|
+
return "None";
|
|
35
|
+
if (v === true)
|
|
36
|
+
return "True";
|
|
37
|
+
if (v === false)
|
|
38
|
+
return "False";
|
|
39
|
+
if (typeof v === "number")
|
|
40
|
+
return Number.isInteger(v) ? String(v) : pyFloatRepr(v);
|
|
41
|
+
if (typeof v === "string") {
|
|
42
|
+
const q = v.includes("'") && !v.includes('"') ? '"' : "'";
|
|
43
|
+
let s = v.replace(/\\/g, "\\\\").replace(/\n/g, "\\n").replace(/\r/g, "\\r").replace(/\t/g, "\\t")
|
|
44
|
+
.replace(/[\x00-\x1f\x7f]/g, (c) => "\\x" + c.charCodeAt(0).toString(16).padStart(2, "0"));
|
|
45
|
+
if (q === "'")
|
|
46
|
+
s = s.replace(/'/g, "\\'");
|
|
47
|
+
return q + s + q;
|
|
48
|
+
}
|
|
49
|
+
if (Array.isArray(v))
|
|
50
|
+
return "[" + v.map(pyRepr).join(", ") + "]";
|
|
51
|
+
if (typeof v === "object")
|
|
52
|
+
return "{" + Object.entries(v).map(([k, x]) => `${pyRepr(k)}: ${pyRepr(x)}`).join(", ") + "}";
|
|
53
|
+
return String(v);
|
|
54
|
+
}
|
|
55
|
+
/** float(s) as Python parses text (None when it would raise ValueError/TypeError). */
|
|
56
|
+
const PY_FLOAT = /^[+-]?(?:(?:\d(?:_?\d)*(?:\.(?:\d(?:_?\d)*)?)?|\.\d(?:_?\d)*)(?:[eE][+-]?\d(?:_?\d)*)?|inf(?:inity)?|nan)$/i;
|
|
57
|
+
export function pyFloat(v) {
|
|
58
|
+
if (typeof v === "number")
|
|
59
|
+
return v;
|
|
60
|
+
if (typeof v !== "string")
|
|
61
|
+
return null;
|
|
62
|
+
const s = pyStrip(v);
|
|
63
|
+
if (!PY_FLOAT.test(s))
|
|
64
|
+
return null;
|
|
65
|
+
const t = s.replace(/_/g, "").toLowerCase();
|
|
66
|
+
if (t.endsWith("inf") || t.endsWith("infinity"))
|
|
67
|
+
return t.startsWith("-") ? -Infinity : Infinity;
|
|
68
|
+
if (t.endsWith("nan"))
|
|
69
|
+
return NaN;
|
|
70
|
+
return Number(t);
|
|
71
|
+
}
|
|
72
|
+
/** sum() of floats as CPython 3.12+ adds them (Neumaier compensated summation). */
|
|
73
|
+
export function pySum(xs) {
|
|
74
|
+
if (!xs.length)
|
|
75
|
+
return 0;
|
|
76
|
+
let f = xs[0], c = 0;
|
|
77
|
+
for (let i = 1; i < xs.length; i++) {
|
|
78
|
+
const x = xs[i];
|
|
79
|
+
const t = f + x;
|
|
80
|
+
if (Math.abs(f) >= Math.abs(x))
|
|
81
|
+
c += (f - t) + x;
|
|
82
|
+
else
|
|
83
|
+
c += (x - t) + f;
|
|
84
|
+
f = t;
|
|
85
|
+
}
|
|
86
|
+
if (c && Number.isFinite(c))
|
|
87
|
+
f += c;
|
|
88
|
+
return f;
|
|
89
|
+
}
|
|
90
|
+
/** f"{x:,.0f}": rounded half to even, with thousands separators. */
|
|
91
|
+
export function fmt0(x) {
|
|
92
|
+
if (!Number.isFinite(x))
|
|
93
|
+
return x > 0 ? "inf" : x < 0 ? "-inf" : "nan";
|
|
94
|
+
const fl = Math.floor(x);
|
|
95
|
+
let r = x - fl === 0.5 ? (fl % 2 === 0 ? fl : fl + 1) : Math.round(x);
|
|
96
|
+
const neg = r < 0 || (r === 0 && (x < 0 || Object.is(x, -0)));
|
|
97
|
+
r = Math.abs(r);
|
|
98
|
+
return (neg ? "-" : "") + BigInt(r).toLocaleString("en-US");
|
|
99
|
+
}
|
|
100
|
+
/** f"{n:,}" of an int. */
|
|
101
|
+
export const comma = (n) => n.toLocaleString("en-US");
|
|
102
|
+
export const pad2 = (n) => String(n).padStart(2, "0");
|
|
103
|
+
/** f"{f:.15g}", and f"{f:.15f}".rstrip("0").rstrip(".") where %g would write an exponent. */
|
|
104
|
+
function num15(f) {
|
|
105
|
+
if (f === 0)
|
|
106
|
+
return Object.is(f, -0) ? "-0" : "0";
|
|
107
|
+
const exp = Number(f.toExponential(14).split("e")[1]);
|
|
108
|
+
if (exp < -4 || exp >= 15) {
|
|
109
|
+
const s = Math.abs(f) >= 1e21 ? BigInt(f).toString() + ".000000000000000" : f.toFixed(15);
|
|
110
|
+
return s.replace(/0+$/, "").replace(/\.$/, "");
|
|
111
|
+
}
|
|
112
|
+
let s = f.toPrecision(15);
|
|
113
|
+
if (s.includes("."))
|
|
114
|
+
s = s.replace(/0+$/, "").replace(/\.$/, "");
|
|
115
|
+
return s;
|
|
116
|
+
}
|
|
117
|
+
export class LayoutError extends Error {
|
|
118
|
+
}
|
|
119
|
+
// ---------------------------------------------------------------------- XLSX reader
|
|
120
|
+
const ENT = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" };
|
|
121
|
+
function unxml(s) {
|
|
122
|
+
return s.replace(/&(#x[0-9a-fA-F]+|#\d+|\w+);/g, (all, e) => e[0] === "#" ? String.fromCodePoint(e[1] === "x" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10)) : ENT[e] ?? all);
|
|
123
|
+
}
|
|
124
|
+
function attrs(tag) {
|
|
125
|
+
const out = {};
|
|
126
|
+
for (const m of tag.matchAll(/([\w:.-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g))
|
|
127
|
+
out[m[1]] = unxml(m[2] ?? m[3]);
|
|
128
|
+
return out;
|
|
129
|
+
}
|
|
130
|
+
const texts = (xml) => [...xml.matchAll(/<t(?:\s[^>]*)?>([\s\S]*?)<\/t>/g)].map((m) => unxml(m[1])).join("");
|
|
131
|
+
export function zipMembers(body) {
|
|
132
|
+
const names = [];
|
|
133
|
+
unzipSync(body, { filter: (f) => (names.push(f.name), false) });
|
|
134
|
+
return names;
|
|
135
|
+
}
|
|
136
|
+
export function zipRead(body, name) {
|
|
137
|
+
return unzipSync(body, { filter: (f) => f.name === name })[name];
|
|
138
|
+
}
|
|
139
|
+
/** [(sheet name, rows, hidden column indexes)]; a row is a list of cell texts (null for empty), in sheet row
|
|
140
|
+
* order with empty rows kept, so rows[i] is sheet row i + 1. */
|
|
141
|
+
export function readBook(body) {
|
|
142
|
+
let z;
|
|
143
|
+
try {
|
|
144
|
+
z = unzipSync(body);
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
throw new LayoutError("not an XLSX file (the server may have returned an HTML page)");
|
|
148
|
+
}
|
|
149
|
+
const read = (n) => {
|
|
150
|
+
if (!z[n])
|
|
151
|
+
throw new LayoutError(`not an XLSX file (no ${n} in the zip)`);
|
|
152
|
+
return new TextDecoder().decode(z[n]).replace(/\r\n?/g, "\n");
|
|
153
|
+
};
|
|
154
|
+
const shared = z["xl/sharedStrings.xml"]
|
|
155
|
+
? [...read("xl/sharedStrings.xml").matchAll(/<si(?:\s[^>]*)?>([\s\S]*?)<\/si>/g)].map((m) => texts(m[1])) : [];
|
|
156
|
+
const target = {};
|
|
157
|
+
for (const m of read("xl/_rels/workbook.xml.rels").matchAll(/<Relationship\b[^>]*>/g)) {
|
|
158
|
+
const a = attrs(m[0]);
|
|
159
|
+
target[a.Id] = a.Target;
|
|
160
|
+
}
|
|
161
|
+
const out = [];
|
|
162
|
+
for (const m of read("xl/workbook.xml").matchAll(/<sheet\b[^>]*>/g)) {
|
|
163
|
+
const a = attrs(m[0]);
|
|
164
|
+
let p = target[a["r:id"]] ?? "";
|
|
165
|
+
p = p.startsWith("/") ? p.replace(/^\/+/, "") : "xl/" + p;
|
|
166
|
+
const xml = read(p);
|
|
167
|
+
const hidden = new Set();
|
|
168
|
+
for (const c of xml.matchAll(/<col\b[^>]*>/g)) {
|
|
169
|
+
const ca = attrs(c[0]);
|
|
170
|
+
if (ca.hidden === "1" || ca.hidden === "true") {
|
|
171
|
+
for (let i = parseInt(ca.min, 10) - 1; i < parseInt(ca.max, 10); i++)
|
|
172
|
+
hidden.add(i);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
const rows = [];
|
|
176
|
+
for (const rm of xml.matchAll(/<row\b([^>]*?)(?:\/>|>([\s\S]*?)<\/row>)/g)) {
|
|
177
|
+
const vals = new Map();
|
|
178
|
+
for (const cm of (rm[2] || "").matchAll(/<c\b([^>]*?)(?:\/>|>([\s\S]*?)<\/c>)/g)) {
|
|
179
|
+
const ca = attrs(cm[1]);
|
|
180
|
+
let col = 0;
|
|
181
|
+
for (const ch of (/^[A-Z]+/.exec(ca.r || "") || ["A"])[0])
|
|
182
|
+
col = col * 26 + ch.charCodeAt(0) - 64;
|
|
183
|
+
const inner = cm[2] || "";
|
|
184
|
+
const v = /<v(?:\s[^>]*)?(?:\/>|>([\s\S]*?)<\/v>)/.exec(inner);
|
|
185
|
+
const t = ca.t;
|
|
186
|
+
if (t === "s" && v)
|
|
187
|
+
vals.set(col - 1, shared[parseInt(v[1] ?? "", 10)] ?? null);
|
|
188
|
+
else if (t === "inlineStr")
|
|
189
|
+
vals.set(col - 1, texts(inner));
|
|
190
|
+
else if (v)
|
|
191
|
+
vals.set(col - 1, v[1] ? unxml(v[1]) : null);
|
|
192
|
+
}
|
|
193
|
+
const ra = attrs(rm[1]);
|
|
194
|
+
const n = parseInt(ra.r || "", 10) || rows.length + 1;
|
|
195
|
+
while (rows.length < n - 1)
|
|
196
|
+
rows.push([]);
|
|
197
|
+
const width = vals.size ? Math.max(...vals.keys()) + 1 : 0;
|
|
198
|
+
rows.push(Array.from({ length: width }, (_, i) => vals.get(i) ?? null));
|
|
199
|
+
}
|
|
200
|
+
out.push([a.name, rows, hidden]);
|
|
201
|
+
}
|
|
202
|
+
return out;
|
|
203
|
+
}
|
|
204
|
+
export function norm(x) {
|
|
205
|
+
return pyStrip(String(x !== null && x !== undefined ? x : "").replace(/\xa0/g, " ").replace(WS_RUN, " "));
|
|
206
|
+
}
|
|
207
|
+
const PD_MISSING = ["--", "-", "—", "–", "N/A", "NA", "#DIV/0!", "#N/A", "#VALUE!"];
|
|
208
|
+
const NUM = /^(-?\d+(?:\.\d+)?(?:E[-+]?\d+)?)\s*(\*+|\d\])?$/i;
|
|
209
|
+
/** [value, value_num, value_flag, footnote] of a published cell. Numbers are written with up to 15 significant
|
|
210
|
+
* digits (Excel's own precision: 549.70000000000005 is 549.7); codes go to the flag and are never a zero. */
|
|
211
|
+
export function cell(v) {
|
|
212
|
+
const s = norm(v);
|
|
213
|
+
if (!s)
|
|
214
|
+
return ["", "", "blank", ""];
|
|
215
|
+
if (PD_MISSING.includes(s))
|
|
216
|
+
return [s, "", s, ""];
|
|
217
|
+
const m = NUM.exec(s);
|
|
218
|
+
if (!m)
|
|
219
|
+
return [s, "", s, ""]; // "1]" (a footnote standing for a value, e.g. included with free)
|
|
220
|
+
const num = num15(Number(m[1]));
|
|
221
|
+
return [num, num, "", m[2] || ""];
|
|
222
|
+
}
|
|
223
|
+
const MONTH_NAMES = ["january", "february", "march", "april", "may", "june", "july", "august", "september", "october",
|
|
224
|
+
"november", "december"];
|
|
225
|
+
const DAYS = (y, m) => new Date(Date.UTC(y, m, 0)).getUTCDate();
|
|
226
|
+
function isoDate(y, m, d) {
|
|
227
|
+
return `${String(y).padStart(4, "0")}-${pad2(m)}-${pad2(d)}`;
|
|
228
|
+
}
|
|
229
|
+
/** datetime.strptime(s, "%B %d, %Y").date().isoformat(), or null where strptime would raise. */
|
|
230
|
+
function strptimeLong(s) {
|
|
231
|
+
const m = /^([A-Za-z]+)\s+(3[01]|[12]\d|0[1-9]|[1-9]|\s[1-9]),\s+(\d{4})$/.exec(s);
|
|
232
|
+
if (!m)
|
|
233
|
+
return null;
|
|
234
|
+
const mo = MONTH_NAMES.indexOf(m[1].toLowerCase()) + 1;
|
|
235
|
+
const d = parseInt(m[2].trim(), 10), y = parseInt(m[3], 10);
|
|
236
|
+
if (!mo || y < 1 || d > DAYS(y, mo))
|
|
237
|
+
return null;
|
|
238
|
+
return isoDate(y, mo, d);
|
|
239
|
+
}
|
|
240
|
+
export function asOf(rows) {
|
|
241
|
+
for (const r of rows.slice(0, 6)) {
|
|
242
|
+
const m = /Data as of\s+([A-Za-z]+\s+\d{1,2},\s*\d{4})/.exec(r.filter((x) => x).map(norm).join(" "));
|
|
243
|
+
if (m)
|
|
244
|
+
return strptimeLong(m[1].replace(/,\s*/g, ", ")) ?? m[1];
|
|
245
|
+
}
|
|
246
|
+
return "";
|
|
247
|
+
}
|
|
248
|
+
/** A note when "Data as of" precedes the table's own latest data (the month after its latest month; for fiscal
|
|
249
|
+
* years, November of the year's first month): the printed date is wrong. 27wilatest-5 to -9 print 2025 for 2026. */
|
|
250
|
+
export function asofNotes(key, rows) {
|
|
251
|
+
const asof = rows.find((x) => x.data_as_of)?.data_as_of ?? "";
|
|
252
|
+
const dm = /^(\d{4})-(\d{2})-(\d{2})$/.exec(asof);
|
|
253
|
+
if (!dm)
|
|
254
|
+
return [];
|
|
255
|
+
const d = [Number(dm[1]), Number(dm[2]), Number(dm[3])];
|
|
256
|
+
let latest = null;
|
|
257
|
+
for (const x of rows) {
|
|
258
|
+
if (!x.value_num)
|
|
259
|
+
continue;
|
|
260
|
+
const ym = x.period_type === "month" ? [parseInt(x.period.slice(0, 4), 10), parseInt(x.period.slice(5, 7), 10)]
|
|
261
|
+
: [x.fiscal_year - 1, 10];
|
|
262
|
+
if (!latest || ym[0] > latest[0] || (ym[0] === latest[0] && ym[1] > latest[1]))
|
|
263
|
+
latest = ym;
|
|
264
|
+
}
|
|
265
|
+
if (!latest)
|
|
266
|
+
return [];
|
|
267
|
+
const [y, m] = latest;
|
|
268
|
+
const need = [y + (m === 12 ? 1 : 0), (m % 12) + 1, 1];
|
|
269
|
+
const cmp = d[0] - need[0] || d[1] - need[1] || d[2] - need[2];
|
|
270
|
+
if (cmp >= 0)
|
|
271
|
+
return [];
|
|
272
|
+
const per = rows.some((x) => x.period_type === "month") ? `${y}-${pad2(m)}` : `FY ${y + 1}`;
|
|
273
|
+
return [`${key}: the workbook says "Data as of" ${asof}, before its own latest data (${per}): the printed date ` +
|
|
274
|
+
"is wrong (most likely its year). data_as_of keeps the date as printed; cite the release by its file " +
|
|
275
|
+
"name and the date you retrieved it rather than that date"];
|
|
276
|
+
}
|
|
277
|
+
export function labelOk(head, label) {
|
|
278
|
+
const h = norm(head).replace(new RegExp(`[${WS}]+`, "g"), "").toLowerCase();
|
|
279
|
+
return !!h && (label.match(/[A-Za-z0-9]+/g) || []).every((w) => h.includes(w.toLowerCase()));
|
|
280
|
+
}
|
|
281
|
+
const MONTHS = Object.fromEntries(["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"].map((m, i) => [m, i + 1]));
|
|
282
|
+
/** [year, month] of "Oct 2022", "July 2026 Initial", "June\n2026\nPreliminary"; null otherwise. */
|
|
283
|
+
export function month(s) {
|
|
284
|
+
const m = /^([A-Za-z]{3})[a-z]*\.?\s+(\d{4})\b/.exec(norm(s));
|
|
285
|
+
if (m && m[1].toLowerCase() in MONTHS)
|
|
286
|
+
return [parseInt(m[2], 10), MONTHS[m[1].toLowerCase()]];
|
|
287
|
+
return null;
|
|
288
|
+
}
|
|
289
|
+
export const fyOf = (ym) => (ym[1] >= 10 ? ym[0] + 1 : ym[0]);
|
|
290
|
+
const ymLe = (a, b) => a[0] < b[0] || (a[0] === b[0] && a[1] <= b[1]);
|
|
291
|
+
const ymEq = (a, b) => !!a && a[0] === b[0] && a[1] === b[1];
|
|
292
|
+
function excelMonth(v) {
|
|
293
|
+
const f = pyFloat(v);
|
|
294
|
+
if (f === null || !Number.isFinite(f))
|
|
295
|
+
return month(v);
|
|
296
|
+
const t = Date.UTC(1899, 11, 30) + Math.trunc(f) * 86400000;
|
|
297
|
+
const d = new Date(t);
|
|
298
|
+
if (!Number.isFinite(t) || d.getUTCFullYear() < 1 || d.getUTCFullYear() > 9999)
|
|
299
|
+
return month(v);
|
|
300
|
+
return [d.getUTCFullYear(), d.getUTCMonth() + 1];
|
|
301
|
+
}
|
|
302
|
+
function prelimYears(rows) {
|
|
303
|
+
const text = rows.flatMap((r) => r.filter((x) => typeof x === "string").map(norm)).join(" ");
|
|
304
|
+
return new Set([...text.matchAll(/FY ?(\d{4}) data are preliminary/gi)].map((m) => parseInt(m[1], 10)));
|
|
305
|
+
}
|
|
306
|
+
// ---------------------------------------------------------------------- areas
|
|
307
|
+
const EXTRA_AREAS = [["AS", "American Samoa"], ["PR", "Puerto Rico"], ["MP", "Northern Mariana Islands"]];
|
|
308
|
+
const AREA_ALIASES = { "commonwealth of northern mariana islands": "MP", "northern marianas": "MP",
|
|
309
|
+
"u.s. virgin islands": "VI", "us virgin islands": "VI" };
|
|
310
|
+
export const REGIONS = { NERO: "Northeast", MARO: "Mid-Atlantic", SERO: "Southeast", MWRO: "Midwest",
|
|
311
|
+
SWRO: "Southwest", MPRO: "Mountain Plains", WRO: "Western" };
|
|
312
|
+
/** The states, DC and territories under the names the workbooks use (USPS codes; AS, PR, MP added). */
|
|
313
|
+
export class Areas {
|
|
314
|
+
name;
|
|
315
|
+
abbr;
|
|
316
|
+
constructor(agencies) {
|
|
317
|
+
this.name = new Map([...agencies, ...EXTRA_AREAS]);
|
|
318
|
+
this.abbr = new Map([...this.name].map(([a, n]) => [n.toLowerCase(), a]));
|
|
319
|
+
for (const [k, v] of Object.entries(AREA_ALIASES))
|
|
320
|
+
this.abbr.set(k, v);
|
|
321
|
+
}
|
|
322
|
+
code(x) {
|
|
323
|
+
const k = pyStrip(norm(x).toLowerCase().replace(/\*+$/, ""));
|
|
324
|
+
if (this.name.has(k.toUpperCase()))
|
|
325
|
+
return k.toUpperCase();
|
|
326
|
+
return this.abbr.get(k) ?? null;
|
|
327
|
+
}
|
|
328
|
+
codes(v) {
|
|
329
|
+
const vals = Array.isArray(v) ? v : String(v).split(",").filter((x) => pyStrip(x));
|
|
330
|
+
const out = [];
|
|
331
|
+
for (const x of vals) {
|
|
332
|
+
const c = this.code(x);
|
|
333
|
+
if (!c) {
|
|
334
|
+
throw new SpecError(`unknown state or territory ${pyRepr(x)}: give a USPS code (TX) or a name (Texas); ` +
|
|
335
|
+
"territories: AS, GU, MP, PR, VI");
|
|
336
|
+
}
|
|
337
|
+
if (!out.includes(c))
|
|
338
|
+
out.push(c);
|
|
339
|
+
}
|
|
340
|
+
return out;
|
|
341
|
+
}
|
|
342
|
+
/** [row_type, state, state_abbr] of a state-table row name. */
|
|
343
|
+
classify(raw) {
|
|
344
|
+
const n = norm(raw);
|
|
345
|
+
if (n.toUpperCase() === "TOTAL")
|
|
346
|
+
return ["total", "Total", ""];
|
|
347
|
+
if (n.startsWith("DOD") || n.startsWith("Dept. of Defense"))
|
|
348
|
+
return ["dod", n, ""];
|
|
349
|
+
const c = this.code(n);
|
|
350
|
+
if (c)
|
|
351
|
+
return ["state", n, c];
|
|
352
|
+
return ["other", n, ""];
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
/** Python's ValueError from a spec helper (check_spec reports it as a problem). */
|
|
356
|
+
export class SpecError extends Error {
|
|
357
|
+
}
|
|
358
|
+
// ---------------------------------------------------------------------- table rows
|
|
359
|
+
function mkRow(t, title, asof, state, abbr, agency, rtype, period, ptype, fy, meas, unit, raw, status) {
|
|
360
|
+
const [v, num, flag, foot] = cell(raw);
|
|
361
|
+
return { table: t, table_title: title, data_as_of: asof, state, state_abbr: abbr, agency, row_type: rtype, period,
|
|
362
|
+
period_type: ptype, fiscal_year: fy, measure: meas, unit, value: v, value_num: num, value_flag: flag, status,
|
|
363
|
+
footnote: foot };
|
|
364
|
+
}
|
|
365
|
+
const at = (r, j) => (j < r.length ? r[j] : null);
|
|
366
|
+
export function titleOf(rows) {
|
|
367
|
+
for (const r of rows)
|
|
368
|
+
if (r.length && r[0] && norm(r[0]))
|
|
369
|
+
return norm(r[0]);
|
|
370
|
+
return "";
|
|
371
|
+
}
|
|
372
|
+
function layoutOf(rows) {
|
|
373
|
+
const firsts = rows.slice(0, 12).filter((r) => r.length && r[0]).map((r) => norm(r[0]));
|
|
374
|
+
if (firsts.map((f) => f.toUpperCase()).includes("ANNUAL SUMMARY"))
|
|
375
|
+
return "monthly";
|
|
376
|
+
for (const r of rows.slice(0, 8)) {
|
|
377
|
+
if (r.length && r[0] && norm(r[0]).toLowerCase().startsWith("state")) {
|
|
378
|
+
const c = r.slice(1).filter((x) => x).map(norm);
|
|
379
|
+
if (c.length && /^FY \d{4}/.test(c[0]))
|
|
380
|
+
return "state_fy";
|
|
381
|
+
if (c.length && month(c[0]))
|
|
382
|
+
return "state_latest";
|
|
383
|
+
}
|
|
384
|
+
if (r.length && r[0] && norm(r[0]).toLowerCase().startsWith("fiscal year"))
|
|
385
|
+
return "annual";
|
|
386
|
+
}
|
|
387
|
+
return "";
|
|
388
|
+
}
|
|
389
|
+
/** The [name, rows, hidden] sheet holding table d: its title matches and its layout is d's. */
|
|
390
|
+
export function findSheet(book, d) {
|
|
391
|
+
for (const s of book)
|
|
392
|
+
if (new RegExp(d.title, "i").test(titleOf(s[1])) && layoutOf(s[1]) === d.layout)
|
|
393
|
+
return s;
|
|
394
|
+
return null;
|
|
395
|
+
}
|
|
396
|
+
function header(rows, first) {
|
|
397
|
+
for (let i = 0; i < Math.min(10, rows.length); i++) {
|
|
398
|
+
const r = rows[i];
|
|
399
|
+
if (r.length && r[0] && norm(r[0]).toLowerCase().startsWith(first))
|
|
400
|
+
return i;
|
|
401
|
+
}
|
|
402
|
+
throw new LayoutError(`no header row starting with ${pyRepr(first)}`);
|
|
403
|
+
}
|
|
404
|
+
/** A value under a visible column the table does not use means the columns moved: fail. */
|
|
405
|
+
function checkColumns(rows, start, hidden, used, what) {
|
|
406
|
+
for (const r of rows.slice(start)) {
|
|
407
|
+
for (let j = 1; j < r.length; j++) {
|
|
408
|
+
const x = r[j];
|
|
409
|
+
if (!used.has(j) && !hidden.has(j) && norm(x) && cell(x)[1]) {
|
|
410
|
+
throw new LayoutError(`${what}: a value in unlabeled column ${j + 1} (${pyRepr(norm(r[0]))} row)`);
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
/** [[column index, [measure, label, unit]]]: the visible header cells in order, each matching its label. */
|
|
416
|
+
function labeled(head, cols, hidden, what) {
|
|
417
|
+
const vis = [];
|
|
418
|
+
for (let j = 1; j < head.length; j++)
|
|
419
|
+
if (!hidden.has(j) && norm(head[j]))
|
|
420
|
+
vis.push(j);
|
|
421
|
+
if (vis.length !== cols.length)
|
|
422
|
+
throw new LayoutError(`${what}: ${vis.length} labeled columns, endpoints.json lists ${cols.length}`);
|
|
423
|
+
vis.forEach((j, i) => {
|
|
424
|
+
if (!labelOk(head[j], cols[i][1])) {
|
|
425
|
+
throw new LayoutError(`${what}: column ${j + 1} is ${pyRepr(norm(head[j]))}, expected ${pyRepr(cols[i][1])}`);
|
|
426
|
+
}
|
|
427
|
+
});
|
|
428
|
+
return vis.map((j, i) => [j, cols[i]]);
|
|
429
|
+
}
|
|
430
|
+
function parseAnnual(key, d, rows, hidden) {
|
|
431
|
+
const title = titleOf(rows), asof = asOf(rows), pre = prelimYears(rows);
|
|
432
|
+
const h = header(rows, "fiscal year");
|
|
433
|
+
const cols = labeled(rows[h], d.columns, hidden, key);
|
|
434
|
+
checkColumns(rows, h + 1, hidden, new Set(cols.map(([j]) => j)), key);
|
|
435
|
+
const out = [];
|
|
436
|
+
let last = 0;
|
|
437
|
+
for (const r of rows.slice(h + 1)) {
|
|
438
|
+
const m = /^(\d{4})(?:\.0)?\s*(\d\])?$/.exec(r.length ? norm(r[0]) : "");
|
|
439
|
+
if (!m)
|
|
440
|
+
continue;
|
|
441
|
+
const fy = parseInt(m[1], 10);
|
|
442
|
+
if (fy <= last)
|
|
443
|
+
throw new LayoutError(`${key}: fiscal year ${fy} after ${last}`);
|
|
444
|
+
last = fy;
|
|
445
|
+
for (const [j, [meas, , unit]] of cols) {
|
|
446
|
+
const row = mkRow(key, title, asof, "U.S.", "", "", "national", `FY ${fy}`, "fiscal_year", fy, meas, unit, at(r, j), pre.has(fy) ? "preliminary" : "");
|
|
447
|
+
row.footnote = [m[2] || "", row.footnote].filter((x) => x).join(" ");
|
|
448
|
+
out.push(row);
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
if (!out.length)
|
|
452
|
+
throw new LayoutError(`${key}: no fiscal-year rows`);
|
|
453
|
+
return out;
|
|
454
|
+
}
|
|
455
|
+
function parseMonthly(key, d, rows, hidden) {
|
|
456
|
+
const title = titleOf(rows), asof = asOf(rows), pre = prelimYears(rows);
|
|
457
|
+
const h = header(rows, "fiscal year");
|
|
458
|
+
const cols = labeled(rows[h], d.columns, hidden, key);
|
|
459
|
+
checkColumns(rows, h + 1, hidden, new Set(cols.map(([j]) => j)), key);
|
|
460
|
+
const out = [];
|
|
461
|
+
let section = "", cur = null, prev = null;
|
|
462
|
+
for (const r of rows.slice(h + 1)) {
|
|
463
|
+
const f = r.length ? norm(r[0]) : "";
|
|
464
|
+
if (f.toUpperCase() === "ANNUAL SUMMARY" || f.toUpperCase() === "MONTHLY DATA") {
|
|
465
|
+
section = f.toUpperCase();
|
|
466
|
+
continue;
|
|
467
|
+
}
|
|
468
|
+
const has = r.slice(1).some((x) => norm(x));
|
|
469
|
+
const mfy = /^FY ?(\d{4})$/.exec(f);
|
|
470
|
+
const ym = month(f);
|
|
471
|
+
let fy, period, ptype;
|
|
472
|
+
if (mfy && section === "MONTHLY DATA" && !has) {
|
|
473
|
+
cur = parseInt(mfy[1], 10);
|
|
474
|
+
prev = null;
|
|
475
|
+
continue;
|
|
476
|
+
}
|
|
477
|
+
if (mfy && section === "ANNUAL SUMMARY") {
|
|
478
|
+
fy = parseInt(mfy[1], 10);
|
|
479
|
+
[period, ptype] = [`FY ${fy}`, "fiscal_year"];
|
|
480
|
+
}
|
|
481
|
+
else if (ym && section === "MONTHLY DATA") {
|
|
482
|
+
if (cur === null || fyOf(ym) !== cur || (prev && ymLe(ym, prev))) {
|
|
483
|
+
throw new LayoutError(`${key}: month ${pyRepr(f)} is out of place (under FY ${cur === null ? "None" : cur})`);
|
|
484
|
+
}
|
|
485
|
+
prev = ym;
|
|
486
|
+
fy = cur;
|
|
487
|
+
[period, ptype] = [`${ym[0]}-${pad2(ym[1])}`, "month"];
|
|
488
|
+
}
|
|
489
|
+
else
|
|
490
|
+
continue;
|
|
491
|
+
for (const [j, [meas, , unit]] of cols) {
|
|
492
|
+
out.push(mkRow(key, title, asof, "U.S.", "", "", "national", period, ptype, fy, meas, unit, at(r, j), pre.has(fy) ? "preliminary" : ""));
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
if (!out.some((x) => x.period_type === "month"))
|
|
496
|
+
throw new LayoutError(`${key}: no monthly rows`);
|
|
497
|
+
return out;
|
|
498
|
+
}
|
|
499
|
+
/** [[row_type, state, abbr, agency, row]] down to the TOTAL row; indented names are parts of the state above. */
|
|
500
|
+
function stateRows(rows, start, areas) {
|
|
501
|
+
const out = [];
|
|
502
|
+
let parent = null;
|
|
503
|
+
for (const r of rows.slice(start)) {
|
|
504
|
+
if (!r.length || !r[0] || !norm(r[0]))
|
|
505
|
+
continue;
|
|
506
|
+
const raw = String(r[0]);
|
|
507
|
+
const [rtype, state, abbr] = areas.classify(raw);
|
|
508
|
+
if (rtype !== "total" && raw.slice(0, 1) === " " && parent) {
|
|
509
|
+
out.push(["sub_agency", parent[0], parent[1], norm(raw), r]);
|
|
510
|
+
continue;
|
|
511
|
+
}
|
|
512
|
+
out.push([rtype, state, abbr, "", r]);
|
|
513
|
+
if (rtype === "total")
|
|
514
|
+
return out;
|
|
515
|
+
parent = [state, abbr];
|
|
516
|
+
}
|
|
517
|
+
throw new LayoutError("no TOTAL row");
|
|
518
|
+
}
|
|
519
|
+
/** Agency sums vs the printed totals (and WIC tribal parts vs their state), per period and measure. */
|
|
520
|
+
function sumNotes(key, rows, tolRel = 0.0005) {
|
|
521
|
+
const notes = [];
|
|
522
|
+
const by = new Map();
|
|
523
|
+
for (const x of rows) {
|
|
524
|
+
const k = JSON.stringify([x.measure, x.period]);
|
|
525
|
+
if (!by.has(k))
|
|
526
|
+
by.set(k, []);
|
|
527
|
+
by.get(k).push(x);
|
|
528
|
+
}
|
|
529
|
+
for (const [k, xs] of by) {
|
|
530
|
+
const [meas, period] = JSON.parse(k);
|
|
531
|
+
const tot = xs.filter((x) => x.row_type === "total" && x.value_num);
|
|
532
|
+
const parts = xs.filter((x) => ["state", "dod", "other"].includes(x.row_type) && x.value_num).map((x) => Number(x.value_num));
|
|
533
|
+
if (tot.length && parts.length && !meas.includes("avg") && !meas.includes("share")) {
|
|
534
|
+
const t = Number(tot[0].value_num);
|
|
535
|
+
const s = pySum(parts);
|
|
536
|
+
if (Math.abs(s - t) > Math.max(2.0, tolRel * Math.abs(t))) {
|
|
537
|
+
notes.push(`${key} ${meas} ${period}: agency sum ${fmt0(s)} vs printed total ${fmt0(t)}`);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
return notes;
|
|
542
|
+
}
|
|
543
|
+
function subNotes(key, rows) {
|
|
544
|
+
const notes = [];
|
|
545
|
+
const by = new Map();
|
|
546
|
+
for (const x of rows) {
|
|
547
|
+
if ((x.row_type === "state" || x.row_type === "sub_agency") && x.value_num) {
|
|
548
|
+
const k = JSON.stringify([x.state, x.period]);
|
|
549
|
+
if (!by.has(k))
|
|
550
|
+
by.set(k, new Map());
|
|
551
|
+
const g = by.get(k);
|
|
552
|
+
if (!g.has(x.row_type))
|
|
553
|
+
g.set(x.row_type, []);
|
|
554
|
+
g.get(x.row_type).push(Number(x.value_num));
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
for (const [k, g] of by) {
|
|
558
|
+
if (rows.slice(0, 1).some((x) => x.measure.includes("avg")))
|
|
559
|
+
break;
|
|
560
|
+
const [st, period] = JSON.parse(k);
|
|
561
|
+
if (g.has("sub_agency") && g.has("state")) {
|
|
562
|
+
const s = pySum(g.get("sub_agency")), t = g.get("state")[0];
|
|
563
|
+
if (Math.abs(s - t) > Math.max(2.0, 0.0005 * Math.abs(t))) {
|
|
564
|
+
notes.push(`${key} ${st} ${period}: its agencies sum to ${fmt0(s)}, the state row is ${fmt0(t)}`);
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
return notes;
|
|
569
|
+
}
|
|
570
|
+
function parseStateFy(key, d, rows, hidden, areas) {
|
|
571
|
+
const title = titleOf(rows), asof = asOf(rows);
|
|
572
|
+
const h = header(rows, "state");
|
|
573
|
+
const cols = [];
|
|
574
|
+
rows[h].forEach((x, j) => {
|
|
575
|
+
const m = /^FY (\d{4})/.exec(norm(x));
|
|
576
|
+
if (j && !hidden.has(j) && m)
|
|
577
|
+
cols.push([j, parseInt(m[1], 10)]);
|
|
578
|
+
});
|
|
579
|
+
if (!cols.length || cols.some(([, y], i) => y !== cols[0][1] + i)) {
|
|
580
|
+
throw new LayoutError(`${key}: fiscal-year columns ${pyRepr(rows[h].slice(1).map(norm))}`);
|
|
581
|
+
}
|
|
582
|
+
checkColumns(rows, h + 1, hidden, new Set(cols.map(([j]) => j)), key);
|
|
583
|
+
const status = new Map();
|
|
584
|
+
for (const r of rows.slice(h + 1, h + 3)) {
|
|
585
|
+
if (r.length && !norm(r.length ? r[0] : "")) {
|
|
586
|
+
for (const [j] of cols)
|
|
587
|
+
if (j < r.length && /prelim/i.test(norm(r[j])))
|
|
588
|
+
status.set(j, "preliminary");
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
const pre = prelimYears(rows);
|
|
592
|
+
const out = [];
|
|
593
|
+
for (const [rtype, state, abbr, agency, r] of stateRows(rows, h + 1, areas)) {
|
|
594
|
+
for (const [j, fy] of cols) {
|
|
595
|
+
out.push(mkRow(key, title, asof, state, abbr, agency, rtype, `FY ${fy}`, "fiscal_year", fy, d.measure, d.unit, at(r, j), status.get(j) || (pre.has(fy) ? "preliminary" : "")));
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
return [out, [...sumNotes(key, out), ...subNotes(key, out)]];
|
|
599
|
+
}
|
|
600
|
+
function parseStateLatest(key, d, rows, hidden, areas) {
|
|
601
|
+
const title = titleOf(rows), asof = asOf(rows);
|
|
602
|
+
const h = header(rows, "state");
|
|
603
|
+
const head = rows[h];
|
|
604
|
+
const cols = [];
|
|
605
|
+
head.forEach((x, j) => {
|
|
606
|
+
const ym = month(x);
|
|
607
|
+
if (j && !hidden.has(j) && ym)
|
|
608
|
+
cols.push([j, ym, norm(x)]);
|
|
609
|
+
});
|
|
610
|
+
const used = new Set(cols.map(([j]) => j));
|
|
611
|
+
head.forEach((x, j) => {
|
|
612
|
+
if (/^percent change/i.test(norm(x)))
|
|
613
|
+
used.add(j);
|
|
614
|
+
});
|
|
615
|
+
checkColumns(rows, h + 1, hidden, used, key);
|
|
616
|
+
if (cols.length !== 3)
|
|
617
|
+
throw new LayoutError(`${key}: ${cols.length} month columns, expected 3`);
|
|
618
|
+
const notes = [], keep = [];
|
|
619
|
+
const [y, m] = cols[2][1];
|
|
620
|
+
const want = [[y - 1, m], [m > 1 ? y : y - 1, m > 1 ? m - 1 : 12], [y, m]];
|
|
621
|
+
cols.forEach(([j, ym, lab], i) => {
|
|
622
|
+
const w = want[i];
|
|
623
|
+
if (!ymEq(ym, w)) {
|
|
624
|
+
notes.push(`${key}${INCOMPLETE} column ${pyRepr(lab)} dropped: its month is not ${w[0]}-${pad2(w[1])} (a year ` +
|
|
625
|
+
`before / the month before ${y}-${pad2(m)}), so the header cannot be trusted`);
|
|
626
|
+
}
|
|
627
|
+
else {
|
|
628
|
+
const st = /initial/i.test(lab) ? "initial" : /prelim/i.test(lab) ? "preliminary" : "";
|
|
629
|
+
keep.push([j, ym, st]);
|
|
630
|
+
}
|
|
631
|
+
});
|
|
632
|
+
const out = [];
|
|
633
|
+
for (const [rtype, state, abbr, agency, r] of stateRows(rows, h + 1, areas)) {
|
|
634
|
+
for (const [j, ym, st] of keep) {
|
|
635
|
+
out.push(mkRow(key, title, asof, state, abbr, agency, rtype, `${ym[0]}-${pad2(ym[1])}`, "month", fyOf(ym), d.measure, d.unit, at(r, j), st));
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
return [out, [...notes, ...sumNotes(key, out), ...subNotes(key, out)]];
|
|
639
|
+
}
|
|
640
|
+
/** Rows and notes of catalog table key from its workbook; LayoutError when it does not match. */
|
|
641
|
+
export function parseTable(key, d, body, areas) {
|
|
642
|
+
const found = findSheet(readBook(body), d);
|
|
643
|
+
if (!found)
|
|
644
|
+
throw new LayoutError(`${key}: no sheet titled like ${pyRepr(d.title)} with the ${d.layout} layout`);
|
|
645
|
+
const [, rows, hidden] = found;
|
|
646
|
+
let out, notes;
|
|
647
|
+
if (d.layout === "annual")
|
|
648
|
+
[out, notes] = [parseAnnual(key, d, rows, hidden), []];
|
|
649
|
+
else if (d.layout === "monthly")
|
|
650
|
+
[out, notes] = [parseMonthly(key, d, rows, hidden), []];
|
|
651
|
+
else if (d.layout === "state_fy")
|
|
652
|
+
[out, notes] = parseStateFy(key, d, rows, hidden, areas);
|
|
653
|
+
else
|
|
654
|
+
[out, notes] = parseStateLatest(key, d, rows, hidden, areas);
|
|
655
|
+
return [out, [...asofNotes(key, out), ...notes]];
|
|
656
|
+
}
|
|
657
|
+
// ---------------------------------------------------------------------- SNAP state monthly (zip of FYyy.xlsx)
|
|
658
|
+
const SNAP_COLS = [["households", "households (monthly)"], ["persons", "persons (monthly)"],
|
|
659
|
+
["benefits", "dollars"], ["benefit_per_household", "dollars per household per month"],
|
|
660
|
+
["benefit_per_person", "dollars per person per month"]];
|
|
661
|
+
const base = (n) => n.slice(n.lastIndexOf("/") + 1);
|
|
662
|
+
/** The FYyy.xlsx member of a zip (given its member names), or null. */
|
|
663
|
+
export function snapZipMember(body, names, fy) {
|
|
664
|
+
const rx = new RegExp(`^FY${pad2(fy % 100)}\\.xlsx$`, "i");
|
|
665
|
+
for (const n of names)
|
|
666
|
+
if (rx.test(base(n)))
|
|
667
|
+
return zipRead(body, n);
|
|
668
|
+
return null;
|
|
669
|
+
}
|
|
670
|
+
export function zipXlsxYears(names) {
|
|
671
|
+
const out = [];
|
|
672
|
+
for (const n of names) {
|
|
673
|
+
const m = /^FY(\d{2})\.(xlsx?)$/i.exec(base(n));
|
|
674
|
+
if (m)
|
|
675
|
+
out.push((parseInt(m[1], 10) < 69 ? 2000 : 1900) + parseInt(m[1], 10));
|
|
676
|
+
}
|
|
677
|
+
return out.sort((a, b) => a - b);
|
|
678
|
+
}
|
|
679
|
+
export function parseSnapYear(body, fy, areas) {
|
|
680
|
+
const out = [], notes = [];
|
|
681
|
+
const t = "snap_state_monthly";
|
|
682
|
+
for (const [sheet, rows] of readBook(body)) {
|
|
683
|
+
const head = rows.slice(0, 3).flatMap((r) => r.filter((x) => x).map(norm)).join(" ");
|
|
684
|
+
if (!new RegExp(`Fiscal Year ${fy}\\b`).test(head)) {
|
|
685
|
+
throw new LayoutError(`FY${pad2(fy % 100)}.xlsx sheet ${pyRepr(sheet)} is not fiscal year ${fy}: ${pyRepr(head.slice(0, 120))}`);
|
|
686
|
+
}
|
|
687
|
+
const title = rows.length > 1 && rows[1].length ? norm(rows[1][0]) : head;
|
|
688
|
+
let h = null;
|
|
689
|
+
for (let i = 0; i < Math.min(12, rows.length); i++) {
|
|
690
|
+
const r = rows[i];
|
|
691
|
+
if (r.length && norm(r[0]).toLowerCase().startsWith("fiscal year and month")) {
|
|
692
|
+
h = i;
|
|
693
|
+
break;
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
const sub = h !== null && h + 1 < rows.length ? rows[h + 1] : [];
|
|
697
|
+
const labs = sub.length ? sub.slice(1, 6).map((x) => norm(x).toLowerCase()) : [];
|
|
698
|
+
const wantLabs = ["household", "persons", "", "household", "persons"];
|
|
699
|
+
if (h === null || labs.length !== wantLabs.length || labs.some((x, i) => x !== wantLabs[i]) ||
|
|
700
|
+
!labelOk(rows[h].length > 3 ? rows[h][3] : "", "Cost")) {
|
|
701
|
+
throw new LayoutError(`FY ${fy} sheet ${pyRepr(sheet)}: the header is not Participation (Household, Persons), ` +
|
|
702
|
+
`Cost, Cost Per (Household, Persons): ${pyRepr(labs)}`);
|
|
703
|
+
}
|
|
704
|
+
let block = null;
|
|
705
|
+
let prev = null;
|
|
706
|
+
for (const r of rows.slice(h + 2)) {
|
|
707
|
+
const f = r.length ? norm(r[0]) : "";
|
|
708
|
+
if (!f)
|
|
709
|
+
continue;
|
|
710
|
+
if (f.toLowerCase().startsWith("footnotes"))
|
|
711
|
+
break;
|
|
712
|
+
const ym = month(f);
|
|
713
|
+
if (!ym && f !== "Total") {
|
|
714
|
+
if (Object.prototype.hasOwnProperty.call(REGIONS, f))
|
|
715
|
+
block = ["region", REGIONS[f] + " Region", "", f];
|
|
716
|
+
else if (f.toLowerCase() === "us summary")
|
|
717
|
+
block = ["national", "U.S.", "", ""];
|
|
718
|
+
else {
|
|
719
|
+
const [rt, st, ab] = areas.classify(f);
|
|
720
|
+
block = [rt, st, ab, ""];
|
|
721
|
+
}
|
|
722
|
+
prev = null;
|
|
723
|
+
continue;
|
|
724
|
+
}
|
|
725
|
+
if (block === null)
|
|
726
|
+
throw new LayoutError(`FY ${fy} sheet ${pyRepr(sheet)}: ${pyRepr(f)} before any state name`);
|
|
727
|
+
let period, ptype;
|
|
728
|
+
if (ym) {
|
|
729
|
+
if (fyOf(ym) !== fy || (prev && ymLe(ym, prev))) {
|
|
730
|
+
throw new LayoutError(`FY ${fy} sheet ${pyRepr(sheet)} ${block[1]}: month ${pyRepr(f)} is out of place`);
|
|
731
|
+
}
|
|
732
|
+
prev = ym;
|
|
733
|
+
[period, ptype] = [`${ym[0]}-${pad2(ym[1])}`, "month"];
|
|
734
|
+
}
|
|
735
|
+
else
|
|
736
|
+
[period, ptype] = [`FY ${fy}`, "fiscal_year"];
|
|
737
|
+
SNAP_COLS.forEach(([meas, unit], i) => {
|
|
738
|
+
const j = i + 1;
|
|
739
|
+
out.push(mkRow(t, title, "", block[1], block[2], block[3], block[0], period, ptype, fy, meas, ptype === "month" || meas !== "benefits" ? unit : "dollars (fiscal year)", at(r, j), ""));
|
|
740
|
+
});
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
// sums: states of a region vs the region, regions vs the U.S., per month
|
|
744
|
+
const by = new Map();
|
|
745
|
+
for (const x of out) {
|
|
746
|
+
if (x.period_type === "month" && ["persons", "households", "benefits"].includes(x.measure) && x.value_num) {
|
|
747
|
+
const k = JSON.stringify([x.measure, x.period]);
|
|
748
|
+
if (!by.has(k))
|
|
749
|
+
by.set(k, new Map());
|
|
750
|
+
const g = by.get(k);
|
|
751
|
+
if (!g.has(x.row_type))
|
|
752
|
+
g.set(x.row_type, []);
|
|
753
|
+
g.get(x.row_type).push(Number(x.value_num));
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
const keys = [...by.keys()].sort((a, b) => {
|
|
757
|
+
const [x, y] = [JSON.parse(a), JSON.parse(b)];
|
|
758
|
+
return x[0] !== y[0] ? (x[0] < y[0] ? -1 : 1) : x[1] < y[1] ? -1 : x[1] > y[1] ? 1 : 0;
|
|
759
|
+
});
|
|
760
|
+
for (const k of keys) {
|
|
761
|
+
const [meas, period] = JSON.parse(k);
|
|
762
|
+
const g = by.get(k);
|
|
763
|
+
const nat = g.get("national"), reg = g.get("region"), sta = g.get("state");
|
|
764
|
+
if (nat && reg && Math.abs(pySum(reg) - nat[0]) > 2) {
|
|
765
|
+
notes.push(`snap_state_monthly FY ${fy} ${meas} ${period}: regions sum to ${fmt0(pySum(reg))}, U.S. ${fmt0(nat[0])}`);
|
|
766
|
+
}
|
|
767
|
+
if (reg && sta && Math.abs(pySum(sta) - pySum(reg)) > 2) {
|
|
768
|
+
notes.push(`snap_state_monthly FY ${fy} ${meas} ${period}: states sum to ${fmt0(pySum(sta))}, regions ${fmt0(pySum(reg))}`);
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
if (!out.some((x) => x.row_type === "state"))
|
|
772
|
+
throw new LayoutError(`FY ${fy}: no state rows`);
|
|
773
|
+
return [out, notes];
|
|
774
|
+
}
|
|
775
|
+
// ---------------------------------------------------------------------- WIC agency monthly (wicagencies<FY>ytd.xlsx)
|
|
776
|
+
export function wicSheetKey(name, sheets) {
|
|
777
|
+
for (const [k, d] of Object.entries(sheets))
|
|
778
|
+
if (new RegExp(d.sheet, "i").test(name))
|
|
779
|
+
return k;
|
|
780
|
+
return null;
|
|
781
|
+
}
|
|
782
|
+
export function parseWicSheet(key, d, rows, hidden, fy, areas) {
|
|
783
|
+
const title = titleOf(rows);
|
|
784
|
+
const top = rows.slice(0, 4).flatMap((r) => r.filter((x) => x).map(norm)).join(" ");
|
|
785
|
+
if (!new RegExp(`FISCAL YEAR ${fy}\\b`, "i").test(top)) {
|
|
786
|
+
throw new LayoutError(`${key}: the sheet is not fiscal year ${fy} (${pyRepr(top.slice(0, 80))})`);
|
|
787
|
+
}
|
|
788
|
+
const asof = asOf(rows);
|
|
789
|
+
const pre = /all data are preliminary/i.test(rows.slice(-8).flatMap((r) => r.filter((x) => x).map(norm)).join(" "))
|
|
790
|
+
? "preliminary" : "";
|
|
791
|
+
const h = header(rows, "state agency");
|
|
792
|
+
const head = rows[h];
|
|
793
|
+
const months = [];
|
|
794
|
+
let total = null;
|
|
795
|
+
for (let j = 1; j < head.length; j++) {
|
|
796
|
+
const x = head[j];
|
|
797
|
+
if (hidden.has(j) || !norm(x))
|
|
798
|
+
continue;
|
|
799
|
+
if (/^(average|cumulative)/i.test(norm(x))) {
|
|
800
|
+
total = [j, norm(x)];
|
|
801
|
+
continue;
|
|
802
|
+
}
|
|
803
|
+
const ym = excelMonth(x);
|
|
804
|
+
if (!ym)
|
|
805
|
+
throw new LayoutError(`${key}: header cell ${pyRepr(norm(x))} is neither a month nor the year column`);
|
|
806
|
+
months.push([j, ym]);
|
|
807
|
+
}
|
|
808
|
+
const want = [[fy - 1, 10], [fy - 1, 11], [fy - 1, 12]];
|
|
809
|
+
for (let m = 1; m < 10; m++)
|
|
810
|
+
want.push([fy, m]);
|
|
811
|
+
if (months.length > want.length || months.some(([, ym], i) => !ymEq(ym, want[i]))) {
|
|
812
|
+
throw new LayoutError(`${key}: month columns ${"[" + months.map(([, ym]) => `(${ym[0]}, ${ym[1]})`).join(", ") + "]"} ` +
|
|
813
|
+
`are not October ${fy - 1} onward`);
|
|
814
|
+
}
|
|
815
|
+
if (total === null)
|
|
816
|
+
throw new LayoutError(`${key}: no average or cumulative column`);
|
|
817
|
+
checkColumns(rows, h + 1, hidden, new Set([...months.map(([j]) => j), total[0]]), key);
|
|
818
|
+
const out = [], notes = [], agencies = [];
|
|
819
|
+
const regionNames = new Set(Object.values(REGIONS));
|
|
820
|
+
for (const r of rows.slice(h + 1)) {
|
|
821
|
+
const f = r.length && r[0] ? norm(r[0]) : "";
|
|
822
|
+
if (!f || !r.slice(1).some((x) => norm(x)))
|
|
823
|
+
continue;
|
|
824
|
+
const m = /^(.+), ([A-Z]{2})$/.exec(f);
|
|
825
|
+
let rt, st, ab, ag;
|
|
826
|
+
if (f.toUpperCase() === "TOTAL")
|
|
827
|
+
[rt, st, ab, ag] = ["total", "Total", "", ""];
|
|
828
|
+
else if (regionNames.has(f.endsWith(" Region") ? f.slice(0, -" Region".length) : f))
|
|
829
|
+
[rt, st, ab, ag] = ["region", f, "", ""];
|
|
830
|
+
else if (m && areas.code(m[2])) {
|
|
831
|
+
const c = areas.code(m[2]);
|
|
832
|
+
[rt, st, ab, ag] = ["tribal_org", areas.name.get(c), c, m[1]];
|
|
833
|
+
}
|
|
834
|
+
else {
|
|
835
|
+
[rt, st, ab] = areas.classify(f);
|
|
836
|
+
[rt, ag] = rt === "state" ? ["state_agency", ""] : [rt, ""];
|
|
837
|
+
}
|
|
838
|
+
for (const [j, ym] of months) {
|
|
839
|
+
out.push(mkRow("wic_agency_monthly", title, asof, st, ab, ag, rt, `${ym[0]}-${pad2(ym[1])}`, "month", fy, key, d.unit, at(r, j), pre));
|
|
840
|
+
}
|
|
841
|
+
out.push(mkRow("wic_agency_monthly", title, asof, st, ab, ag, rt, `FY ${fy}`, "fiscal_year", fy, key, d.unit + (" (" + total[1].split(":")[0].toLowerCase() + ")"), at(r, total[0]), pre));
|
|
842
|
+
agencies.push([rt, r]);
|
|
843
|
+
}
|
|
844
|
+
// agencies since the last region add up to that region; regions add up to TOTAL
|
|
845
|
+
if (!key.includes("avg")) {
|
|
846
|
+
for (const [j] of [...months, [total[0], null]]) {
|
|
847
|
+
let run = 0.0, regions = 0.0;
|
|
848
|
+
for (const [rt, r] of agencies) {
|
|
849
|
+
const s = cell(at(r, j))[1];
|
|
850
|
+
const v = s ? Number(s) : 0.0;
|
|
851
|
+
if (rt === "region") {
|
|
852
|
+
if (Math.abs(run - v) > Math.max(2.0, 0.0005 * Math.abs(v))) {
|
|
853
|
+
notes.push(`wic_agency_monthly FY ${fy} ${key} ${norm(r[0])} column ${j + 1}: agencies sum to ${fmt0(run)}, region ${fmt0(v)}`);
|
|
854
|
+
}
|
|
855
|
+
[regions, run] = [regions + v, 0.0];
|
|
856
|
+
}
|
|
857
|
+
else if (rt === "total") {
|
|
858
|
+
if (Math.abs(regions - v) > Math.max(2.0, 0.0005 * Math.abs(v))) {
|
|
859
|
+
notes.push(`wic_agency_monthly FY ${fy} ${key} column ${j + 1}: regions sum to ${fmt0(regions)}, TOTAL ${fmt0(v)}`);
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
else
|
|
863
|
+
run += v;
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
return [out, [...asofNotes(`wic_agency_monthly FY ${fy} ${key}`, out), ...notes]];
|
|
868
|
+
}
|
|
869
|
+
//# sourceMappingURL=_usda-fna_book.js.map
|