data-analyst-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -2
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +716 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +24 -0
- package/dist/report/scan_secrets.js +265 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +247 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +54 -4
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +170 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,983 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Census Data API adapter (standard pack, own fetch because dataset paths contain slashes).
|
|
3
|
+
*
|
|
4
|
+
* What it adds to the generic REST flow:
|
|
5
|
+
* - datasets: "latest/acs/acs5", "latest/acs/acs1/subject", "latest/pep/charv" (or the catalog aliases acs5,
|
|
6
|
+
* acs1, ...) resolve to the newest vintage the API actually serves, by asking the dataset discovery document
|
|
7
|
+
* /data/<year>/<path>.json from last year downward (no hard-coded year).
|
|
8
|
+
* - geography: state (name, USPS or FIPS), county and place names resolve to codes with "Did you mean"
|
|
9
|
+
* suggestions. Names come from the same dataset's NAME list when a key is set (right vintage); otherwise, or when
|
|
10
|
+
* that call fails, from the Census 2020 reference files on www2.census.gov (noted in the output; Connecticut's
|
|
11
|
+
* planning regions used from ACS 2022 on are not in them).
|
|
12
|
+
* - margins of error: for ACS every ...E estimate gets its ...M margin of error (...PE gets ...PM) as its own
|
|
13
|
+
* column. They are never dropped (moe: false is refused for ACS).
|
|
14
|
+
* - normalize: Census special values (-666666666, -999999999, -888888888, -222222222, -333333333, -555555555;
|
|
15
|
+
* anything <= -1,000,000) become empty cells with a `<column>_flag` column that says what the code means; null
|
|
16
|
+
* cells get a flag too; top-coded medians (home value 2000001, gross rent 2001/3501/4001) keep their value and
|
|
17
|
+
* get a flag. Values are otherwise exactly as published (text).
|
|
18
|
+
* - errors: since 2026-05-12 the API requires a key and redirects keyless data queries to missing_key.html (bad keys
|
|
19
|
+
* to invalid_key.html); these become key errors. A query that matches no geography answers HTTP 204: "no rows".
|
|
20
|
+
* - microdata (ACS PUMS, CPS): see _census_micro.ts.
|
|
21
|
+
* - search: variables.json of a dataset (default: latest ACS 5-year detailed tables), cached on disk for 30 days
|
|
22
|
+
* (metadata only; data queries are never cached), plus the catalog of common variables.
|
|
23
|
+
*
|
|
24
|
+
* Differences forced by this server's hook signatures (checkSpec and manualItems are synchronous here):
|
|
25
|
+
* - the county/place name check of the Python check_spec needs the network, so it runs at the start of fetch
|
|
26
|
+
* (same messages, a "spec" error before any data request);
|
|
27
|
+
* - manualItems resolves "latest" and geography names from what this process has already looked up (a fetch or
|
|
28
|
+
* manualItemsAsync); otherwise it falls back as the Python version did offline (release-calendar year; FIPS
|
|
29
|
+
* codes needed for names). manualItemsAsync is the full version.
|
|
30
|
+
*/
|
|
31
|
+
import fs from "node:fs";
|
|
32
|
+
import path from "node:path";
|
|
33
|
+
import zlib from "node:zlib";
|
|
34
|
+
import { parse } from "csv-parse/sync";
|
|
35
|
+
import { Cache } from "../core/cache.js";
|
|
36
|
+
import * as gr from "../core/generic-rest.js";
|
|
37
|
+
import { DataError } from "../core/http.js";
|
|
38
|
+
import { parsePyJson, pyFloatRepr, pyQuote, pyStr } from "../core/py.js";
|
|
39
|
+
import { asecRwIndex, asecRwValues, ASEC_REPS, ASEC_RW_DIR, datasetYear, fetchMicro, MAX_VARS, mergeChunks, microKind, microPlan } from "./_census_micro.js";
|
|
40
|
+
import { censusGet, ensureList, getCloseMatches, given, pyFloat, pyInt, pyRepr, respJson, splitlines, STATES, startsWithBrace, stateOf as rawState, title, truthy, ValueErr } from "./_census_util.js";
|
|
41
|
+
const WWW2 = "https://www2.census.gov/geo/docs/reference/codes2020";
|
|
42
|
+
const SENTINELS = new Map([
|
|
43
|
+
[-666666666, "-666666666: estimate could not be computed (too few sample observations, or a median in an open-ended interval)"],
|
|
44
|
+
[-999999999, "-999999999 (N): cannot be displayed, too few sample cases in this area"],
|
|
45
|
+
[-888888888, "-888888888 (X): not applicable or not available"],
|
|
46
|
+
[-222222222, "-222222222 (**): margin of error could not be computed, too few sample observations"],
|
|
47
|
+
[-333333333, "-333333333 (***): margin of error could not be computed, median in an open-ended interval"],
|
|
48
|
+
[-555555555, "-555555555 (*****): estimate is controlled; no sampling error, treat the margin of error as zero"],
|
|
49
|
+
]);
|
|
50
|
+
const TOPCODES = {
|
|
51
|
+
B25077_001E: { "2000001": "top-coded: $2,000,000 or more" },
|
|
52
|
+
B25064_001E: { "2001": "top-coded: $2,000 or more", "3501": "top-coded: $3,500 or more", "4001": "top-coded: $4,000 or more" },
|
|
53
|
+
};
|
|
54
|
+
// SIPP microdata: one row per person-month; these come with every request (keys to link months, the weight)
|
|
55
|
+
const SIPP_KEEP = ["SSUID", "PNUM", "MONTHCODE", "WPFINWGT"];
|
|
56
|
+
const MEMO_MS = 24 * 3600 * 1000;
|
|
57
|
+
/** newest vintage of each "latest/<rest>" (the Python _LATEST), for a day */
|
|
58
|
+
const LATEST = new Map();
|
|
59
|
+
/** geography name lists already fetched in this process: "<ds>|<level>|<state fips>" -> [items, note] */
|
|
60
|
+
const GEO_MEMO = new Map();
|
|
61
|
+
/** A geography name list is needed and not at hand (the async callers fetch it and retry). */
|
|
62
|
+
class NeedList extends Error {
|
|
63
|
+
level;
|
|
64
|
+
fips;
|
|
65
|
+
constructor(level, fips) {
|
|
66
|
+
super(`the ${level} names of state ${fips} need the Census name list (not fetched here)`);
|
|
67
|
+
this.level = level;
|
|
68
|
+
this.fips = fips;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
// ---------------------------------------------------------------------- helpers
|
|
72
|
+
function stateOf(x, as = "fips") {
|
|
73
|
+
try {
|
|
74
|
+
return rawState(x, as);
|
|
75
|
+
}
|
|
76
|
+
catch (e) {
|
|
77
|
+
if (!(e instanceof ValueErr))
|
|
78
|
+
throw e;
|
|
79
|
+
const close = getCloseMatches(title(pyStr(x).trim()), STATES.map(([, , n]) => n), 3, 0.6);
|
|
80
|
+
throw new ValueErr(`${e.message}.` + (close.length ? ` Did you mean: ${close.join("; ")}?` : ""));
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
function clientOf(pack) {
|
|
84
|
+
return gr.clientFor(pack);
|
|
85
|
+
}
|
|
86
|
+
function checkError(_status, body) {
|
|
87
|
+
const head = body.subarray(0, 600).toString("latin1").trimStart().toLowerCase();
|
|
88
|
+
if (head.startsWith("<")) {
|
|
89
|
+
if (head.includes("missing key")) {
|
|
90
|
+
return "the Census Data API refused a request without a key (redirect to missing_key.html; a key is " +
|
|
91
|
+
"required for every data query since 2026-05-12): set CENSUS_API_KEY";
|
|
92
|
+
}
|
|
93
|
+
if (head.includes("invalid key"))
|
|
94
|
+
return "the Census Data API rejected the key (redirect to invalid_key.html): check CENSUS_API_KEY";
|
|
95
|
+
}
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
export function isAcs(ds) {
|
|
99
|
+
return `/${ds.replace(/^\/+|\/+$/g, "")}/`.includes("/acs/");
|
|
100
|
+
}
|
|
101
|
+
/** SIPP microdata (2024/sipp, latest/sipp): rows are person-months, no geography but the state recode. */
|
|
102
|
+
export function isSipp(ds) {
|
|
103
|
+
return /(^|\/)sipp$/.test(pyStr(ds).trim().replace(/^\/+|\/+$/g, ""));
|
|
104
|
+
}
|
|
105
|
+
function withSippKeys(vars) {
|
|
106
|
+
return [...vars, ...SIPP_KEEP.filter((v) => !vars.includes(v))];
|
|
107
|
+
}
|
|
108
|
+
export function moeOf(v) {
|
|
109
|
+
if (/^[A-Z]{1,3}\d{2,6}[A-Z]{0,2}_\d{3,4}PE$/.test(v))
|
|
110
|
+
return v.slice(0, -2) + "PM";
|
|
111
|
+
if (/^[A-Z]{1,3}\d{2,6}[A-Z]{0,2}(_C\d{2})?_\d{3,4}[A-Z]?E$/.test(v))
|
|
112
|
+
return v.slice(0, -1) + "M";
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
export function withMoe(vars) {
|
|
116
|
+
const out = [];
|
|
117
|
+
for (const v of vars) {
|
|
118
|
+
if (!out.includes(v))
|
|
119
|
+
out.push(v);
|
|
120
|
+
const m = moeOf(v);
|
|
121
|
+
if (m && !vars.includes(m) && !out.includes(m))
|
|
122
|
+
out.push(m);
|
|
123
|
+
}
|
|
124
|
+
return out;
|
|
125
|
+
}
|
|
126
|
+
const stripSlashes = (s) => s.replace(/^\/+|\/+$/g, "");
|
|
127
|
+
// ---------------------------------------------------------------------- datasets
|
|
128
|
+
/** Release-calendar guess (only for offline manual links): ACS 1-year in September, 5-year in December. */
|
|
129
|
+
export function guessYear(rest, today = new Date()) {
|
|
130
|
+
const y = today.getFullYear(), m = today.getMonth() + 1;
|
|
131
|
+
if (rest.includes("acs5"))
|
|
132
|
+
return m === 12 ? y - 1 : y - 2;
|
|
133
|
+
if (rest.includes("acs1") || rest.includes("acsse"))
|
|
134
|
+
return m >= 10 ? y - 1 : y - 2;
|
|
135
|
+
return y - 2;
|
|
136
|
+
}
|
|
137
|
+
async function latestYear(client, rest) {
|
|
138
|
+
const memo = LATEST.get(rest);
|
|
139
|
+
if (memo && Date.now() - memo[1] < MEMO_MS)
|
|
140
|
+
return memo[0];
|
|
141
|
+
const thisYear = new Date().getFullYear();
|
|
142
|
+
const first = rest.startsWith("cps/") ? thisYear : thisYear - 1; // a CPS file carries its survey year (ASEC: March)
|
|
143
|
+
for (let y = first; y > thisYear - 8; y--) {
|
|
144
|
+
let r;
|
|
145
|
+
try {
|
|
146
|
+
r = await censusGet(client, `/${y}/${rest}.json`, null, true);
|
|
147
|
+
}
|
|
148
|
+
catch (e) {
|
|
149
|
+
if (e instanceof DataError && e.kind === "fixture_missing")
|
|
150
|
+
continue; // offline tests: years not recorded count as absent
|
|
151
|
+
throw e;
|
|
152
|
+
}
|
|
153
|
+
if (r.status === 200 && startsWithBrace(r.body)) {
|
|
154
|
+
LATEST.set(rest, [y, Date.now()]);
|
|
155
|
+
return y;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
throw new DataError(`census: no vintage of ${rest} found on the API for ${thisYear - 7}-${thisYear - 1}`, "not_found");
|
|
159
|
+
}
|
|
160
|
+
async function resolveDataset(client, ds0) {
|
|
161
|
+
const ds = stripSlashes(pyStr(ds0).trim());
|
|
162
|
+
if (ds.startsWith("latest/")) {
|
|
163
|
+
const rest = ds.slice("latest/".length);
|
|
164
|
+
const y = await latestYear(client, rest);
|
|
165
|
+
return [`${y}/${rest}`, `'${ds}' resolved to ${y}/${rest} (newest vintage on the API)`];
|
|
166
|
+
}
|
|
167
|
+
return [ds, ""];
|
|
168
|
+
}
|
|
169
|
+
/** resolveDataset from what this process already knows; null when "latest" was not looked up yet. */
|
|
170
|
+
function resolveDatasetKnown(ds0) {
|
|
171
|
+
const ds = stripSlashes(pyStr(ds0).trim());
|
|
172
|
+
if (!ds.startsWith("latest/"))
|
|
173
|
+
return ds;
|
|
174
|
+
const memo = LATEST.get(ds.slice(7));
|
|
175
|
+
return memo && Date.now() - memo[1] < MEMO_MS ? `${memo[0]}/${ds.slice(7)}` : null;
|
|
176
|
+
}
|
|
177
|
+
// ---------------------------------------------------------------------- geography names
|
|
178
|
+
const SUFFIX = /\s+(city and borough|census area|municipality|borough|parish|county|city|town|village|cdp|planning region|municipio)$/;
|
|
179
|
+
function base(s) {
|
|
180
|
+
let t = s.toLowerCase().split(",")[0].trim();
|
|
181
|
+
let prev = null;
|
|
182
|
+
while (prev !== t) {
|
|
183
|
+
prev = t;
|
|
184
|
+
t = t.replace(SUFFIX, "").trim();
|
|
185
|
+
}
|
|
186
|
+
return t;
|
|
187
|
+
}
|
|
188
|
+
const rsplitFirst = (s) => {
|
|
189
|
+
const i = s.lastIndexOf(",");
|
|
190
|
+
return i < 0 ? s : s.slice(0, i);
|
|
191
|
+
};
|
|
192
|
+
/** [{name, code, incorporated}], note. API NAME list of the dataset when a key is set, else the 2020 files. */
|
|
193
|
+
async function geoList(client, ds, level, stFips) {
|
|
194
|
+
let why = "no CENSUS_API_KEY";
|
|
195
|
+
let res = null;
|
|
196
|
+
if (client.key) {
|
|
197
|
+
try {
|
|
198
|
+
const data = respJson(await censusGet(client, `/${ds}`, { get: "NAME", for: `${level}:*`, in: `state:${stFips}` }));
|
|
199
|
+
if (truthy(data)) {
|
|
200
|
+
const [head, ...rows] = data;
|
|
201
|
+
const i = head.indexOf("NAME"), j = head.indexOf(level);
|
|
202
|
+
res = [rows.map((r) => ({ name: rsplitFirst(r[i]).trim(), code: r[j],
|
|
203
|
+
incorporated: / (city|town|village|borough|municipality)$/.test(rsplitFirst(r[i])) })), ""];
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
catch (e) {
|
|
207
|
+
if (!(e instanceof DataError))
|
|
208
|
+
throw e;
|
|
209
|
+
why = `the API name list failed (${e.kind})`;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
if (!res) {
|
|
213
|
+
const usps = stateOf(stFips, "usps").toLowerCase();
|
|
214
|
+
const kind = level === "county" ? "cou" : "place";
|
|
215
|
+
const url = `${WWW2}/${kind}/st${stFips}_${usps}_${kind}2020.txt`;
|
|
216
|
+
const text = (await censusGet(client, url)).text();
|
|
217
|
+
const out = [];
|
|
218
|
+
const lines = splitlines(text).filter((ln) => ln.trim());
|
|
219
|
+
const head = lines[0].split("|");
|
|
220
|
+
for (const ln of lines.slice(1)) {
|
|
221
|
+
const vals = ln.split("|");
|
|
222
|
+
const f = {};
|
|
223
|
+
head.forEach((h, k) => {
|
|
224
|
+
if (k < vals.length)
|
|
225
|
+
f[h] = vals[k];
|
|
226
|
+
});
|
|
227
|
+
if (level === "county")
|
|
228
|
+
out.push({ name: f.COUNTYNAME ?? "", code: f.COUNTYFP ?? "", incorporated: false });
|
|
229
|
+
else
|
|
230
|
+
out.push({ name: f.PLACENAME ?? "", code: f.PLACEFP ?? "", incorporated: (f.TYPE ?? "").toUpperCase().startsWith("INCORPORATED") });
|
|
231
|
+
}
|
|
232
|
+
res = [out, `${level} names matched against the Census 2020 reference file (${why}); codes that changed after ` +
|
|
233
|
+
"2020 (Connecticut planning regions from ACS 2022 on) are not in it"];
|
|
234
|
+
}
|
|
235
|
+
GEO_MEMO.set(`${ds}|${level}|${stFips}`, [res[0], res[1], Date.now()]);
|
|
236
|
+
return res;
|
|
237
|
+
}
|
|
238
|
+
function knownLists(ds) {
|
|
239
|
+
return (level, fips) => {
|
|
240
|
+
const m = GEO_MEMO.get(`${ds}|${level}|${fips}`);
|
|
241
|
+
return m && Date.now() - m[2] < MEMO_MS ? [m[0], m[1]] : undefined;
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
/** [code, matched name, note] for a county or place given by name or code. */
|
|
245
|
+
function resolveGeo(lists, level, value, stFips) {
|
|
246
|
+
const raw = pyStr(value).trim();
|
|
247
|
+
const digits = level === "county" ? 3 : 5;
|
|
248
|
+
if (raw === "*" || new RegExp(`^\\d{${digits}}$`).test(raw))
|
|
249
|
+
return [raw, raw, ""];
|
|
250
|
+
if (level === "county" && /^\d{5}$/.test(raw)) {
|
|
251
|
+
if (raw.slice(0, 2) !== stFips)
|
|
252
|
+
throw new ValueErr(`county FIPS ${raw} is not in state ${stFips}`);
|
|
253
|
+
return [raw.slice(2), raw, ""];
|
|
254
|
+
}
|
|
255
|
+
const got = lists(level, stFips);
|
|
256
|
+
if (!got)
|
|
257
|
+
throw new NeedList(level, stFips);
|
|
258
|
+
const [items, note] = got;
|
|
259
|
+
const stName = stateOf(stFips, "name");
|
|
260
|
+
const want = raw.toLowerCase().split(",")[0].trim();
|
|
261
|
+
const exact = items.filter((x) => x.name.toLowerCase() === want);
|
|
262
|
+
let hits = exact.length ? exact : items.filter((x) => base(x.name) === base(raw));
|
|
263
|
+
if (hits.length > 1 && level === "place") {
|
|
264
|
+
const inc = hits.filter((x) => x.incorporated);
|
|
265
|
+
hits = inc.length === 1 ? inc.slice(0, 1) : hits;
|
|
266
|
+
}
|
|
267
|
+
if (hits.length === 1)
|
|
268
|
+
return [hits[0].code, hits[0].name, note];
|
|
269
|
+
if (hits.length > 1) {
|
|
270
|
+
throw new ValueErr(`${level} ${pyRepr(raw)} is ambiguous in ${stName}: ` +
|
|
271
|
+
hits.slice(0, 8).map((x) => `${x.name} (${x.code})`).join("; ") + ". Give the full name or the code.");
|
|
272
|
+
}
|
|
273
|
+
const names = items.map((x) => x.name);
|
|
274
|
+
const close = names.filter((n) => n.toLowerCase().includes(base(raw))).slice(0, 5);
|
|
275
|
+
const bases = new Map();
|
|
276
|
+
for (const n of names)
|
|
277
|
+
bases.set(base(n), n);
|
|
278
|
+
for (const c of [...getCloseMatches(raw, names, 5, 0.6),
|
|
279
|
+
...getCloseMatches(base(raw), [...bases.keys()], 5, 0.6).map((b) => bases.get(b))]) {
|
|
280
|
+
if (!close.includes(c))
|
|
281
|
+
close.push(c);
|
|
282
|
+
}
|
|
283
|
+
throw new ValueErr(`no ${level} named ${pyRepr(raw)} in ${stName}.` +
|
|
284
|
+
(close.length ? ` Did you mean: ${close.slice(0, 6).join("; ")}?` : ""));
|
|
285
|
+
}
|
|
286
|
+
/** Run fn with geography lists, fetching each list it asks for (fresh for this call, as the Python version did). */
|
|
287
|
+
async function withLists(client, ds, fn) {
|
|
288
|
+
const local = new Map();
|
|
289
|
+
for (let i = 0; i < 4; i++) {
|
|
290
|
+
try {
|
|
291
|
+
return fn((level, fips) => local.get(`${level}|${fips}`));
|
|
292
|
+
}
|
|
293
|
+
catch (e) {
|
|
294
|
+
if (!(e instanceof NeedList))
|
|
295
|
+
throw e;
|
|
296
|
+
local.set(`${e.level}|${e.fips}`, await geoList(client, ds, e.level, e.fips));
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
return fn((level, fips) => local.get(`${level}|${fips}`));
|
|
300
|
+
}
|
|
301
|
+
const joinList = (v) => (Array.isArray(v) ? v.map(pyStr).join(",") : v);
|
|
302
|
+
/** API query parameters from the spec's params (convenience names resolved). Throws ValueErr (or NeedList). */
|
|
303
|
+
function buildQuery(lists, ds, params) {
|
|
304
|
+
const p = { ...params };
|
|
305
|
+
const notes = [];
|
|
306
|
+
const pop = (k, d = undefined) => {
|
|
307
|
+
const v = k in p ? p[k] : d;
|
|
308
|
+
delete p[k];
|
|
309
|
+
return v;
|
|
310
|
+
};
|
|
311
|
+
let vars = ensureList(pop("get", ""));
|
|
312
|
+
const moe = pop("moe", true);
|
|
313
|
+
if (isAcs(ds)) {
|
|
314
|
+
if (moe === false || pyStr(moe).toLowerCase() === "false")
|
|
315
|
+
throw new ValueErr("ACS margins of error are never dropped: remove moe: false");
|
|
316
|
+
const full = withMoe(vars);
|
|
317
|
+
const added = full.filter((v) => !vars.includes(v));
|
|
318
|
+
if (added.length)
|
|
319
|
+
notes.push(`margins of error added: ${added.join(", ")}`);
|
|
320
|
+
vars = full;
|
|
321
|
+
}
|
|
322
|
+
const [st, co, pl, z] = ["state", "county", "place", "zcta"].map((k) => pop(k, null));
|
|
323
|
+
if (isSipp(ds)) {
|
|
324
|
+
const full = withSippKeys(vars);
|
|
325
|
+
const added = full.filter((v) => !vars.includes(v));
|
|
326
|
+
if (added.length) {
|
|
327
|
+
notes.push(`SIPP person-month keys and weight added: ${added.join(", ")} (weight every estimate by WPFINWGT)`);
|
|
328
|
+
}
|
|
329
|
+
const q = { get: full.join(",") };
|
|
330
|
+
if ([co, pl, z].some(given) || truthy(p.for) || truthy(p.in)) {
|
|
331
|
+
throw new ValueErr("SIPP has no geography below the state (the API has none at all): use state, " +
|
|
332
|
+
"which filters on TST_INTV, the state of the interview address");
|
|
333
|
+
}
|
|
334
|
+
if (given(st) && st !== "*") {
|
|
335
|
+
q.TST_INTV = stateOf(st, "fips");
|
|
336
|
+
notes.push(`state ${pyRepr(st)} -> TST_INTV=${q.TST_INTV} (state of the interview address; SIPP is ` +
|
|
337
|
+
"designed for national estimates, so state figures from small samples are unreliable)");
|
|
338
|
+
}
|
|
339
|
+
for (const [k, v] of Object.entries(p))
|
|
340
|
+
if (v !== null && v !== undefined && v !== "")
|
|
341
|
+
q[k] = joinList(v);
|
|
342
|
+
if (!("MONTHCODE" in q)) {
|
|
343
|
+
notes.push("no MONTHCODE filter: each person appears once per reference month (up to 12 rows); " +
|
|
344
|
+
"filter MONTHCODE (12 = December) or average over months");
|
|
345
|
+
}
|
|
346
|
+
return [q, notes];
|
|
347
|
+
}
|
|
348
|
+
const q = { get: vars.join(",") };
|
|
349
|
+
let fips = null;
|
|
350
|
+
if (given(st) && pyStr(st).trim() !== "*")
|
|
351
|
+
fips = stateOf(st, "fips");
|
|
352
|
+
if (given(z)) {
|
|
353
|
+
if (!/^(\d{5}|\*)$/.test(pyStr(z).trim()))
|
|
354
|
+
throw new ValueErr(`zcta must be a 5-digit ZIP code area or *, not ${pyRepr(z)}`);
|
|
355
|
+
q.for = `zip code tabulation area:${pyStr(z).trim()}`;
|
|
356
|
+
}
|
|
357
|
+
else if (given(pl)) {
|
|
358
|
+
if (!fips)
|
|
359
|
+
throw new ValueErr("place needs a state");
|
|
360
|
+
const [code, name, note] = resolveGeo(lists, "place", pl, fips);
|
|
361
|
+
[q.for, q.in] = [`place:${code}`, `state:${fips}`];
|
|
362
|
+
notes.push(`place ${pyRepr(pl)} -> ${name} (${fips}${code})`, ...(note ? [note] : []));
|
|
363
|
+
}
|
|
364
|
+
else if (given(co)) {
|
|
365
|
+
if (!fips)
|
|
366
|
+
throw new ValueErr("county needs a state");
|
|
367
|
+
const [code, name, note] = resolveGeo(lists, "county", co, fips);
|
|
368
|
+
if (truthy(p.for))
|
|
369
|
+
[q.for, q.in] = [pop("for"), `state:${fips} county:${code}`]; // e.g. tract:* within the county
|
|
370
|
+
else
|
|
371
|
+
[q.for, q.in] = [`county:${code}`, `state:${fips}`];
|
|
372
|
+
if (code !== "*")
|
|
373
|
+
notes.push(`county ${pyRepr(co)} -> ${name} (${fips}${code})`, ...(note ? [note] : []));
|
|
374
|
+
}
|
|
375
|
+
else if (given(st)) {
|
|
376
|
+
if (truthy(p.for)) {
|
|
377
|
+
q.for = pop("for");
|
|
378
|
+
q.in = pop("in", null) || `state:${fips}`;
|
|
379
|
+
}
|
|
380
|
+
else
|
|
381
|
+
q.for = `state:${fips || "*"}`;
|
|
382
|
+
}
|
|
383
|
+
for (const [k, v] of Object.entries(p)) {
|
|
384
|
+
if (v === null || v === undefined || v === "")
|
|
385
|
+
continue;
|
|
386
|
+
q[k] = joinList(v);
|
|
387
|
+
}
|
|
388
|
+
return [q, notes];
|
|
389
|
+
}
|
|
390
|
+
// ---------------------------------------------------------------------- spec checks
|
|
391
|
+
function datasetOf(pack, r) {
|
|
392
|
+
const p = r.params || {};
|
|
393
|
+
const ds = pyStr(p.dataset || r.dataset || "");
|
|
394
|
+
const cat = pack.endpoints?.catalog || {};
|
|
395
|
+
return ds in cat ? cat[ds].id : ds;
|
|
396
|
+
}
|
|
397
|
+
function checkSpec(spec, pack) {
|
|
398
|
+
const probs = [];
|
|
399
|
+
for (const r of spec.requests || []) {
|
|
400
|
+
if ((r.endpoint || "query") !== "query")
|
|
401
|
+
continue;
|
|
402
|
+
const name = pyRepr(r.name ?? null);
|
|
403
|
+
const p = r.params || {};
|
|
404
|
+
const ds = datasetOf(pack, r);
|
|
405
|
+
const vars = ensureList(p.get);
|
|
406
|
+
if (microKind(ds)) {
|
|
407
|
+
try {
|
|
408
|
+
microPlan(ds, p);
|
|
409
|
+
}
|
|
410
|
+
catch (e) {
|
|
411
|
+
if (!(e instanceof ValueErr))
|
|
412
|
+
throw e;
|
|
413
|
+
probs.push(`request ${name}: ${e.message}`);
|
|
414
|
+
}
|
|
415
|
+
continue;
|
|
416
|
+
}
|
|
417
|
+
if (!vars.length)
|
|
418
|
+
probs.push(`request ${name}: 'get' needs at least one variable (NAME,B19013_001E)`);
|
|
419
|
+
const n = (isAcs(ds) ? withMoe(vars) : isSipp(ds) ? withSippKeys(vars) : vars).length;
|
|
420
|
+
if (n > MAX_VARS) {
|
|
421
|
+
probs.push(`request ${name}: ${n} variables (margins of error or SIPP keys included); the API takes ` +
|
|
422
|
+
`at most ${MAX_VARS} per call: split the request`);
|
|
423
|
+
}
|
|
424
|
+
if (isAcs(ds) && pyStr(p.moe ?? true).toLowerCase() === "false") {
|
|
425
|
+
probs.push(`request ${name}: ACS margins of error are never dropped: remove moe: false`);
|
|
426
|
+
}
|
|
427
|
+
if (isSipp(ds)) {
|
|
428
|
+
if (["county", "place", "zcta", "for", "in", "ucgid"].some((k) => given(p[k]))) {
|
|
429
|
+
probs.push(`request ${name}: SIPP has no geography below the state: use state (it filters ` +
|
|
430
|
+
"TST_INTV) or no geography for national estimates");
|
|
431
|
+
}
|
|
432
|
+
continue;
|
|
433
|
+
}
|
|
434
|
+
if (stripSlashes(ds).startsWith("timeseries/intltrade")) {
|
|
435
|
+
if (!truthy(p.time)) {
|
|
436
|
+
probs.push(`request ${name}: international trade needs a time (2026-08, or 'from 2025-01 to ` +
|
|
437
|
+
"2025-12'); it has no for= geography (filter CTY_CODE, COMM_LVL, I_COMMODITY)");
|
|
438
|
+
}
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
if (stripSlashes(ds).startsWith("timeseries/") && !truthy(p.time) && !truthy(p.YEAR)) {
|
|
442
|
+
probs.push(`request ${name}: a time series needs time (2026, 2026-Q2, 2026-08, or 'from 2024 to ` +
|
|
443
|
+
"2026'); without it the API returns every period");
|
|
444
|
+
}
|
|
445
|
+
if (!["for", "ucgid", "state", "zcta"].some((k) => given(p[k]))) {
|
|
446
|
+
probs.push(`request ${name}: give a geography: for (state:*, county:*), state, or zcta`);
|
|
447
|
+
}
|
|
448
|
+
if ((truthy(p.county) || truthy(p.place)) && !truthy(p.state))
|
|
449
|
+
probs.push(`request ${name}: county and place need a state`);
|
|
450
|
+
const geo = `${pyStr(p.for ?? "")} ${pyStr(p.zcta ?? "")}`.toLowerCase();
|
|
451
|
+
if ((ds.includes("acs1") && ["tract", "block group", "zip code"].some((g) => geo.includes(g))) || (ds.includes("acs1") && truthy(p.zcta))) {
|
|
452
|
+
probs.push(`request ${name}: ACS 1-year has no tracts, block groups or ZCTAs (areas of 65,000+ ` +
|
|
453
|
+
"only): use the 5-year dataset (acs5)");
|
|
454
|
+
}
|
|
455
|
+
if (given(p.state) && p.state !== "*") {
|
|
456
|
+
try {
|
|
457
|
+
stateOf(p.state, "fips");
|
|
458
|
+
}
|
|
459
|
+
catch (e) {
|
|
460
|
+
if (!(e instanceof ValueErr))
|
|
461
|
+
throw e;
|
|
462
|
+
probs.push(`request ${name}: ${e.message}`);
|
|
463
|
+
}
|
|
464
|
+
// county and place names: geoProblems, at the start of fetch (they need the name lists)
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
return probs;
|
|
468
|
+
}
|
|
469
|
+
/** The county/place name part of the Python check_spec (it needs the Census name lists). */
|
|
470
|
+
async function geoProblems(spec, pack, client) {
|
|
471
|
+
const probs = [];
|
|
472
|
+
for (const r of spec.requests || []) {
|
|
473
|
+
if ((r.endpoint || "query") !== "query")
|
|
474
|
+
continue;
|
|
475
|
+
const p = r.params || {};
|
|
476
|
+
const ds = datasetOf(pack, r);
|
|
477
|
+
if (microKind(ds) || isSipp(ds) || stripSlashes(ds).startsWith("timeseries/intltrade"))
|
|
478
|
+
continue;
|
|
479
|
+
if (!given(p.state) || p.state === "*")
|
|
480
|
+
continue;
|
|
481
|
+
let fips;
|
|
482
|
+
try {
|
|
483
|
+
fips = stateOf(p.state, "fips");
|
|
484
|
+
}
|
|
485
|
+
catch {
|
|
486
|
+
continue; // checkSpec reported it
|
|
487
|
+
}
|
|
488
|
+
for (const level of ["county", "place"]) {
|
|
489
|
+
if (!given(p[level]))
|
|
490
|
+
continue;
|
|
491
|
+
const yearDs = !ds.startsWith("latest/") ? ds : `${guessYear(ds)}/${ds.slice(7)}`;
|
|
492
|
+
try {
|
|
493
|
+
await withLists(client, yearDs, (lists) => resolveGeo(lists, level, p[level], fips));
|
|
494
|
+
}
|
|
495
|
+
catch (e) {
|
|
496
|
+
if (e instanceof ValueErr)
|
|
497
|
+
probs.push(`request ${pyRepr(r.name ?? null)}: ${e.message}`);
|
|
498
|
+
else if (!(e instanceof DataError))
|
|
499
|
+
throw e; // unreachable here: the fetch resolves it later
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
return probs;
|
|
504
|
+
}
|
|
505
|
+
// ---------------------------------------------------------------------- normalize
|
|
506
|
+
function isVar(col) {
|
|
507
|
+
return col === col.toUpperCase() && /\d/.test(col) && col !== "NAME" && col !== "GEO_ID";
|
|
508
|
+
}
|
|
509
|
+
export function normalize(_req, _ep, rows) {
|
|
510
|
+
if (!rows.length)
|
|
511
|
+
return rows;
|
|
512
|
+
const cols = Object.keys(rows[0]).filter((c) => isVar(c) && !c.endsWith("EA") && !c.endsWith("MA") && !c.endsWith("_flag"));
|
|
513
|
+
const flagged = new Set();
|
|
514
|
+
for (const r of rows) {
|
|
515
|
+
for (const c of cols) {
|
|
516
|
+
let v = r[c];
|
|
517
|
+
let flag = "";
|
|
518
|
+
if (v === null || v === undefined || v === "") {
|
|
519
|
+
flag = "null: no data for this geography";
|
|
520
|
+
v = "";
|
|
521
|
+
}
|
|
522
|
+
else {
|
|
523
|
+
const x = pyFloat(pyStr(v));
|
|
524
|
+
if (x !== null && x <= -1_000_000) {
|
|
525
|
+
const i = Math.trunc(x);
|
|
526
|
+
flag = SENTINELS.get(i) ?? `${i}: Census special value (not a number)`;
|
|
527
|
+
v = "";
|
|
528
|
+
}
|
|
529
|
+
else if (pyStr(v) in (TOPCODES[c] || {}))
|
|
530
|
+
flag = TOPCODES[c][pyStr(v)];
|
|
531
|
+
}
|
|
532
|
+
r[c] = v;
|
|
533
|
+
if (flag) {
|
|
534
|
+
r[`${c}_flag`] = flag;
|
|
535
|
+
flagged.add(c);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
if (!flagged.size)
|
|
540
|
+
return rows;
|
|
541
|
+
return rows.map((r) => {
|
|
542
|
+
const o = {};
|
|
543
|
+
for (const [k, v] of Object.entries(r)) {
|
|
544
|
+
if (k.endsWith("_flag") && flagged.has(k.slice(0, -5)))
|
|
545
|
+
continue;
|
|
546
|
+
o[k] = v;
|
|
547
|
+
if (flagged.has(k))
|
|
548
|
+
o[`${k}_flag`] = r[`${k}_flag`] ?? "";
|
|
549
|
+
}
|
|
550
|
+
return o;
|
|
551
|
+
});
|
|
552
|
+
}
|
|
553
|
+
// ---------------------------------------------------------------------- fetch
|
|
554
|
+
function table(data) {
|
|
555
|
+
if (!truthy(data))
|
|
556
|
+
return [];
|
|
557
|
+
const [head, ...rows] = data;
|
|
558
|
+
return rows.map((r) => {
|
|
559
|
+
const o = {};
|
|
560
|
+
const n = Math.min(head.length, r.length);
|
|
561
|
+
for (let i = 0; i < n; i++)
|
|
562
|
+
o[head[i]] = r[i];
|
|
563
|
+
return o;
|
|
564
|
+
});
|
|
565
|
+
}
|
|
566
|
+
const CLEAN = { missing: [""] };
|
|
567
|
+
const CLEAN_FLAT = { missing: [""], flatten: false };
|
|
568
|
+
async function fetchOne(client, pack, r) {
|
|
569
|
+
const [ep, params, probs] = gr.resolve(pack, r);
|
|
570
|
+
if (probs.length)
|
|
571
|
+
throw new DataError(probs.join("; "), "bad_request");
|
|
572
|
+
const kind = r.endpoint || "query";
|
|
573
|
+
const dsIn = params.dataset;
|
|
574
|
+
delete params.dataset;
|
|
575
|
+
const [ds, note] = await resolveDataset(client, dsIn);
|
|
576
|
+
const notes = note ? [`${r.name}: ${note}`] : [];
|
|
577
|
+
if (kind === "group_variables") {
|
|
578
|
+
const data = respJson(await censusGet(client, `/${ds}/groups/${pyStr(params.group)}.json`)) || {};
|
|
579
|
+
const vars = data.variables || {};
|
|
580
|
+
const rows = Object.keys(vars).sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)).map((k) => ({ variable: k, ...gr.flatten(vars[k]) }));
|
|
581
|
+
return [gr.clean(rows, CLEAN), notes];
|
|
582
|
+
}
|
|
583
|
+
if (kind === "groups") {
|
|
584
|
+
const data = respJson(await censusGet(client, `/${ds}/groups.json`)) || {};
|
|
585
|
+
return [gr.clean(data.groups || [], CLEAN), notes];
|
|
586
|
+
}
|
|
587
|
+
if (microKind(ds)) {
|
|
588
|
+
const [tables, mnotes] = await fetchMicro(client, pack, r, ds, params);
|
|
589
|
+
return [tables, [...notes, ...mnotes]];
|
|
590
|
+
}
|
|
591
|
+
let q, gnotes;
|
|
592
|
+
try {
|
|
593
|
+
[q, gnotes] = await withLists(client, ds, (lists) => buildQuery(lists, ds, params));
|
|
594
|
+
}
|
|
595
|
+
catch (e) {
|
|
596
|
+
if (e instanceof ValueErr)
|
|
597
|
+
throw new DataError(`request ${pyRepr(r.name)}: ${e.message}`, "bad_request");
|
|
598
|
+
throw e;
|
|
599
|
+
}
|
|
600
|
+
notes.push(...gnotes.map((n) => `${r.name}: ${n}`));
|
|
601
|
+
const data = respJson(await censusGet(client, `/${ds}`, q));
|
|
602
|
+
if (data === null) {
|
|
603
|
+
throw new DataError(`request ${pyRepr(r.name)}: the Census API returned no rows (HTTP 204: no geography matched ` +
|
|
604
|
+
`for=${q.for === undefined || q.for === null ? "None" : pyStr(q.for)} in=${pyStr(q.in ?? "")}); check the codes`, "bad_request");
|
|
605
|
+
}
|
|
606
|
+
// SIPP microdata has no geography sentinels: blanks mean "not in universe", values stay as published
|
|
607
|
+
const rows = isSipp(ds) ? table(data) : normalize(r, ep, table(data));
|
|
608
|
+
return [gr.clean(rows, CLEAN_FLAT), notes];
|
|
609
|
+
}
|
|
610
|
+
async function fetch(spec, pack, _route) {
|
|
611
|
+
const client = clientOf(pack);
|
|
612
|
+
const probs = await geoProblems(spec, pack, client);
|
|
613
|
+
if (probs.length)
|
|
614
|
+
throw new DataError(`census: the request has problems: ${probs.join("; ")}`, "spec");
|
|
615
|
+
const tables = {};
|
|
616
|
+
const notes = [];
|
|
617
|
+
for (const r of spec.requests) {
|
|
618
|
+
const [got0, n] = await fetchOne(client, pack, r);
|
|
619
|
+
const got = Array.isArray(got0) ? { [r.name]: got0 } : got0;
|
|
620
|
+
if (!got[r.name].length && !r.allow_empty) {
|
|
621
|
+
throw new DataError(`request ${pyRepr(r.name)} returned no rows: check the variables and geography` +
|
|
622
|
+
(Object.keys(got).length > 1 ? " (and the filters: no person matched them)" : ""), "bad_request");
|
|
623
|
+
}
|
|
624
|
+
Object.assign(tables, got);
|
|
625
|
+
notes.push(...n);
|
|
626
|
+
}
|
|
627
|
+
return [tables, notes];
|
|
628
|
+
}
|
|
629
|
+
async function ping(client) {
|
|
630
|
+
try {
|
|
631
|
+
await censusGet(client, "/2023/acs/acs5", { get: "NAME", for: "us:1" });
|
|
632
|
+
return { auth_ok: true, kind: "", detail: "data query answered with the key" };
|
|
633
|
+
}
|
|
634
|
+
catch (e) {
|
|
635
|
+
if (!(e instanceof DataError))
|
|
636
|
+
throw e;
|
|
637
|
+
return { auth_ok: !["auth", "key_missing"].includes(e.kind), kind: e.kind, detail: e.message };
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
// ---------------------------------------------------------------------- manual route
|
|
641
|
+
/** urllib.parse.urlencode(q, safe=":*,()") */
|
|
642
|
+
function urlencodeSafe(q) {
|
|
643
|
+
const qp = (s) => pyQuote(s, ":*,() ").replace(/ /g, "+");
|
|
644
|
+
return Object.entries(q).map(([k, v]) => `${qp(k)}=${qp(pyStr(v))}`).join("&");
|
|
645
|
+
}
|
|
646
|
+
function manualItemsWith(spec, pack, how) {
|
|
647
|
+
const baseUrl = String(pack.endpoints.base_url).replace(/\/+$/, "");
|
|
648
|
+
const items = [];
|
|
649
|
+
for (const r of spec.requests || []) {
|
|
650
|
+
const [, params0, probs] = gr.resolve(pack, r);
|
|
651
|
+
if (probs.length)
|
|
652
|
+
continue;
|
|
653
|
+
const params = structuredClone(params0);
|
|
654
|
+
const rawDs = stripSlashes(pyStr(params.dataset));
|
|
655
|
+
delete params.dataset;
|
|
656
|
+
const [ds, note] = how.dataset(rawDs);
|
|
657
|
+
const kind = r.endpoint || "query";
|
|
658
|
+
let url, needs;
|
|
659
|
+
if (kind === "group_variables")
|
|
660
|
+
[url, needs] = [`${baseUrl}/${ds}/groups/${pyStr(params.group ?? null)}.json`, false];
|
|
661
|
+
else if (kind === "groups")
|
|
662
|
+
[url, needs] = [`${baseUrl}/${ds}/groups.json`, false];
|
|
663
|
+
else if (microKind(ds)) {
|
|
664
|
+
let plan;
|
|
665
|
+
try {
|
|
666
|
+
plan = microPlan(ds, params);
|
|
667
|
+
}
|
|
668
|
+
catch (e) {
|
|
669
|
+
if (e instanceof ValueErr)
|
|
670
|
+
continue; // the fetch reports it as a spec problem
|
|
671
|
+
throw e;
|
|
672
|
+
}
|
|
673
|
+
const calls = plan.geos.flatMap((g) => plan.chunks.map((c) => [g, c]));
|
|
674
|
+
calls.forEach(([g, chunk], i0) => {
|
|
675
|
+
const i = i0 + 1;
|
|
676
|
+
const q = { get: chunk.join(","), ...plan.preds, ...g };
|
|
677
|
+
items.push({ name: `${r.name}#${i}`, request: r.name, url: `${baseUrl}/${ds}?` + urlencodeSafe(q) + "&key={API_KEY}",
|
|
678
|
+
filename: `${r.name}_${String(i).padStart(3, "0")}.json`, also_ext: [], needs_key: true, method: "GET", body: null,
|
|
679
|
+
note: [...note, ...(calls.length > 1 ? [`call ${i} of ${calls.length}; ingest joins the files on ${plan.join.join(", ")}`] : [])].join("; ") });
|
|
680
|
+
});
|
|
681
|
+
const y = datasetYear(ds);
|
|
682
|
+
if (plan.asec_reps && y) { // Census's replicate weight file and its layout, no key
|
|
683
|
+
const stem = ASEC_RW_DIR(y) + `CPS_ASEC_ASCII_REPWGT_${y}`;
|
|
684
|
+
for (const [ext, fname] of [[y >= 2020 ? ".DAT.GZ" : ".dat.gz", `${r.name}_repwgt.dat.gz`], [".SAS", `${r.name}_repwgt.sas`]]) {
|
|
685
|
+
items.push({ name: `${r.name}#${fname}`, request: r.name, url: stem + ext, filename: fname, also_ext: [],
|
|
686
|
+
needs_key: false, method: "GET", body: null, note: "ASEC replicate weights (about 70 MB) and the layout with " +
|
|
687
|
+
"their totals; ingest joins them on PH_SEQ and PPPOS" });
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
continue;
|
|
691
|
+
}
|
|
692
|
+
else {
|
|
693
|
+
let q;
|
|
694
|
+
try {
|
|
695
|
+
[q] = buildQuery(how.lists(ds), ds, params);
|
|
696
|
+
}
|
|
697
|
+
catch (e) {
|
|
698
|
+
if (!(e instanceof ValueErr || e instanceof NeedList || e instanceof DataError))
|
|
699
|
+
throw e;
|
|
700
|
+
q = Object.fromEntries(Object.entries(params).filter(([k]) => !["county", "place", "moe"].includes(k)));
|
|
701
|
+
q.get = (isAcs(ds) ? withMoe(ensureList(params.get)) : ensureList(params.get)).join(",");
|
|
702
|
+
note.push(`geography names could not be resolved here (${e.message}); put the FIPS codes in for/in`);
|
|
703
|
+
}
|
|
704
|
+
url = `${baseUrl}/${ds}?` + urlencodeSafe(q) + "&key={API_KEY}";
|
|
705
|
+
needs = true;
|
|
706
|
+
}
|
|
707
|
+
items.push({ name: `${r.name}#1`, request: r.name, url, filename: `${r.name}.json`, also_ext: [".csv"],
|
|
708
|
+
needs_key: needs, method: "GET", body: null, note: note.join("; ") });
|
|
709
|
+
}
|
|
710
|
+
return items;
|
|
711
|
+
}
|
|
712
|
+
const guessed = (rawDs) => {
|
|
713
|
+
if (!rawDs.startsWith("latest/"))
|
|
714
|
+
return [rawDs, []];
|
|
715
|
+
const ds = `${guessYear(rawDs)}/${rawDs.slice(7)}`;
|
|
716
|
+
return [ds, [`year ${ds.slice(0, 4)} guessed from the release calendar: if the link fails, try one year earlier`]];
|
|
717
|
+
};
|
|
718
|
+
/** Download links (synchronous: "latest" and names from what this process has looked up, else the offline fallback). */
|
|
719
|
+
function manualItems(spec, pack) {
|
|
720
|
+
return manualItemsWith(spec, pack, {
|
|
721
|
+
dataset: (raw) => {
|
|
722
|
+
const known = resolveDatasetKnown(raw);
|
|
723
|
+
return known ? [known, []] : guessed(raw);
|
|
724
|
+
},
|
|
725
|
+
lists: knownLists,
|
|
726
|
+
});
|
|
727
|
+
}
|
|
728
|
+
/** The full manual_items of the Python adapter: resolves "latest" and geography names over the network first. */
|
|
729
|
+
export async function manualItemsAsync(spec, pack) {
|
|
730
|
+
const client = clientOf(pack);
|
|
731
|
+
const ds = new Map();
|
|
732
|
+
const lists = new Map();
|
|
733
|
+
for (const r of spec.requests || []) {
|
|
734
|
+
const [, params, probs] = gr.resolve(pack, r);
|
|
735
|
+
if (probs.length)
|
|
736
|
+
continue;
|
|
737
|
+
const raw = stripSlashes(pyStr(params.dataset));
|
|
738
|
+
let d;
|
|
739
|
+
try {
|
|
740
|
+
d = [(await resolveDataset(client, raw))[0], []];
|
|
741
|
+
}
|
|
742
|
+
catch (e) {
|
|
743
|
+
if (!(e instanceof DataError))
|
|
744
|
+
throw e;
|
|
745
|
+
d = guessed(raw);
|
|
746
|
+
}
|
|
747
|
+
ds.set(raw, d);
|
|
748
|
+
if ((r.endpoint || "query") !== "query" || microKind(d[0]))
|
|
749
|
+
continue;
|
|
750
|
+
const local = lists.get(d[0]) ?? new Map();
|
|
751
|
+
lists.set(d[0], local);
|
|
752
|
+
const p = { ...params };
|
|
753
|
+
delete p.dataset;
|
|
754
|
+
for (let i = 0; i < 4; i++) {
|
|
755
|
+
try {
|
|
756
|
+
buildQuery((level, fips) => local.get(`${level}|${fips}`), d[0], p);
|
|
757
|
+
break;
|
|
758
|
+
}
|
|
759
|
+
catch (e) {
|
|
760
|
+
if (!(e instanceof NeedList))
|
|
761
|
+
break;
|
|
762
|
+
try {
|
|
763
|
+
local.set(`${e.level}|${e.fips}`, await geoList(client, d[0], e.level, e.fips));
|
|
764
|
+
}
|
|
765
|
+
catch (e2) {
|
|
766
|
+
if (!(e2 instanceof DataError))
|
|
767
|
+
throw e2;
|
|
768
|
+
break;
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
return manualItemsWith(spec, pack, {
|
|
774
|
+
dataset: (raw) => {
|
|
775
|
+
const d = ds.get(raw) ?? guessed(raw);
|
|
776
|
+
return [d[0], [...d[1]]];
|
|
777
|
+
},
|
|
778
|
+
lists: (d) => (level, fips) => lists.get(d)?.get(`${level}|${fips}`),
|
|
779
|
+
});
|
|
780
|
+
}
|
|
781
|
+
// ---------------------------------------------------------------------- uploads
|
|
782
|
+
/** A data.census.gov "Download Table" CSV: one row per label, columns "<geography>!!<measure>". Returned long:
|
|
783
|
+
* label, geography, measure, value (a number: commas, %, ± removed; empty when not one) and value_text. */
|
|
784
|
+
function dcgCsv(file) {
|
|
785
|
+
const text = fs.readFileSync(file, "utf8").replace(/^\uFEFF/, "");
|
|
786
|
+
const rd = parse(text, { relax_column_count: true, relax_quotes: true });
|
|
787
|
+
if (!rd.length || !rd[0].slice(1).join("").includes("!!")) {
|
|
788
|
+
throw new DataError(`${path.basename(file)}: not a data.census.gov table export (its columns read ` +
|
|
789
|
+
'"<geography>!!Estimate"); upload that CSV or the API JSON', "bad_request");
|
|
790
|
+
}
|
|
791
|
+
const [head, ...rest] = rd;
|
|
792
|
+
const out = [];
|
|
793
|
+
for (const row of rest) {
|
|
794
|
+
const label = (row.length ? row[0] : "").replace(/\xa0/g, " ").trim();
|
|
795
|
+
const n = Math.min(head.length, row.length);
|
|
796
|
+
for (let i = 1; i < n; i++) {
|
|
797
|
+
const col = head[i];
|
|
798
|
+
const j = col.indexOf("!!");
|
|
799
|
+
const [geo, measure] = j < 0 ? [col, ""] : [col.slice(0, j), col.slice(j + 2)];
|
|
800
|
+
const t = String(row[i]).trim();
|
|
801
|
+
const num = t.replace(/[,%±+]/g, "").trim();
|
|
802
|
+
const f = num ? pyFloat(num) : null;
|
|
803
|
+
let val = "";
|
|
804
|
+
if (f !== null) {
|
|
805
|
+
const repr = pyFloatRepr(f); // a Python float: a whole number is written "29.0"
|
|
806
|
+
val = repr === String(f) ? f : repr;
|
|
807
|
+
}
|
|
808
|
+
out.push({ label, geography: geo.trim(), measure: measure.replace(/!!/g, " ").trim(), value: val, value_text: t });
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
return out;
|
|
812
|
+
}
|
|
813
|
+
/** The year and product of a data.census.gov table JSON, kept on every row; a file of another product or year than
|
|
814
|
+
* the spec asks is refused, so a 5-year table is never ingested as a 1-year one. */
|
|
815
|
+
function uploadVintage(resp, req, file) {
|
|
816
|
+
const m = /\/data\/(\d{4})\/([\w/]+?)(?:\?|$)/.exec(pyStr(resp.dataAPIURI || ""));
|
|
817
|
+
if (!m)
|
|
818
|
+
return {};
|
|
819
|
+
const [, year, product] = m;
|
|
820
|
+
const want = pyStr((req.params || {}).dataset || req.dataset || "");
|
|
821
|
+
const wy = /^(\d{4})\//.exec(want);
|
|
822
|
+
const wp = /acs[135]/.exec(want), gp = /acs[135]/.exec(product);
|
|
823
|
+
if ((wp && gp && wp[0] !== gp[0]) || (wy && wy[1] !== year)) {
|
|
824
|
+
throw new DataError(`${path.basename(file)} is ${resp.objectId || product + " " + year}, but request ` +
|
|
825
|
+
`${pyRepr(req.name ?? null)} asks for ${want}: upload the matching table`, "bad_request");
|
|
826
|
+
}
|
|
827
|
+
return { vintage: year, census_dataset: product, table_id: pyStr(resp.objectId || "") };
|
|
828
|
+
}
|
|
829
|
+
/** The uploaded ASEC rows with PWWGT1-160 from the uploaded replicate weight file (.dat.gz or .dat) and its .sas
|
|
830
|
+
* layout, through the same gates as the API route. */
|
|
831
|
+
function uploadAsecReps(files, rows) {
|
|
832
|
+
const dat = files.filter((p) => /\.(dat\.gz|gz|dat)$/i.test(p));
|
|
833
|
+
const sas = files.filter((p) => p.toLowerCase().endsWith(".sas"));
|
|
834
|
+
if (dat.length !== 1 || sas.length !== 1) {
|
|
835
|
+
throw new DataError("replicates: true needs the year's replicate weight file (CPS_ASEC_ASCII_REPWGT_<year>" +
|
|
836
|
+
".DAT.GZ) and its .SAS layout among the uploads, one of each", "bad_request");
|
|
837
|
+
}
|
|
838
|
+
let blob = fs.readFileSync(dat[0]);
|
|
839
|
+
if (blob[0] === 0x1f && blob[1] === 0x8b)
|
|
840
|
+
blob = zlib.gunzipSync(blob); // gzip magic
|
|
841
|
+
const idx = asecRwIndex(blob, fs.readFileSync(sas[0]).toString("latin1"));
|
|
842
|
+
const out = [];
|
|
843
|
+
let bad = 0;
|
|
844
|
+
for (const r of rows) {
|
|
845
|
+
const got = asecRwValues(blob, idx, [pyInt(pyStr(r.PH_SEQ)), pyInt(pyStr(r.PPPOS))]);
|
|
846
|
+
if (got === null || Math.abs(got[0] - (r.MARSUPWT ? pyFloat(r.MARSUPWT) ?? NaN : 0)) > 0.01) {
|
|
847
|
+
bad++;
|
|
848
|
+
continue;
|
|
849
|
+
}
|
|
850
|
+
out.push({ ...r, ...Object.fromEntries(ASEC_REPS.map((k, i) => [k, got[1][i]])) });
|
|
851
|
+
}
|
|
852
|
+
if (bad) {
|
|
853
|
+
throw new DataError(`${bad} persons are missing from the replicate weight file or have a PWWGT0 other than their ` +
|
|
854
|
+
"MARSUPWT: upload the file of the same year (2020: try the _2020BASE file)", "bad_request");
|
|
855
|
+
}
|
|
856
|
+
return out;
|
|
857
|
+
}
|
|
858
|
+
const readJson = (file) => parsePyJson(fs.readFileSync(file).toString("utf8").replace(/^\uFEFF/, ""));
|
|
859
|
+
const byCodePoint = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
860
|
+
export function parseUpload(pack, req, files) {
|
|
861
|
+
const params = req.params || {};
|
|
862
|
+
const ds = datasetOf(pack, req);
|
|
863
|
+
if (microKind(ds)) { // the manual links' files, <request>_001.json ...: each geography's calls joined
|
|
864
|
+
let plan;
|
|
865
|
+
try {
|
|
866
|
+
plan = microPlan(ds, params);
|
|
867
|
+
}
|
|
868
|
+
catch (e) {
|
|
869
|
+
if (e instanceof ValueErr)
|
|
870
|
+
throw new DataError(`request ${pyRepr(req.name ?? null)}: ${e.message}`, "bad_request");
|
|
871
|
+
throw e;
|
|
872
|
+
}
|
|
873
|
+
const js = files.filter((p) => p.toLowerCase().endsWith(".json")).sort(byCodePoint);
|
|
874
|
+
const k = plan.chunks.length;
|
|
875
|
+
let rows = [];
|
|
876
|
+
if (!js.length || js.length % k) {
|
|
877
|
+
throw new DataError(`${js.length} JSON files for request ${pyRepr(req.name ?? null)}, but each geography takes ${k} ` +
|
|
878
|
+
"calls: upload every file the instructions list", "bad_request");
|
|
879
|
+
}
|
|
880
|
+
for (let i = 0; i < js.length; i += k) {
|
|
881
|
+
const parts = js.slice(i, i + k).map((fp) => {
|
|
882
|
+
const d0 = readJson(fp);
|
|
883
|
+
const data = (truthy(d0) ? d0 : [[]]);
|
|
884
|
+
return [data[0], data.slice(1)];
|
|
885
|
+
});
|
|
886
|
+
const [cols, it] = mergeChunks(parts, plan.join);
|
|
887
|
+
for (const r of it)
|
|
888
|
+
rows.push(Object.fromEntries(cols.map((c, j) => [c, r[j]])));
|
|
889
|
+
}
|
|
890
|
+
if (plan.asec_reps)
|
|
891
|
+
rows = uploadAsecReps(files, rows);
|
|
892
|
+
return rows;
|
|
893
|
+
}
|
|
894
|
+
let rows = [];
|
|
895
|
+
for (const p of [...files].sort(byCodePoint)) {
|
|
896
|
+
if (p.toLowerCase().endsWith(".csv")) {
|
|
897
|
+
rows = rows.concat(dcgCsv(p));
|
|
898
|
+
continue;
|
|
899
|
+
}
|
|
900
|
+
const data = readJson(p);
|
|
901
|
+
if (data && typeof data === "object" && !Array.isArray(data) && Array.isArray((data.response || {}).data)) {
|
|
902
|
+
// data.census.gov's own table JSON: the API layout with every column of the table; keep the asked ones
|
|
903
|
+
const vintage = uploadVintage(data.response, req, p);
|
|
904
|
+
const want = ensureList((req.params || {}).get);
|
|
905
|
+
const keep = ["GEO_ID", "NAME", ...want.flatMap((v) => (v.endsWith("E") ? [v, v.slice(0, -1) + "M"] : [v]))];
|
|
906
|
+
const t = table(data.response.data).map((r) => ({
|
|
907
|
+
...(want.length ? Object.fromEntries(keep.filter((k) => k in r).map((k) => [k, r[k]])) : r),
|
|
908
|
+
geo_fips: pyStr(r.GEO_ID ?? "").split("US").pop(), ...vintage
|
|
909
|
+
}));
|
|
910
|
+
rows = rows.concat(normalize(req, {}, t));
|
|
911
|
+
}
|
|
912
|
+
else if (Array.isArray(data))
|
|
913
|
+
rows = rows.concat(normalize(req, {}, table(data)));
|
|
914
|
+
else if (data && typeof data === "object" && "variables" in data) {
|
|
915
|
+
rows = rows.concat(Object.keys(data.variables).sort(byCodePoint).map((k) => ({ variable: k, ...gr.flatten(data.variables[k]) })));
|
|
916
|
+
}
|
|
917
|
+
else if (data && typeof data === "object" && "groups" in data)
|
|
918
|
+
rows = rows.concat(data.groups);
|
|
919
|
+
else { // never report an unreadable file as an empty table
|
|
920
|
+
throw new DataError(`${path.basename(p)}: not a Census API response (a JSON array with a header row) nor ` +
|
|
921
|
+
"a data.census.gov table JSON or CSV", "bad_request");
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
return gr.clean(rows, CLEAN_FLAT);
|
|
925
|
+
}
|
|
926
|
+
// ---------------------------------------------------------------------- search
|
|
927
|
+
async function variables(client, pack, ds) {
|
|
928
|
+
const url = `${client.base}/${ds}/variables.json`;
|
|
929
|
+
const cache = process.env.DATA_ANALYST_FIXTURES ? null : new Cache("census", pack.policy);
|
|
930
|
+
const hit = cache ? cache.get(url) : null;
|
|
931
|
+
if (hit)
|
|
932
|
+
return parsePyJson(hit[0].toString("utf8"));
|
|
933
|
+
const r = await censusGet(client, `/${ds}/variables.json`);
|
|
934
|
+
if (cache)
|
|
935
|
+
cache.put(url, r.body, { dataset: ds });
|
|
936
|
+
return r.json();
|
|
937
|
+
}
|
|
938
|
+
/** 'median household income' searches the latest ACS 5-year tables; prefix a dataset to choose another:
|
|
939
|
+
* '2020/dec/pl: total population', 'acs1_subject: poverty'. */
|
|
940
|
+
export async function search(query, pack) {
|
|
941
|
+
const m = /^\s*([\w/]+)\s*:\s*([\s\S]+)$/.exec(query);
|
|
942
|
+
const cat = pack.endpoints?.catalog || {};
|
|
943
|
+
let [dsIn, text] = m ? [m[1], m[2]] : ["latest/acs/acs5", query];
|
|
944
|
+
dsIn = dsIn in cat ? cat[dsIn].id : dsIn;
|
|
945
|
+
const words = (text.toLowerCase().match(/[a-z0-9']+/g) || []).filter((w) => w.length > 1);
|
|
946
|
+
const hits = gr.searchCatalog(pack, text, 200).filter((h) => "alias" in h &&
|
|
947
|
+
words.every((w) => Object.values(h).map(pyStr).join(" ").toLowerCase().includes(w)));
|
|
948
|
+
const out = hits.slice(0, 5);
|
|
949
|
+
const client = clientOf(pack);
|
|
950
|
+
try {
|
|
951
|
+
const [ds] = await resolveDataset(client, dsIn);
|
|
952
|
+
const vs = (await variables(client, pack, ds)).variables || {};
|
|
953
|
+
const scored = [];
|
|
954
|
+
for (const [vid, v] of Object.entries(vs)) {
|
|
955
|
+
const label = pyStr(v.label ?? "").replace(/!!/g, " > ").replace(/^[ >]+|[ >]+$/g, "");
|
|
956
|
+
const concept = pyStr(v.concept ?? "");
|
|
957
|
+
const hay = `${vid} ${label} ${concept}`.toLowerCase();
|
|
958
|
+
if (words.length && words.every((w) => hay.includes(w))) {
|
|
959
|
+
const s = words.filter((w) => label.toLowerCase().includes(w)).length * 1000 - label.length - 0.1 * concept.length;
|
|
960
|
+
const attrs = pyStr(v.attributes ?? "");
|
|
961
|
+
scored.push([s, { variable: vid, label, concept, group: v.group ?? "",
|
|
962
|
+
moe: attrs.split(",").find((a) => a.endsWith("M") && !a.endsWith("MA")) ?? "", dataset: ds }]);
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
scored.sort((a, b) => b[0] - a[0]);
|
|
966
|
+
out.push(...scored.slice(0, Math.max(0, 20 - out.length)).map(([, h]) => h));
|
|
967
|
+
if (scored.length > 20 - hits.slice(0, 5).length) {
|
|
968
|
+
out.push({ note: `${scored.length} variables match in ${ds}: add words, or list a table with the group_variables endpoint` });
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
catch (e) {
|
|
972
|
+
if (!(e instanceof DataError))
|
|
973
|
+
throw e;
|
|
974
|
+
out.push({ note: `variable search needs api.census.gov (${e.kind}): ${e.message}` });
|
|
975
|
+
}
|
|
976
|
+
return out;
|
|
977
|
+
}
|
|
978
|
+
const adapter = {
|
|
979
|
+
fetch, checkSpec, checkError, normalize, ping, manualItems, manualItemsAsync, parseUpload,
|
|
980
|
+
search,
|
|
981
|
+
};
|
|
982
|
+
export default adapter;
|
|
983
|
+
//# sourceMappingURL=census.js.map
|