data-analyst-mcp 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +262 -3
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +710 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +21 -0
- package/dist/report/scan_secrets.js +234 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +246 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +56 -6
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +165 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,744 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* IPUMS microdata extracts (USA/ACS, CPS, NHIS, MEPS, ...): the deep pack (port of
|
|
3
|
+
* sources/ipums/fetch.py and ipums_api.py). Define, submit, wait for, download and package one extract.
|
|
4
|
+
*
|
|
5
|
+
* spec (converted to the IPUMS API v2 body):
|
|
6
|
+
* {"collection": "usa", "description": "Prime-age employment by nativity",
|
|
7
|
+
* "samples": ["us2013a", "us2018a", "us2023a"],
|
|
8
|
+
* "variables": ["AGE", "SEX", "CITIZEN", "EDUCD", "EMPSTAT", "INCWAGE", "PERWT"],
|
|
9
|
+
* "case_selections": {"AGE": {"min": 25, "max": 54}, "SEX": ["1"]},
|
|
10
|
+
* "case_select_who": "individuals", "attached_characteristics": {"EDUCD": ["spouse"]},
|
|
11
|
+
* "data_quality_flags": ["INCWAGE"], "adjust_monetary_values": ["INCWAGE"],
|
|
12
|
+
* "max_wait_s": 600} // optional: how long one fetch waits for IPUMS (then fetch again to collect)
|
|
13
|
+
* Output (data/ipums/): ipums_extract.csv (never shared: definition_only), codebook.csv, variables.csv, the DDI
|
|
14
|
+
* .xml, extract_spec.json (the definition, to reproduce), request.json and README.md.
|
|
15
|
+
* A fetch reuses data/ipums/ when it already holds this extract, and collects an extract submitted earlier (its
|
|
16
|
+
* number is saved in extract_spec.json) instead of submitting again; force: true resubmits.
|
|
17
|
+
*/
|
|
18
|
+
import { createHash } from "node:crypto";
|
|
19
|
+
import fs from "node:fs";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
import { StringDecoder } from "node:string_decoder";
|
|
22
|
+
import zlib from "node:zlib";
|
|
23
|
+
import { trail } from "../core/context.js";
|
|
24
|
+
import { Client, DataError } from "../core/http.js";
|
|
25
|
+
import * as keys from "../core/keys.js";
|
|
26
|
+
import { dataDir, fingerprint } from "../core/package.js";
|
|
27
|
+
import { pyJsonDumps, sleep, urlsplit } from "../core/py.js";
|
|
28
|
+
import { writeCsv } from "../core/tabular.js";
|
|
29
|
+
import { KEY_ENV, SOURCE, texts } from "./_ipums_bundle.js";
|
|
30
|
+
import { parseDdi } from "./_ipums_ddi.js";
|
|
31
|
+
export { DOWNLOAD_SCRIPTS, sampleLabel, webSection, writeDownloadScripts, writeManualBundle, writeReproduceBundle } from "./_ipums_bundle.js";
|
|
32
|
+
export { parseDdi } from "./_ipums_ddi.js";
|
|
33
|
+
const BASE = "https://api.ipums.org";
|
|
34
|
+
const VERSION = "2";
|
|
35
|
+
const AUTH = { env: KEY_ENV, home: "ipums", placement: "header", param: "Authorization", prefix: "", required: "required" };
|
|
36
|
+
const HINTS = {
|
|
37
|
+
401: "401 Unauthorized. The key is wrong, or the account is not registered for this collection " +
|
|
38
|
+
"(each collection, e.g. usa.ipums.org or cps.ipums.org, has its own registration and terms of use).",
|
|
39
|
+
403: "403 Forbidden. The account may not have access to this collection or sample.",
|
|
40
|
+
429: "429 Too Many Requests. Rate limited by IPUMS; wait and retry.",
|
|
41
|
+
};
|
|
42
|
+
const ALL_STATUSES = Array.from({ length: 500 }, (_, i) => i + 100);
|
|
43
|
+
export const SELECTABLE = ["usa", "cps", "ipumsi"]; // collections with case selection and monetary adjustment
|
|
44
|
+
const DEFAULT_WAIT_S = 600;
|
|
45
|
+
const pad = (n) => String(n).padStart(2, "0");
|
|
46
|
+
const today = () => {
|
|
47
|
+
const d = new Date();
|
|
48
|
+
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
|
|
49
|
+
};
|
|
50
|
+
const commas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ",");
|
|
51
|
+
const sha = (b) => createHash("sha256").update(b).digest("hex");
|
|
52
|
+
/** An IpumsError: kind auth / network / http / key_missing, and "other" (failed, canceled, expired, checksum). */
|
|
53
|
+
const ipumsError = (msg, kind = "other") => new DataError(msg, (kind === "other" ? "unexpected" : kind));
|
|
54
|
+
const isObj = (x) => !!x && typeof x === "object" && !Array.isArray(x);
|
|
55
|
+
// --------------------------------------------------------------------------- spec -> API body
|
|
56
|
+
function codes(sel) {
|
|
57
|
+
if (isObj(sel) && ("min" in sel || "max" in sel)) {
|
|
58
|
+
const lo = Math.trunc(Number(sel.min ?? 0)), hi = Math.trunc(Number(sel.max));
|
|
59
|
+
const width = Math.trunc(Number(sel.width ?? 0));
|
|
60
|
+
if (!Number.isFinite(lo) || !Number.isFinite(hi) || !Number.isFinite(width)) {
|
|
61
|
+
throw new TypeError("min and max must be whole numbers");
|
|
62
|
+
}
|
|
63
|
+
const out = [];
|
|
64
|
+
for (let v = lo; v <= hi; v++)
|
|
65
|
+
out.push(String(v).padStart(width, "0"));
|
|
66
|
+
return out;
|
|
67
|
+
}
|
|
68
|
+
if (Array.isArray(sel))
|
|
69
|
+
return sel.map(String);
|
|
70
|
+
throw new TypeError(`Bad case selection ${JSON.stringify(sel)}: use a list of codes or {'min':..,'max':..}`);
|
|
71
|
+
}
|
|
72
|
+
/** What is wrong with the spec before anything is sent (fix_spec). */
|
|
73
|
+
export function specProblems(spec) {
|
|
74
|
+
if (!isObj(spec))
|
|
75
|
+
return ["spec.json must be a JSON object"];
|
|
76
|
+
const bad = [];
|
|
77
|
+
if (typeof spec.collection !== "string" || !spec.collection)
|
|
78
|
+
bad.push('"collection" is required (usa, cps, ipumsi, atus, nhis, meps, ...)');
|
|
79
|
+
for (const k of ["samples", "variables"])
|
|
80
|
+
if (!Array.isArray(spec[k]) || !spec[k].length)
|
|
81
|
+
bad.push(`"${k}" must be a non-empty list`);
|
|
82
|
+
for (const [name, sel] of Object.entries(spec.case_selections || {})) {
|
|
83
|
+
if (isObj(sel) && "detailed" in sel)
|
|
84
|
+
continue;
|
|
85
|
+
try {
|
|
86
|
+
codes(sel);
|
|
87
|
+
}
|
|
88
|
+
catch (e) {
|
|
89
|
+
bad.push(`case_selections.${name}: ${e.message}`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
if (![undefined, null, "individuals", "households"].includes(spec.case_select_who)) {
|
|
93
|
+
bad.push('"case_select_who" must be "individuals" or "households"');
|
|
94
|
+
}
|
|
95
|
+
const c = spec.collection;
|
|
96
|
+
if (typeof c === "string" && c && !SELECTABLE.includes(c)) { // IPUMS would refuse the extract with HTTP 400
|
|
97
|
+
const only = `is available only for ${SELECTABLE.join(", ")}`;
|
|
98
|
+
if ((spec.case_selections && Object.keys(spec.case_selections).length) || spec.case_select_who) {
|
|
99
|
+
bad.push(`"case_selections" ${only}: remove it (and "case_select_who"), keep the selection variables ` +
|
|
100
|
+
`in "variables", and select the group in analysis on the whole ${c} file with a domain mask ` +
|
|
101
|
+
"(share_z / mean_z): dropping rows first loses design information for standard errors");
|
|
102
|
+
}
|
|
103
|
+
if (spec.adjust_monetary_values?.length) {
|
|
104
|
+
bad.push(`"adjust_monetary_values" ${only}: remove it; there is no CPI99 for ${c}, so deflate in ` +
|
|
105
|
+
"analysis with the BLS CPI-U annual average (cite the series) and state the dollar year");
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
const lim = typeof c === "string" ? texts().API_LIMITS[c] : null;
|
|
109
|
+
if (lim) {
|
|
110
|
+
for (const k of lim.unsupported) {
|
|
111
|
+
const v = spec[k];
|
|
112
|
+
if (v && !(Array.isArray(v) && !v.length) && !(isObj(v) && !Object.keys(v).length))
|
|
113
|
+
bad.push(`"${k}" is not supported by the IPUMS API for ${c}: remove it`);
|
|
114
|
+
}
|
|
115
|
+
const ds = spec.data_structure || {};
|
|
116
|
+
const on = isObj(ds) ? (ds.rectangular || {}).on : null;
|
|
117
|
+
if (on && !lim.rect_on.includes(on)) {
|
|
118
|
+
bad.push(`"data_structure" rectangular on '${on}' is not available for ${c} ` +
|
|
119
|
+
`(use ${lim.rect_on.map((x) => `'${x}'`).join(" or ")}, or {"hierarchical": {}})`);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (spec.max_wait_s !== undefined && !(Number(spec.max_wait_s) > 0))
|
|
123
|
+
bad.push('"max_wait_s" must be a number of seconds');
|
|
124
|
+
return bad;
|
|
125
|
+
}
|
|
126
|
+
export function buildBody(spec) {
|
|
127
|
+
const variables = {};
|
|
128
|
+
const names = spec.variables.map((v) => String(v).toUpperCase());
|
|
129
|
+
const up = (o) => Object.fromEntries(Object.entries(o || {}).map(([k, v]) => [k.toUpperCase(), v]));
|
|
130
|
+
const cs = up(spec.case_selections), ac = up(spec.attached_characteristics);
|
|
131
|
+
const dq = spec.data_quality_flags || [];
|
|
132
|
+
const adj = new Set((spec.adjust_monetary_values || []).map((v) => String(v).toUpperCase()));
|
|
133
|
+
for (const extra of [...Object.keys(cs), ...Object.keys(ac)])
|
|
134
|
+
if (!names.includes(extra))
|
|
135
|
+
names.push(extra); // must be in the extract
|
|
136
|
+
for (const name of names) {
|
|
137
|
+
const v = {};
|
|
138
|
+
if (name in cs) {
|
|
139
|
+
const sel = cs[name];
|
|
140
|
+
v.caseSelections = isObj(sel) && "detailed" in sel ? { detailed: sel.detailed.map(String) } : { general: codes(sel) };
|
|
141
|
+
}
|
|
142
|
+
if (name in ac)
|
|
143
|
+
v.attachedCharacteristics = ac[name];
|
|
144
|
+
if (dq === true || (Array.isArray(dq) && dq.map((d) => String(d).toUpperCase()).includes(name)))
|
|
145
|
+
v.dataQualityFlags = true;
|
|
146
|
+
if (adj.has(name))
|
|
147
|
+
v.adjustMonetaryValues = true;
|
|
148
|
+
variables[name] = v;
|
|
149
|
+
}
|
|
150
|
+
const body = { description: String(spec.description ?? "extract via data-analyst").slice(0, 250),
|
|
151
|
+
dataStructure: spec.data_structure || { rectangular: { on: "P" } }, dataFormat: spec.data_format ?? "csv",
|
|
152
|
+
samples: Object.fromEntries(spec.samples.map((s) => [s, {}])), variables };
|
|
153
|
+
if (spec.case_select_who)
|
|
154
|
+
body.caseSelectWho = spec.case_select_who;
|
|
155
|
+
return body;
|
|
156
|
+
}
|
|
157
|
+
/** Rough persons per sample, for a size warning before submitting. */
|
|
158
|
+
export function approxRows(samples) {
|
|
159
|
+
const R = texts().APPROX_ROWS;
|
|
160
|
+
let n = 0;
|
|
161
|
+
for (const s of samples) {
|
|
162
|
+
if (s.startsWith("us"))
|
|
163
|
+
n += s.endsWith("c") ? R.usa_acs5 : R.usa_acs1;
|
|
164
|
+
else if (s.startsWith("cps") && s.endsWith("_03s"))
|
|
165
|
+
n += R.cps_asec;
|
|
166
|
+
else if (s.startsWith("cps"))
|
|
167
|
+
n += R.cps_monthly;
|
|
168
|
+
else if (/^ih\d{4}$/.test(s))
|
|
169
|
+
n += Number(s.slice(2)) >= 2019 ? R.nhis : R.nhis_pre2019;
|
|
170
|
+
else if (/^mp\d{4}$/.test(s))
|
|
171
|
+
n += R.meps;
|
|
172
|
+
else
|
|
173
|
+
n += 500_000;
|
|
174
|
+
}
|
|
175
|
+
return n;
|
|
176
|
+
}
|
|
177
|
+
/** Hash of the API request body: two specs that would produce the same extract hash the same. */
|
|
178
|
+
export const specFingerprint = (body) => sha(pyJsonDumps(body, true)).slice(0, 16);
|
|
179
|
+
// --------------------------------------------------------------------------- API client
|
|
180
|
+
export class IpumsClient {
|
|
181
|
+
http;
|
|
182
|
+
key;
|
|
183
|
+
constructor(apiKey, timeout = 120) {
|
|
184
|
+
const [k] = apiKey ? [apiKey] : keys.findAuthKey(AUTH);
|
|
185
|
+
if (!k) {
|
|
186
|
+
throw ipumsError(`No IPUMS API key found. Looked in: ${keys.keyLocations(KEY_ENV, "ipums").join("; ")}. Get a key at ` +
|
|
187
|
+
"https://account.ipums.org/api_keys, then store it with the setup command; it is read at once (no restart needed).", "key_missing");
|
|
188
|
+
}
|
|
189
|
+
keys.remember(k);
|
|
190
|
+
this.key = k;
|
|
191
|
+
this.http = new Client(BASE, { name: "IPUMS", auth: AUTH, key: k, rps: 1.5, burst: 5, retries: 4, timeout,
|
|
192
|
+
deadline: 900, headers: { "Content-Type": "application/json" } });
|
|
193
|
+
}
|
|
194
|
+
toString() {
|
|
195
|
+
return "IpumsClient(key=[REDACTED])";
|
|
196
|
+
}
|
|
197
|
+
toJSON() {
|
|
198
|
+
return this.toString();
|
|
199
|
+
}
|
|
200
|
+
/** One cheap metadata call: {reachable, auth_ok, kind, detail}; never throws. */
|
|
201
|
+
async ping(collection = "usa") {
|
|
202
|
+
try {
|
|
203
|
+
await this.request("GET", "/metadata/samples", { collection, pageSize: 1 }, undefined, 1);
|
|
204
|
+
return { reachable: true, auth_ok: true, kind: "ok", detail: "" };
|
|
205
|
+
}
|
|
206
|
+
catch (e) {
|
|
207
|
+
const reachable = e.kind !== "network";
|
|
208
|
+
return { reachable, auth_ok: reachable ? false : null, kind: e.kind === "unexpected" ? "other" : e.kind,
|
|
209
|
+
detail: String(e.message).split("\n")[0].slice(0, 300) };
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
async request(method, p, params = {}, body, retries = 4) {
|
|
213
|
+
const q = { ...params };
|
|
214
|
+
if (!("version" in q))
|
|
215
|
+
q.version = VERSION;
|
|
216
|
+
this.http.retries = retries;
|
|
217
|
+
let r;
|
|
218
|
+
try {
|
|
219
|
+
r = await this.http.request(method, p, q, { jsonBody: body, okStatuses: ALL_STATUSES });
|
|
220
|
+
}
|
|
221
|
+
catch (e) {
|
|
222
|
+
if (!(e instanceof DataError))
|
|
223
|
+
throw e;
|
|
224
|
+
const kind = e.kind === "network" ? "network" : e.kind === "auth" || e.kind === "key_missing" ? "auth" : "http";
|
|
225
|
+
throw ipumsError(`${method} ${p} ${kind === "network" ? "network error" : "failed"}: ${e.message}`, kind);
|
|
226
|
+
}
|
|
227
|
+
if (r.status >= 200 && r.status < 300)
|
|
228
|
+
return r.body.toString("utf8").trim() ? r.json() : {};
|
|
229
|
+
const detail = r.body.subarray(0, 2000).toString("utf8");
|
|
230
|
+
throw ipumsError(`${method} ${p} failed: HTTP ${r.status}. ${HINTS[r.status] ?? ""}\n${detail}`, r.status === 401 || r.status === 403 ? "auth" : "http");
|
|
231
|
+
}
|
|
232
|
+
/** [{name: "us2023a", description: "2023 ACS"}, ...]. */
|
|
233
|
+
async listSamples(collection) {
|
|
234
|
+
const out = [];
|
|
235
|
+
for (let page = 1;; page++) {
|
|
236
|
+
const r = await this.request("GET", "/metadata/samples", { collection, pageNumber: page, pageSize: 500 });
|
|
237
|
+
out.push(...(r.data || []));
|
|
238
|
+
if (!r.links?.nextPage || !r.data?.length)
|
|
239
|
+
return out;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
async submit(collection, definition) {
|
|
243
|
+
const r = await this.request("POST", "/extracts", { collection }, definition, 1);
|
|
244
|
+
return Math.trunc(Number(r.number));
|
|
245
|
+
}
|
|
246
|
+
info(collection, number) {
|
|
247
|
+
return this.request("GET", `/extracts/${number}`, { collection });
|
|
248
|
+
}
|
|
249
|
+
/** Poll until completed (10 s, then up to 60 s apart). Throws on failed, canceled, expired or timeout. */
|
|
250
|
+
async wait(collection, number, maxWaitS) {
|
|
251
|
+
const start = Date.now();
|
|
252
|
+
let delay = 10;
|
|
253
|
+
for (;;) {
|
|
254
|
+
const info = await this.info(collection, number);
|
|
255
|
+
const status = info.status;
|
|
256
|
+
if (status === "completed") {
|
|
257
|
+
if (info.downloadLinks && Object.keys(info.downloadLinks).length)
|
|
258
|
+
return info;
|
|
259
|
+
// an expired extract stays "completed" but loses its links: waiting will not bring them back
|
|
260
|
+
throw ipumsError(`Extract ${collection}#${number} is complete but its files have expired (IPUMS deletes them about ` +
|
|
261
|
+
"72 hours after they are built). Submit it again: call fetch with the same spec and force: true.");
|
|
262
|
+
}
|
|
263
|
+
if (status === "failed" || status === "canceled") {
|
|
264
|
+
const errs = info.errors || info.errorMessages || "";
|
|
265
|
+
throw ipumsError(`Extract ${collection}#${number} ${status}. ${typeof errs === "string" ? errs : JSON.stringify(errs)}`);
|
|
266
|
+
}
|
|
267
|
+
if ((Date.now() - start) / 1000 > maxWaitS) {
|
|
268
|
+
throw new DataError(`Timed out after ${maxWaitS}s waiting for extract ${collection}#${number} (last status ${status}). ` +
|
|
269
|
+
"IPUMS is still building it: call fetch again with the same spec later; it collects this extract (its number is " +
|
|
270
|
+
"saved in data/ipums/extract_spec.json) instead of submitting a new one.", "http");
|
|
271
|
+
}
|
|
272
|
+
await sleep(delay * 1000);
|
|
273
|
+
delay = Math.min(60, delay + 10);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
/** One extract file to dest (atomic), checking the SHA-256 IPUMS reports. Streamed when live. */
|
|
277
|
+
async download(url, dest, sha256) {
|
|
278
|
+
const tmp = dest + ".part";
|
|
279
|
+
let digest;
|
|
280
|
+
if (process.env.DATA_ANALYST_FIXTURES || keys.restricted()) {
|
|
281
|
+
let r;
|
|
282
|
+
try {
|
|
283
|
+
r = await this.http.request("GET", url, null, { okStatuses: [200] });
|
|
284
|
+
}
|
|
285
|
+
catch (e) {
|
|
286
|
+
const kind = e?.kind === "network" ? "network" : e?.kind === "auth" || e?.kind === "key_missing" ? "auth" : "http";
|
|
287
|
+
throw ipumsError(`Download failed for ${url}: ${e?.message || e}`, kind);
|
|
288
|
+
}
|
|
289
|
+
fs.writeFileSync(tmp, r.body);
|
|
290
|
+
digest = sha(r.body);
|
|
291
|
+
}
|
|
292
|
+
else
|
|
293
|
+
digest = await this.stream(url, tmp);
|
|
294
|
+
if (sha256 && digest !== sha256) {
|
|
295
|
+
fs.rmSync(tmp, { force: true });
|
|
296
|
+
throw ipumsError(`Checksum mismatch for ${url}`);
|
|
297
|
+
}
|
|
298
|
+
fs.renameSync(tmp, dest);
|
|
299
|
+
return dest;
|
|
300
|
+
}
|
|
301
|
+
/** Stream a download to disk; the key goes only to the IPUMS host (never past a redirect to another host). */
|
|
302
|
+
async stream(url, tmp) {
|
|
303
|
+
const host = urlsplit(url).netloc.toLowerCase();
|
|
304
|
+
let cur = url;
|
|
305
|
+
let resp = null;
|
|
306
|
+
try {
|
|
307
|
+
for (let hop = 0; hop < 10; hop++) {
|
|
308
|
+
const same = urlsplit(cur).netloc.toLowerCase() === host && cur.startsWith("https://");
|
|
309
|
+
resp = await globalThis.fetch(cur, { headers: { "User-Agent": this.http.ua, ...(same ? { Authorization: this.key } : {}) },
|
|
310
|
+
redirect: "manual", signal: AbortSignal.timeout(this.http.timeout * 1000 * 30) });
|
|
311
|
+
const loc = resp.headers.get("location");
|
|
312
|
+
if (![301, 302, 303, 307, 308].includes(resp.status) || !loc)
|
|
313
|
+
break;
|
|
314
|
+
await resp.arrayBuffer().catch(() => undefined);
|
|
315
|
+
cur = new URL(loc, cur).toString();
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
catch (e) {
|
|
319
|
+
throw ipumsError(`Download failed for ${keys.redact(url)}: ${keys.redact(e?.cause?.code || e?.message || e)}`, "network");
|
|
320
|
+
}
|
|
321
|
+
if (!resp || resp.status !== 200 || !resp.body) {
|
|
322
|
+
throw ipumsError(`Download failed for ${keys.redact(url)}: HTTP ${resp?.status}`, resp && (resp.status === 401 || resp.status === 403) ? "auth" : "http");
|
|
323
|
+
}
|
|
324
|
+
const h = createHash("sha256");
|
|
325
|
+
const fd = fs.openSync(tmp, "w");
|
|
326
|
+
let n = 0;
|
|
327
|
+
try {
|
|
328
|
+
for await (const chunk of resp.body) {
|
|
329
|
+
h.update(chunk);
|
|
330
|
+
fs.writeSync(fd, chunk);
|
|
331
|
+
n += chunk.length;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
finally {
|
|
335
|
+
fs.closeSync(fd);
|
|
336
|
+
}
|
|
337
|
+
const digest = h.digest("hex");
|
|
338
|
+
trail().push({ method: "GET", url: keys.redact(url), status: 200, bytes: n, sha256: digest, cached: false,
|
|
339
|
+
fetched_at: new Date().toISOString().replace(/\.\d{3}Z$/, "+00:00") });
|
|
340
|
+
return digest;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
/** Key check for setup: one cheap authenticated metadata call. [true, detail] when IPUMS accepts the key; a
|
|
344
|
+
* rejected key throws a DataError of kind "auth", an unreachable host kind "network". */
|
|
345
|
+
export async function verify(key) {
|
|
346
|
+
const c = new Client(BASE, { name: "IPUMS", auth: AUTH, key, retries: 1, timeout: 30, deadline: 60 });
|
|
347
|
+
const r = await c.get("/metadata/samples", { collection: "usa", version: "2", pageSize: 1 });
|
|
348
|
+
const data = r.body.toString("utf8").trim() ? r.json() : {};
|
|
349
|
+
if (isObj(data) && data.data !== undefined && data.data !== null) {
|
|
350
|
+
return [true, "IPUMS accepted the key (metadata call answered). Register for each collection you use (usa, cps, ...)."];
|
|
351
|
+
}
|
|
352
|
+
return [null, "IPUMS answered, but not with the expected metadata; the key could not be confirmed."];
|
|
353
|
+
}
|
|
354
|
+
/** Lines of a file (plain or gzip, by content) as raw text, each with its line end. */
|
|
355
|
+
async function* fileLines(p) {
|
|
356
|
+
const fd = fs.openSync(p, "r");
|
|
357
|
+
const magic = Buffer.alloc(2);
|
|
358
|
+
fs.readSync(fd, magic, 0, 2, 0);
|
|
359
|
+
fs.closeSync(fd);
|
|
360
|
+
const s = fs.createReadStream(p, { highWaterMark: 1 << 20 });
|
|
361
|
+
const src = magic[0] === 0x1f && magic[1] === 0x8b ? s.pipe(zlib.createGunzip()) : s;
|
|
362
|
+
const dec = new StringDecoder("utf8");
|
|
363
|
+
let rest = "";
|
|
364
|
+
for await (const c of src) {
|
|
365
|
+
const parts = (rest + dec.write(c)).split(/(?<=\n)/);
|
|
366
|
+
rest = parts.pop() || "";
|
|
367
|
+
yield* parts;
|
|
368
|
+
}
|
|
369
|
+
rest += dec.end();
|
|
370
|
+
if (rest)
|
|
371
|
+
yield rest;
|
|
372
|
+
}
|
|
373
|
+
function csvHeaderCells(line) {
|
|
374
|
+
const out = [];
|
|
375
|
+
let cur = "", q = false;
|
|
376
|
+
const s = line.replace(/\r?\n$/, "");
|
|
377
|
+
for (let i = 0; i < s.length; i++) {
|
|
378
|
+
const c = s[i];
|
|
379
|
+
if (q) {
|
|
380
|
+
if (c === '"' && s[i + 1] === '"')
|
|
381
|
+
(cur += '"', i++);
|
|
382
|
+
else if (c === '"')
|
|
383
|
+
q = false;
|
|
384
|
+
else
|
|
385
|
+
cur += c;
|
|
386
|
+
}
|
|
387
|
+
else if (c === '"')
|
|
388
|
+
q = true;
|
|
389
|
+
else if (c === ",")
|
|
390
|
+
(out.push(cur), (cur = ""));
|
|
391
|
+
else
|
|
392
|
+
cur += c;
|
|
393
|
+
}
|
|
394
|
+
out.push(cur);
|
|
395
|
+
return out;
|
|
396
|
+
}
|
|
397
|
+
function writeRequestJson(d, spec, body, number, route, acquired, files, problems = []) {
|
|
398
|
+
const retrieved = new Date().toISOString().replace(/\.\d{3}Z$/, "+00:00");
|
|
399
|
+
const req = { source: SOURCE, spec, fingerprint: fingerprint(spec), route, acquired, retrieved, vintage: "",
|
|
400
|
+
extract: `${spec.collection ?? ""}:${number}`,
|
|
401
|
+
requests: trail().filter((s) => s.url.includes("/extracts") || s.url.includes("/downloads/")),
|
|
402
|
+
tables: files, problems, notes: [texts().TERMS_LINE], policy: { redistribution: "definition_only", cache: "run_only" },
|
|
403
|
+
api_body_sha256: sha(pyJsonDumps(body, true)) };
|
|
404
|
+
fs.writeFileSync(path.join(d, "request.json"), JSON.stringify(req, null, 2));
|
|
405
|
+
return retrieved;
|
|
406
|
+
}
|
|
407
|
+
/** Turn a data file (.csv or .csv.gz) and its DDI into the data/ipums/ package. */
|
|
408
|
+
export async function packageFiles(c, collection, number, dataPath, ddiPath, spec, body, o = {}) {
|
|
409
|
+
const t = texts();
|
|
410
|
+
const route = o.route ?? "api", acquired = o.acquired ?? "IPUMS API download";
|
|
411
|
+
const d = dataDir(c.out, SOURCE);
|
|
412
|
+
fs.mkdirSync(d, { recursive: true });
|
|
413
|
+
// Decompress the data to a plain CSV (what most tools expect) and count rows.
|
|
414
|
+
const csvPath = path.join(d, "ipums_extract.csv");
|
|
415
|
+
const tmpPath = csvPath + ".part";
|
|
416
|
+
let nrows = 0;
|
|
417
|
+
let columns = [];
|
|
418
|
+
const fd = fs.openSync(tmpPath, "w");
|
|
419
|
+
try {
|
|
420
|
+
let buf = [];
|
|
421
|
+
for await (const line of fileLines(dataPath)) {
|
|
422
|
+
if (!columns.length && nrows === 0 && !buf.length && columns.length === 0 && line !== undefined && !columns.done) {
|
|
423
|
+
columns = csvHeaderCells(line).map((x) => x.trim().toUpperCase());
|
|
424
|
+
columns.done = true;
|
|
425
|
+
fs.writeSync(fd, columns.join(",") + "\n");
|
|
426
|
+
continue;
|
|
427
|
+
}
|
|
428
|
+
buf.push(line);
|
|
429
|
+
nrows++;
|
|
430
|
+
if (buf.length >= 10000) {
|
|
431
|
+
fs.writeSync(fd, buf.join(""));
|
|
432
|
+
buf = [];
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
if (buf.length)
|
|
436
|
+
fs.writeSync(fd, buf.join(""));
|
|
437
|
+
}
|
|
438
|
+
finally {
|
|
439
|
+
fs.closeSync(fd);
|
|
440
|
+
}
|
|
441
|
+
fs.renameSync(tmpPath, csvPath);
|
|
442
|
+
if (o.removeSource && path.resolve(dataPath) !== path.resolve(csvPath))
|
|
443
|
+
fs.rmSync(dataPath, { force: true });
|
|
444
|
+
if (ddiPath && path.resolve(path.dirname(ddiPath)) !== path.resolve(d))
|
|
445
|
+
fs.copyFileSync(ddiPath, path.join(d, path.basename(ddiPath)));
|
|
446
|
+
if (ddiPath) { // an earlier extract's DDI no longer describes ipums_extract.csv
|
|
447
|
+
const keep = path.basename(ddiPath);
|
|
448
|
+
for (const old of fs.readdirSync(d)) {
|
|
449
|
+
if (old.toLowerCase().endsWith(".xml") && old !== keep)
|
|
450
|
+
fs.renameSync(path.join(d, old), path.join(d, old + ".superseded"));
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
const [varRows, codeRows, ddiCite] = ddiPath ? parseDdi(ddiPath) : [[], [], ""];
|
|
454
|
+
fs.writeFileSync(path.join(d, "variables.csv"), writeCsv(varRows, ["variable", "label", "type", "start", "width", "decimals"]));
|
|
455
|
+
fs.writeFileSync(path.join(d, "codebook.csv"), writeCsv(codeRows, ["variable", "variable_label", "value", "value_label"]));
|
|
456
|
+
let sampleDesc = {};
|
|
457
|
+
try {
|
|
458
|
+
if (o.client)
|
|
459
|
+
sampleDesc = Object.fromEntries((await o.client.listSamples(collection)).map((s) => [s.name, s.description]));
|
|
460
|
+
}
|
|
461
|
+
catch {
|
|
462
|
+
sampleDesc = {};
|
|
463
|
+
}
|
|
464
|
+
fs.writeFileSync(path.join(d, "extract_spec.json"), JSON.stringify({ collection, extract_number: number, downloaded: today(),
|
|
465
|
+
acquired, rows: nrows, spec, api_body: body }, null, 2));
|
|
466
|
+
const labels = Object.fromEntries(varRows.map((r) => [r.variable, r.label]));
|
|
467
|
+
const hints = t.WEIGHT_HINTS[collection] ??
|
|
468
|
+
Object.assign({}, ...Object.values(t.WEIGHT_HINTS));
|
|
469
|
+
const weights = columns.filter((x) => x in hints);
|
|
470
|
+
const citation = ddiCite || t.CITATIONS[collection] || "Cite IPUMS per https://www.ipums.org/about/citation";
|
|
471
|
+
const unit = JSON.stringify(body.dataStructure || {}).includes("householdOnly") ? "households" : "persons";
|
|
472
|
+
const caseLines = Object.entries(body.variables).filter(([, v]) => "caseSelections" in v)
|
|
473
|
+
.map(([k, v]) => `- \`${k}\` in ${pyDictRepr(v.caseSelections)}`);
|
|
474
|
+
const lines = [
|
|
475
|
+
`# IPUMS extract ${collection.toUpperCase()} #${number}`, "",
|
|
476
|
+
`- Description: ${body.description ?? ""}`,
|
|
477
|
+
`- Downloaded: ${today()} (${acquired})`,
|
|
478
|
+
`- Rows (${unit}): ${commas(nrows)}`,
|
|
479
|
+
"- File: `ipums_extract.csv` (comma-separated, header = IPUMS mnemonics, raw IPUMS codes)",
|
|
480
|
+
"- Value labels: `codebook.csv`; variable labels/types: `variables.csv`; full DDI: the `.xml`", "",
|
|
481
|
+
"## Samples", ...Object.keys(body.samples).map((s) => `- \`${s}\`: ${sampleDesc[s] ?? ""}`), "",
|
|
482
|
+
"## Variables", ...columns.map((x) => `- \`${x}\`: ${labels[x] ?? ""}`), "",
|
|
483
|
+
"## Case selections", ...(caseLines.length ? caseLines : ["- none (full sample)"]), "",
|
|
484
|
+
"## Weights", ...(weights.length ? weights.map((w) => `- \`${w}\`: ${hints[w]}`)
|
|
485
|
+
: ["- No weight variable in extract. Estimates will NOT be population-representative."]), "",
|
|
486
|
+
"## Reading the data", "```python", "import pandas as pd", "df = pd.read_csv('ipums_extract.csv')",
|
|
487
|
+
"cb = pd.read_csv('codebook.csv', dtype={'value': str})", "```",
|
|
488
|
+
`Watch for IPUMS missing/top codes (${t.MISSING_HINT[collection] ?? t.MISSING_HINT.usa}). Check \`codebook.csv\` before computing means.`,
|
|
489
|
+
"", "## Citation", citation, "",
|
|
490
|
+
`Suggested source line: "Analyst calculations using data from ${t.SOURCE_LINE[collection] ?? "IPUMS"}." ` +
|
|
491
|
+
'(or "<Organization>, using data from ..." when the product names an organization)', "",
|
|
492
|
+
"## Use and redistribution", "", `Rule: **definition_only**. ${t.TERMS_LINE}`
|
|
493
|
+
];
|
|
494
|
+
fs.writeFileSync(path.join(d, "README.md"), lines.join("\n") + "\n");
|
|
495
|
+
const tables = [{ name: "ipums_extract", file: "ipums_extract.csv", rows: nrows, columns, sha256: fileSha(csvPath) },
|
|
496
|
+
{ name: "codebook", file: "codebook.csv", rows: codeRows.length, columns: ["variable", "variable_label", "value", "value_label"] },
|
|
497
|
+
{ name: "variables", file: "variables.csv", rows: varRows.length, columns: ["variable", "label", "type", "start", "width", "decimals"] }];
|
|
498
|
+
const retrieved = writeRequestJson(d, spec, body, number, route, acquired, tables.map((x) => (x.name === "ipums_extract" ? x : { name: x.name, file: x.file, rows: x.rows })));
|
|
499
|
+
return { data_dir: d, tables: tables.map(({ name, file, rows, columns: cols }) => ({ name, file, rows, columns: cols })),
|
|
500
|
+
notes: [...(o.notes || []), `Extract ${collection}:${number}. ${t.TERMS_LINE}`], problems: [], retrieved, citation,
|
|
501
|
+
vintage: "", route, files: [path.join(d, "extract_spec.json"), ...(ddiPath ? [path.join(d, path.basename(ddiPath))] : [])] };
|
|
502
|
+
}
|
|
503
|
+
/** str() of a small dict of lists, as the README shows case selections ({'general': ['25', ...]}). */
|
|
504
|
+
function pyDictRepr(v) {
|
|
505
|
+
const r = (x) => Array.isArray(x) ? "[" + x.map(r).join(", ") + "]"
|
|
506
|
+
: isObj(x) ? "{" + Object.entries(x).map(([k, y]) => `'${k}': ${r(y)}`).join(", ") + "}"
|
|
507
|
+
: typeof x === "string" ? `'${x}'` : x === true ? "True" : x === false ? "False" : String(x);
|
|
508
|
+
return r(v);
|
|
509
|
+
}
|
|
510
|
+
function fileSha(p) {
|
|
511
|
+
const h = createHash("sha256");
|
|
512
|
+
const fd = fs.openSync(p, "r");
|
|
513
|
+
const b = Buffer.alloc(1 << 20);
|
|
514
|
+
try {
|
|
515
|
+
for (let n; (n = fs.readSync(fd, b, 0, b.length, null));)
|
|
516
|
+
h.update(b.subarray(0, n));
|
|
517
|
+
}
|
|
518
|
+
finally {
|
|
519
|
+
fs.closeSync(fd);
|
|
520
|
+
}
|
|
521
|
+
return h.digest("hex");
|
|
522
|
+
}
|
|
523
|
+
async function packageInfo(c, client, collection, info, spec, body) {
|
|
524
|
+
const d = dataDir(c.out, SOURCE);
|
|
525
|
+
fs.mkdirSync(d, { recursive: true });
|
|
526
|
+
const links = info.downloadLinks;
|
|
527
|
+
const get = async (k) => {
|
|
528
|
+
const link = links[k];
|
|
529
|
+
if (!link)
|
|
530
|
+
return null;
|
|
531
|
+
const dest = path.join(d, String(link.url).split("/").pop());
|
|
532
|
+
return client.download(link.url, dest, link.sha256);
|
|
533
|
+
};
|
|
534
|
+
const dataPath = await get("data");
|
|
535
|
+
const ddiPath = await get("ddiCodebook");
|
|
536
|
+
await get("codebookPreview");
|
|
537
|
+
if (!dataPath)
|
|
538
|
+
throw ipumsError(`Extract ${collection}#${info.number} has no data file link`);
|
|
539
|
+
return packageFiles(c, collection, info.number, dataPath, ddiPath, spec, body, { client, removeSource: true, route: "api" });
|
|
540
|
+
}
|
|
541
|
+
/** What data/ipums/ already holds for this body, or null when there is nothing. */
|
|
542
|
+
function existingData(out, body) {
|
|
543
|
+
const d = dataDir(out, SOURCE);
|
|
544
|
+
const specFile = path.join(d, "extract_spec.json"), csvPath = path.join(d, "ipums_extract.csv");
|
|
545
|
+
if (!fs.existsSync(specFile))
|
|
546
|
+
return null;
|
|
547
|
+
let saved;
|
|
548
|
+
try {
|
|
549
|
+
saved = JSON.parse(fs.readFileSync(specFile, "utf8"));
|
|
550
|
+
}
|
|
551
|
+
catch {
|
|
552
|
+
return null;
|
|
553
|
+
}
|
|
554
|
+
const savedBody = saved.api_body || {};
|
|
555
|
+
return { path: csvPath, csv_exists: fs.existsSync(csvPath), extract_number: saved.extract_number, status: saved.status ?? "downloaded",
|
|
556
|
+
downloaded: saved.downloaded, rows: saved.rows,
|
|
557
|
+
spec_matches: !!body && Object.keys(savedBody).length > 0 && specFingerprint(savedBody) === specFingerprint(body) };
|
|
558
|
+
}
|
|
559
|
+
function reuse(c, spec, ex) {
|
|
560
|
+
const d = dataDir(c.out, SOURCE);
|
|
561
|
+
let req = {};
|
|
562
|
+
try {
|
|
563
|
+
req = JSON.parse(fs.readFileSync(path.join(d, "request.json"), "utf8"));
|
|
564
|
+
}
|
|
565
|
+
catch {
|
|
566
|
+
/* optional */
|
|
567
|
+
}
|
|
568
|
+
const tables = (req.tables || []).filter((x) => fs.existsSync(path.join(d, x.file))).map((x) => ({ name: x.name,
|
|
569
|
+
file: x.file, rows: x.rows ?? 0, columns: x.columns ?? (x.name === "codebook" ? ["variable", "variable_label", "value", "value_label"]
|
|
570
|
+
: x.name === "variables" ? ["variable", "label", "type", "start", "width", "decimals"] : []) }));
|
|
571
|
+
const readme = fs.existsSync(path.join(d, "README.md")) ? fs.readFileSync(path.join(d, "README.md"), "utf8") : "";
|
|
572
|
+
const m = /## Citation\n([^\n]+)/.exec(readme);
|
|
573
|
+
return { data_dir: d, tables, problems: [], retrieved: req.retrieved || "", vintage: "", route: "reuse_existing",
|
|
574
|
+
citation: m ? m[1] : texts().CITATIONS[spec.collection] ?? "",
|
|
575
|
+
notes: [`The data for this extract are already in ${ex.path} (IPUMS extract ${spec.collection}#${ex.extract_number}, ` +
|
|
576
|
+
`downloaded ${ex.downloaded}). Use force: true to resubmit.`, texts().TERMS_LINE],
|
|
577
|
+
files: [path.join(d, "extract_spec.json"), ...fs.readdirSync(d).filter((f) => f.toLowerCase().endsWith(".xml")).map((f) => path.join(d, f))] };
|
|
578
|
+
}
|
|
579
|
+
const BLOCKING = ["key_missing", "auth", "network"];
|
|
580
|
+
async function fetch(ctx) {
|
|
581
|
+
const spec = ctx.spec;
|
|
582
|
+
const probs = specProblems(spec);
|
|
583
|
+
if (probs.length)
|
|
584
|
+
throw new DataError("The request has problems: " + probs.join("; "), "spec");
|
|
585
|
+
const c = { pack: ctx.pack, out: ctx.workspace };
|
|
586
|
+
const collection = spec.collection;
|
|
587
|
+
const body = buildBody(spec);
|
|
588
|
+
const maxWait = Number(spec.max_wait_s) || DEFAULT_WAIT_S;
|
|
589
|
+
// Cheapest routes first: the data are already here, or the extract is already queued at IPUMS.
|
|
590
|
+
const ex = existingData(c.out, body);
|
|
591
|
+
if (ex && ex.csv_exists && ex.spec_matches && !ctx.force)
|
|
592
|
+
return reuse(c, spec, ex);
|
|
593
|
+
const client = new IpumsClient();
|
|
594
|
+
if (ex && ex.spec_matches && ex.extract_number && !ex.csv_exists && !ctx.force) {
|
|
595
|
+
const number = Math.trunc(Number(ex.extract_number));
|
|
596
|
+
try {
|
|
597
|
+
const info = await client.wait(collection, number, maxWait);
|
|
598
|
+
return await packageInfo(c, client, collection, info, spec, body);
|
|
599
|
+
}
|
|
600
|
+
catch (e) {
|
|
601
|
+
if (!(e instanceof DataError) || BLOCKING.includes(e.kind) || e.message.includes("Timed out") || e.message.includes("Checksum mismatch"))
|
|
602
|
+
throw e;
|
|
603
|
+
// cannot resume (expired, failed): submit again below
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
const number = await client.submit(collection, body);
|
|
607
|
+
const d = dataDir(c.out, SOURCE);
|
|
608
|
+
fs.mkdirSync(d, { recursive: true });
|
|
609
|
+
fs.writeFileSync(path.join(d, "extract_spec.json"), JSON.stringify({ collection, extract_number: number, spec, api_body: body,
|
|
610
|
+
status: "submitted" }, null, 2));
|
|
611
|
+
const info = await client.wait(collection, number, maxWait);
|
|
612
|
+
return packageInfo(c, client, collection, info, spec, body);
|
|
613
|
+
}
|
|
614
|
+
// --------------------------------------------------------------------------- ingest a hand-downloaded extract
|
|
615
|
+
/** The data file and the DDI codebook among uploaded files and folders. */
|
|
616
|
+
export function findUploads(paths) {
|
|
617
|
+
const files = [];
|
|
618
|
+
const walk = (p) => {
|
|
619
|
+
if (fs.statSync(p).isDirectory())
|
|
620
|
+
for (const n of fs.readdirSync(p))
|
|
621
|
+
walk(path.join(p, n));
|
|
622
|
+
else
|
|
623
|
+
files.push(p);
|
|
624
|
+
};
|
|
625
|
+
for (const p of paths) {
|
|
626
|
+
if (!fs.existsSync(p))
|
|
627
|
+
throw new DataError(`Not found: ${p}`, "not_found");
|
|
628
|
+
walk(p);
|
|
629
|
+
}
|
|
630
|
+
const skip = new Set(["codebook.csv", "variables.csv"]);
|
|
631
|
+
let data = files.filter((f) => /\.csv(\.gz)?$/i.test(f) && !skip.has(path.basename(f).toLowerCase()));
|
|
632
|
+
const named = data.filter((f) => /^[a-z]+_\d+\.csv(\.gz)?$/i.test(path.basename(f)));
|
|
633
|
+
// ipums_extract.csv is this pack's own output name: use it only when nothing else was uploaded
|
|
634
|
+
const other = data.filter((f) => path.basename(f).toLowerCase() !== "ipums_extract.csv");
|
|
635
|
+
data = named.length ? named : other.length ? other : data;
|
|
636
|
+
if (data.length !== 1) {
|
|
637
|
+
throw new DataError("Expected exactly one IPUMS data file (.csv or .csv.gz); found " + (data.length ? data.join(", ") : "none") +
|
|
638
|
+
". Pass the data file and the .xml codebook to ingest.", "not_found");
|
|
639
|
+
}
|
|
640
|
+
const stem = path.basename(data[0]).replace(/\.csv(\.gz)?$/i, "").toLowerCase();
|
|
641
|
+
const xml = files.filter((f) => f.toLowerCase().endsWith(".xml"));
|
|
642
|
+
const same = xml.filter((f) => path.basename(f, path.extname(f)).toLowerCase() === stem);
|
|
643
|
+
const pick = same.length ? same : xml;
|
|
644
|
+
if (pick.length > 1)
|
|
645
|
+
throw new DataError("More than one .xml file found; pass only the IPUMS codebook: " + pick.join(", "), "spec");
|
|
646
|
+
return [data[0], pick[0] ?? null];
|
|
647
|
+
}
|
|
648
|
+
async function ingest(ctx) {
|
|
649
|
+
const spec = ctx.spec;
|
|
650
|
+
const probs = specProblems(spec);
|
|
651
|
+
if (probs.length)
|
|
652
|
+
throw new DataError("The request has problems: " + probs.join("; "), "spec");
|
|
653
|
+
const collection = spec.collection;
|
|
654
|
+
const body = buildBody(spec);
|
|
655
|
+
if (!ctx.files.length)
|
|
656
|
+
throw new DataError("ingest needs the data file and the .xml codebook (or their folder)", "spec");
|
|
657
|
+
const [dataPath, ddiPath] = findUploads(ctx.files);
|
|
658
|
+
const m = /_(\d+)\.csv/i.exec(path.basename(dataPath));
|
|
659
|
+
const number = m ? Number(m[1]) : "manual";
|
|
660
|
+
const notes = [];
|
|
661
|
+
if (m && !path.basename(dataPath).toLowerCase().startsWith(collection.toLowerCase() + "_")) {
|
|
662
|
+
notes.push(`${path.basename(dataPath)} does not look like a ${collection} extract`);
|
|
663
|
+
}
|
|
664
|
+
if (!ddiPath) {
|
|
665
|
+
notes.push("no .xml codebook found; codebook.csv and variables.csv are empty. Ask the user to upload the DDI file too.");
|
|
666
|
+
}
|
|
667
|
+
const r = await packageFiles({ pack: ctx.pack, out: ctx.workspace }, collection, number, dataPath, ddiPath, spec, body, { acquired: "downloaded by hand by the user (IPUMS website, download script or curl) and uploaded", route: "manual_download", notes });
|
|
668
|
+
const cols = r.tables[0].columns;
|
|
669
|
+
const missing = Object.keys(body.variables).filter((v) => !cols.includes(v));
|
|
670
|
+
if (missing.length) {
|
|
671
|
+
r.notes.push("requested variables not in the uploaded file: " + missing.join(", ") +
|
|
672
|
+
". The file may belong to a different extract; check the extract definition.");
|
|
673
|
+
}
|
|
674
|
+
return r;
|
|
675
|
+
}
|
|
676
|
+
// --------------------------------------------------------------------------- schema, search
|
|
677
|
+
const COLLECTIONS = {
|
|
678
|
+
usa: "IPUMS USA: ACS (from 2000) and decennial census microdata. Samples like us2023a (1-year ACS), us2023c (5-year).",
|
|
679
|
+
cps: "IPUMS CPS: Current Population Survey, basic monthly (cps2025_01b) and ASEC (cps2023_03s, income of the prior year).",
|
|
680
|
+
nhis: "IPUMS NHIS: National Health Interview Survey (ih2024). No case selection, no data quality flags.",
|
|
681
|
+
meps: "IPUMS MEPS: Medical Expenditure Panel Survey (mp2022). No case selection, no attached characteristics.",
|
|
682
|
+
ipumsi: "IPUMS International: census microdata of many countries.",
|
|
683
|
+
atus: "IPUMS ATUS: American Time Use Survey. No case selection.",
|
|
684
|
+
};
|
|
685
|
+
async function schema(names, _pack) {
|
|
686
|
+
const t = texts();
|
|
687
|
+
const out = [];
|
|
688
|
+
for (const n of names) {
|
|
689
|
+
const lim = t.API_LIMITS[n];
|
|
690
|
+
const item = { name: n, summary: COLLECTIONS[n] ?? `IPUMS collection ${n}`,
|
|
691
|
+
spec: { collection: n, description: "text (up to 250 characters)", samples: "list of sample IDs (below)",
|
|
692
|
+
variables: "list of IPUMS variable names (uppercase mnemonics); see docs/variables.md",
|
|
693
|
+
case_selections: SELECTABLE.includes(n) ? '{"VAR": ["code", ...]} or {"VAR": {"min": 25, "max": 54}} or {"VAR": {"detailed": [...]}}'
|
|
694
|
+
: "not available for this collection",
|
|
695
|
+
case_select_who: SELECTABLE.includes(n) ? '"individuals" (default) or "households"' : "not available",
|
|
696
|
+
attached_characteristics: '{"VAR": ["spouse", "mother", "father", "head"]}', data_quality_flags: '["VAR", ...] or true',
|
|
697
|
+
adjust_monetary_values: SELECTABLE.includes(n) ? '["INCWAGE", ...] (CPI-adjusted)' : "not available",
|
|
698
|
+
data_structure: '{"rectangular": {"on": "P"}} (default) or {"hierarchical": {}}',
|
|
699
|
+
max_wait_s: `optional: seconds one fetch waits for IPUMS to build the extract (default ${DEFAULT_WAIT_S}); then fetch again to collect it` },
|
|
700
|
+
unsupported: lim?.unsupported ?? [], rectangular_on: lim?.rect_on ?? ["P"], weights: t.WEIGHT_HINTS[n] ?? {},
|
|
701
|
+
missing_codes: t.MISSING_HINT[n] ?? "", key: "IPUMS_API_KEY (the website builds the same extract with a free account and no key)" };
|
|
702
|
+
try {
|
|
703
|
+
item.samples = (await new IpumsClient().listSamples(n)).map((s) => ({ name: s.name, description: s.description }));
|
|
704
|
+
}
|
|
705
|
+
catch (e) {
|
|
706
|
+
item.samples = { error: `the sample list could not be read now (${String(e?.message || e).split("\n")[0].slice(0, 200)}); ` +
|
|
707
|
+
"IPUMS lists them on each collection's website (Select Samples)" };
|
|
708
|
+
}
|
|
709
|
+
out.push(item);
|
|
710
|
+
}
|
|
711
|
+
return out;
|
|
712
|
+
}
|
|
713
|
+
/** --list-samples: "<collection> <words>" finds sample IDs by ID or description. */
|
|
714
|
+
async function search(query, _pack) {
|
|
715
|
+
const words = query.trim().split(/\s+/);
|
|
716
|
+
const known = [...Object.keys(COLLECTIONS)];
|
|
717
|
+
const coll = known.includes(words[0]?.toLowerCase()) ? words.shift().toLowerCase() : "usa";
|
|
718
|
+
const m = words.join(" ").toLowerCase();
|
|
719
|
+
let all;
|
|
720
|
+
try {
|
|
721
|
+
all = await new IpumsClient().listSamples(coll);
|
|
722
|
+
}
|
|
723
|
+
catch (e) {
|
|
724
|
+
const why = String(e?.message || e).split("\n")[0].slice(0, 300);
|
|
725
|
+
return [{ collection: coll, error: `the sample list could not be read (${why})` }];
|
|
726
|
+
}
|
|
727
|
+
return all.filter((s) => s.name.toLowerCase().includes(m) || s.description.toLowerCase().includes(m))
|
|
728
|
+
.map((s) => ({ collection: coll, name: s.name, description: s.description }));
|
|
729
|
+
}
|
|
730
|
+
const pack = {
|
|
731
|
+
operations: Object.entries(COLLECTIONS).map(([name, summary]) => ({ name, summary })),
|
|
732
|
+
schema,
|
|
733
|
+
// a collection request_schema does not list (e.g. a newer IPUMS collection) needs no schema read first
|
|
734
|
+
uses: (spec) => (isObj(spec) && typeof spec.collection === "string" && spec.collection in COLLECTIONS ? [spec.collection] : []),
|
|
735
|
+
specExample: { collection: "usa", description: "Prime-age employment by nativity", samples: ["us2013a", "us2018a", "us2023a"],
|
|
736
|
+
variables: ["AGE", "SEX", "CITIZEN", "EDUCD", "EMPSTAT", "INCWAGE", "PERWT"],
|
|
737
|
+
case_selections: { AGE: { min: 25, max: 54 } } },
|
|
738
|
+
search,
|
|
739
|
+
checkSpec: (spec) => specProblems(spec),
|
|
740
|
+
fetch,
|
|
741
|
+
ingest,
|
|
742
|
+
};
|
|
743
|
+
export default pack;
|
|
744
|
+
//# sourceMappingURL=ipums.js.map
|