data-analyst-mcp 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +262 -3
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +710 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +21 -0
- package/dist/report/scan_secrets.js +234 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +246 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +56 -6
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +165 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Download an IPUMS extract with the IPUMS API: submit it, wait for it, and download the data file
|
|
3
|
+
# and the codebook. (macOS, Linux, Git Bash, WSL; on Windows PowerShell use download_ipums.ps1.)
|
|
4
|
+
#
|
|
5
|
+
# Usage: bash download_ipums.sh [EXTRACT_NUMBER | --new]
|
|
6
|
+
#
|
|
7
|
+
# - Asks for your IPUMS API key (hidden input), or uses $IPUMS_API_KEY when it is set.
|
|
8
|
+
# The key is never shown, saved, or put on a command line.
|
|
9
|
+
# - Submits the extract defined in this file, waits until IPUMS has built it
|
|
10
|
+
# (minutes for CPS, NHIS or MEPS, up to about 30 minutes for multi-year ACS), and downloads
|
|
11
|
+
# the data file (.csv.gz) and the DDI codebook (.xml) into the folder that holds this script.
|
|
12
|
+
# - Run it again after an interruption: it continues with the same extract instead of submitting
|
|
13
|
+
# a new one. EXTRACT_NUMBER overrides that number; --new submits a new extract.
|
|
14
|
+
# - Needs only curl, which macOS and most Linux systems have.
|
|
15
|
+
#
|
|
16
|
+
# Get a key at https://account.ipums.org/api_keys (the account must be registered for @COLLECTION_UPPER@).
|
|
17
|
+
set -euo pipefail
|
|
18
|
+
|
|
19
|
+
COLLECTION="@COLLECTION@"
|
|
20
|
+
BAKED_NUMBER="@NUMBER@"
|
|
21
|
+
SPEC_ID="@SPEC_ID@" # names the saved-number file, so another extract's file is ignored
|
|
22
|
+
API="${IPUMS_API_BASE:-https://api.ipums.org}"
|
|
23
|
+
API="${API%/}"
|
|
24
|
+
POLL_SECONDS=15
|
|
25
|
+
MAX_WAIT=14400 # 4 hours
|
|
26
|
+
|
|
27
|
+
# The extract definition (the request body for the IPUMS API). Do not edit it by hand.
|
|
28
|
+
write_spec() {
|
|
29
|
+
cat <<'IPUMS_EXTRACT_SPEC'
|
|
30
|
+
@SPEC_JSON@
|
|
31
|
+
IPUMS_EXTRACT_SPEC
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
NEW=0
|
|
35
|
+
case "${1:-}" in
|
|
36
|
+
-h|--help) sed -n '2,16p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
|
37
|
+
--new) NEW=1; shift ;;
|
|
38
|
+
esac
|
|
39
|
+
|
|
40
|
+
die() { echo "ERROR: $*" >&2; exit 1; }
|
|
41
|
+
|
|
42
|
+
command -v curl >/dev/null 2>&1 || die "curl is not installed. Install it (for example: sudo apt install curl) and run this script again."
|
|
43
|
+
|
|
44
|
+
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
45
|
+
NUMBER_FILE="$HERE/extract_number_${COLLECTION}_${SPEC_ID}.txt"
|
|
46
|
+
BODY_FILE="$(mktemp "${TMPDIR:-/tmp}/ipums_spec.XXXXXX")"
|
|
47
|
+
trap 'rm -f "$BODY_FILE"' EXIT
|
|
48
|
+
write_spec > "$BODY_FILE"
|
|
49
|
+
|
|
50
|
+
# --------------------------------------------------------------------------- API key
|
|
51
|
+
KEY="${IPUMS_API_KEY:-}"
|
|
52
|
+
if [ -n "$KEY" ]; then
|
|
53
|
+
echo "Using the IPUMS API key in \$IPUMS_API_KEY."
|
|
54
|
+
elif [ -t 0 ]; then
|
|
55
|
+
printf 'Paste your IPUMS API key (input is hidden), then press Enter: '
|
|
56
|
+
IFS= read -rs KEY; echo
|
|
57
|
+
else
|
|
58
|
+
IFS= read -r KEY || true
|
|
59
|
+
fi
|
|
60
|
+
KEY="$(printf '%s' "$KEY" | tr -d '[:space:]')"
|
|
61
|
+
[ -n "$KEY" ] || die "No key entered. Get one at https://account.ipums.org/api_keys"
|
|
62
|
+
KEY="${KEY//\\/\\\\}"; KEY="${KEY//\"/\\\"}" # escape for the curl config line
|
|
63
|
+
|
|
64
|
+
# The key reaches curl as a config file on stdin (-K -), never as a command-line argument.
|
|
65
|
+
auth_config() { printf 'header = "Authorization: %s"\n' "$KEY"; }
|
|
66
|
+
|
|
67
|
+
# --------------------------------------------------------------------------- HTTP helpers
|
|
68
|
+
# api METHOD URL [curl args...]: sets BODY and CODE. Retries network errors, 429 and 5xx.
|
|
69
|
+
api() {
|
|
70
|
+
local method="$1" url="$2" tries=0 out rc
|
|
71
|
+
shift 2
|
|
72
|
+
while :; do
|
|
73
|
+
tries=$((tries + 1))
|
|
74
|
+
set +e
|
|
75
|
+
out="$(auth_config | curl -sS -K - -X "$method" -w '\n%{http_code}' "$@" "$url")"
|
|
76
|
+
rc=$?
|
|
77
|
+
set -e
|
|
78
|
+
if [ $rc -eq 0 ]; then
|
|
79
|
+
CODE="${out##*$'\n'}"
|
|
80
|
+
BODY="${out%$'\n'*}"
|
|
81
|
+
case "$CODE" in 429|5??) ;; *) return 0 ;; esac
|
|
82
|
+
fi
|
|
83
|
+
if [ $tries -ge 5 ]; then
|
|
84
|
+
[ $rc -eq 0 ] && return 0
|
|
85
|
+
die "Could not reach IPUMS ($API). Check the internet connection, VPN and proxy, then run this script again."
|
|
86
|
+
fi
|
|
87
|
+
sleep $((tries * 10))
|
|
88
|
+
done
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
# check_http WHAT: stop with a clear message unless the last reply was 2xx.
|
|
92
|
+
check_http() {
|
|
93
|
+
case "$CODE" in
|
|
94
|
+
2??) ;;
|
|
95
|
+
401|403) die "IPUMS rejected the API key (HTTP $CODE). Check the key, and that your account is registered for $(printf '%s' "$COLLECTION" | tr '[:lower:]' '[:upper:]') at https://account.ipums.org/api_keys" ;;
|
|
96
|
+
404) die "IPUMS has no $COLLECTION extract number $NUMBER for this account. Check the number; to submit a new extract instead, run: bash \"$0\" --new" ;;
|
|
97
|
+
*) echo "IPUMS replied (HTTP $CODE):" >&2
|
|
98
|
+
printf '%s\n' "$BODY" >&2
|
|
99
|
+
die "$1 failed. Send the reply above to the analyst. Do not change the extract by hand." ;;
|
|
100
|
+
esac
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
# --------------------------------------------------------------------------- JSON helpers (no jq needed)
|
|
104
|
+
flat() { printf '%s' "$1" | tr -d '\r\n' | sed 's#\\/#/#g'; }
|
|
105
|
+
json_number() { flat "$1" | grep -o '"number" *: *[0-9][0-9]*' | head -n 1 | grep -o '[0-9][0-9]*$' || true; }
|
|
106
|
+
json_status() { flat "$1" | grep -o '"status" *: *"[a-z_]*"' | head -n 1 | sed 's/.*"\([a-z_]*\)"$/\1/' || true; }
|
|
107
|
+
# json_link JSON OBJECT FIELD: FIELD ("url" or "sha256") of downloadLinks.OBJECT
|
|
108
|
+
json_link() { flat "$1" | sed -n "s/.*\"$2\" *: *{[^}]*\"$3\" *: *\"\\([^\"]*\\)\".*/\\1/p" | head -n 1; }
|
|
109
|
+
|
|
110
|
+
# --------------------------------------------------------------------------- 1. submit (or reuse a number)
|
|
111
|
+
NUMBER="${1:-}"
|
|
112
|
+
if [ "$NEW" = 0 ]; then
|
|
113
|
+
if [ -z "$NUMBER" ] && [ -s "$NUMBER_FILE" ]; then
|
|
114
|
+
NUMBER="$(tr -cd '0-9' < "$NUMBER_FILE")"
|
|
115
|
+
[ -n "$NUMBER" ] && echo "Continuing with extract $NUMBER (saved in $NUMBER_FILE by an earlier run)."
|
|
116
|
+
fi
|
|
117
|
+
[ -n "$NUMBER" ] || NUMBER="$BAKED_NUMBER"
|
|
118
|
+
fi
|
|
119
|
+
if [ -n "$NUMBER" ]; then
|
|
120
|
+
case "$NUMBER" in *[!0-9]*) die "The extract number must be a whole number, not '$NUMBER'." ;; esac
|
|
121
|
+
echo "Using $COLLECTION extract $NUMBER (already submitted)."
|
|
122
|
+
else
|
|
123
|
+
[ -s "$BODY_FILE" ] || die "The extract definition is missing from this script."
|
|
124
|
+
cp "$BODY_FILE" "$HERE/extract_spec.json" 2>/dev/null || true # for the curl steps: always this extract's definition
|
|
125
|
+
echo "Submitting the extract to IPUMS ($COLLECTION)..."
|
|
126
|
+
api POST "$API/extracts?collection=$COLLECTION&version=2" \
|
|
127
|
+
-H "Content-Type: application/json" --data-binary "@$BODY_FILE"
|
|
128
|
+
check_http "Submitting the extract"
|
|
129
|
+
NUMBER="$(json_number "$BODY")"
|
|
130
|
+
[ -n "$NUMBER" ] || { printf '%s\n' "$BODY" >&2; die "IPUMS accepted the request but sent no extract number."; }
|
|
131
|
+
printf '%s\n' "$NUMBER" > "$NUMBER_FILE" 2>/dev/null || true
|
|
132
|
+
echo "Submitted: $COLLECTION extract $NUMBER."
|
|
133
|
+
fi
|
|
134
|
+
|
|
135
|
+
# --------------------------------------------------------------------------- 2. wait until it is built
|
|
136
|
+
echo "Waiting for IPUMS to build the extract (checked every $POLL_SECONDS seconds; Ctrl+C is safe, run again to continue)."
|
|
137
|
+
START=$(date +%s)
|
|
138
|
+
while :; do
|
|
139
|
+
api GET "$API/extracts/$NUMBER?collection=$COLLECTION&version=2"
|
|
140
|
+
check_http "Checking the extract status"
|
|
141
|
+
STATUS="$(json_status "$BODY")"
|
|
142
|
+
ELAPSED=$(( $(date +%s) - START ))
|
|
143
|
+
case "$STATUS" in
|
|
144
|
+
completed) printf '\r status: %-10s (%d min %02d s)\n' "$STATUS" $((ELAPSED / 60)) $((ELAPSED % 60)); break ;;
|
|
145
|
+
failed|canceled)
|
|
146
|
+
echo
|
|
147
|
+
echo "IPUMS replied:" >&2
|
|
148
|
+
printf '%s\n' "$BODY" >&2
|
|
149
|
+
die "The extract $STATUS. Send the reply above to the analyst. Do not resubmit or change it by hand." ;;
|
|
150
|
+
esac
|
|
151
|
+
printf '\r status: %-10s (%d min %02d s)' "${STATUS:-unknown}" $((ELAPSED / 60)) $((ELAPSED % 60))
|
|
152
|
+
[ "$ELAPSED" -lt "$MAX_WAIT" ] || { echo; die "Still not ready after $((MAX_WAIT / 3600)) hours. Run this script again later: it continues with extract $NUMBER."; }
|
|
153
|
+
sleep "$POLL_SECONDS"
|
|
154
|
+
done
|
|
155
|
+
|
|
156
|
+
# --------------------------------------------------------------------------- 3. download the two files
|
|
157
|
+
DATA_URL="$(json_link "$BODY" data url)"
|
|
158
|
+
DDI_URL="$(json_link "$BODY" ddiCodebook url)"
|
|
159
|
+
if [ -z "$DATA_URL" ] || [ -z "$DDI_URL" ]; then
|
|
160
|
+
die "Extract $NUMBER is complete but has no download links: IPUMS deletes extract files after about 72 hours. To submit a new extract, run: bash \"$0\" --new"
|
|
161
|
+
fi
|
|
162
|
+
DATA_SHA="$(json_link "$BODY" data sha256)"
|
|
163
|
+
|
|
164
|
+
# download URL: saves under the name IPUMS gave the file; prints the path
|
|
165
|
+
download() {
|
|
166
|
+
local url="$1" name dest progress=-sS
|
|
167
|
+
name="${url%%\?*}"; name="${name##*/}"
|
|
168
|
+
dest="$HERE/$name"
|
|
169
|
+
[ -t 2 ] && progress=-#
|
|
170
|
+
echo "Downloading $name ..." >&2
|
|
171
|
+
auth_config | curl "$progress" -fL -K - -o "$dest" "$url" \
|
|
172
|
+
|| die "The download of $name failed. Run this script again (it reuses extract $NUMBER)."
|
|
173
|
+
printf '%s' "$dest"
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
DATA_FILE="$(download "$DATA_URL")"
|
|
177
|
+
DDI_FILE="$(download "$DDI_URL")"
|
|
178
|
+
|
|
179
|
+
if [ -n "$DATA_SHA" ]; then
|
|
180
|
+
SUM=""
|
|
181
|
+
if command -v sha256sum >/dev/null 2>&1; then SUM="$(sha256sum "$DATA_FILE" | cut -d' ' -f1)"
|
|
182
|
+
elif command -v shasum >/dev/null 2>&1; then SUM="$(shasum -a 256 "$DATA_FILE" | cut -d' ' -f1)"; fi
|
|
183
|
+
if [ -n "$SUM" ]; then
|
|
184
|
+
[ "$SUM" = "$DATA_SHA" ] || die "The data file is damaged (checksum mismatch). Delete it and run this script again."
|
|
185
|
+
echo "Checksum OK."
|
|
186
|
+
fi
|
|
187
|
+
fi
|
|
188
|
+
|
|
189
|
+
echo
|
|
190
|
+
echo "Done. Upload these two files to the chat (do not unzip or edit them):"
|
|
191
|
+
echo " $DATA_FILE"
|
|
192
|
+
echo " $DDI_FILE"
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# IPUMS API v2 cheat sheet (microdata)
|
|
2
|
+
|
|
3
|
+
This server's `fetch` for ipums already implements all of this. Read this file
|
|
4
|
+
when you need a feature the friendly spec doesn't expose, or when debugging an API error.
|
|
5
|
+
|
|
6
|
+
## Auth
|
|
7
|
+
- Header `Authorization: <key>`. The key comes from env `IPUMS_API_KEY`; never print or log it.
|
|
8
|
+
- The **401** error usually means one of two things. Either the key is wrong, or the account isn't registered for that collection. Registration is separate per collection (usa.ipums.org, cps.ipums.org, ...). Tell the user which collection to register for, then stop.
|
|
9
|
+
|
|
10
|
+
- When the key is missing, rejected or the API is unreachable, `fetch` returns an error (`key_missing`,
|
|
11
|
+
`key_rejected` or `agency_unavailable`) with its fix. `manual_bundle` cannot write an IPUMS bundle yet, so a
|
|
12
|
+
sandbox user builds the same extract on the collection's website (free account, no key) and uploads the
|
|
13
|
+
`.csv.gz` and `.xml`; `ingest` packages them. The three calls are POST `/extracts`, GET `/extracts/{number}`
|
|
14
|
+
until `status` is `completed`, then GET `downloadLinks.data.url` and `downloadLinks.ddiCodebook.url`
|
|
15
|
+
(all with the Authorization header).
|
|
16
|
+
|
|
17
|
+
## Endpoints (`https://api.ipums.org`, every call takes `?collection=<c>&version=2`)
|
|
18
|
+
| What | Call |
|
|
19
|
+
|---|---|
|
|
20
|
+
| List sample IDs | `GET /metadata/samples?collection=usa&version=2&pageSize=500` returns `{data:[{name, description}], links:{nextPage}}` |
|
|
21
|
+
| Submit extract | `POST /extracts?collection=usa&version=2` with a JSON body (below) returns `{number, status:"queued"}` |
|
|
22
|
+
| Status | `GET /extracts/{number}?collection=usa&version=2` returns `status`: queued, started, produced, completed, failed, canceled |
|
|
23
|
+
| Recent extracts | `GET /extracts?collection=usa&version=2&pageSize=10` |
|
|
24
|
+
| Download | `GET <downloadLinks.data.url>` with the same Authorization header |
|
|
25
|
+
|
|
26
|
+
There is **no variable-metadata endpoint for microdata**. Get variable names from
|
|
27
|
+
`docs/variables.md`, or from the collection's website (`https://usa.ipums.org/usa-action/variables/<NAME>`).
|
|
28
|
+
After the download, `codebook.csv` holds the authoritative codes.
|
|
29
|
+
|
|
30
|
+
## Extract body
|
|
31
|
+
```json
|
|
32
|
+
{
|
|
33
|
+
"description": "Prime-age employment by nativity",
|
|
34
|
+
"dataStructure": {"rectangular": {"on": "P"}},
|
|
35
|
+
"dataFormat": "csv",
|
|
36
|
+
"samples": {"us2023a": {}, "us2018a": {}},
|
|
37
|
+
"variables": {
|
|
38
|
+
"AGE": {"caseSelections": {"general": ["25", "26", "27"]}},
|
|
39
|
+
"EDUCD": {},
|
|
40
|
+
"INCWAGE": {"dataQualityFlags": true, "adjustMonetaryValues": true},
|
|
41
|
+
"EDUC": {"attachedCharacteristics": ["spouse", "head"]}
|
|
42
|
+
},
|
|
43
|
+
"caseSelectWho": "individuals"
|
|
44
|
+
}
|
|
45
|
+
```
|
|
46
|
+
- `dataFormat`: `csv` (the default here, a gzipped CSV), `fixed_width`, `stata`, `spss`, `sas9`. CSV is the most portable choice for analysts. Every extract also comes with a DDI XML codebook.
|
|
47
|
+
- `dataStructure`: `{"rectangular":{"on":"P"}}` gives one row per person with household variables repeated on each row. `{"hierarchical":{}}` gives separate H and P records; avoid it. `{"householdOnly":{}}` is USA only.
|
|
48
|
+
- `caseSelections.general` lists codes as strings. The API accepts unpadded codes (`"25"`, as tested on CPS AGE). For a *detailed* variable code, use `{"detailed": [...]}`.
|
|
49
|
+
- Case selection on several variables is AND-ed. Several codes within one variable are OR-ed.
|
|
50
|
+
- `caseSelectWho`: `individuals` (the default) keeps only matching persons. `households` keeps everyone in a household that contains a matching person; use it for family or household measures.
|
|
51
|
+
- `attachedCharacteristics`: `mother`, `father`, `spouse`, `head`. Each adds a column such as `EDUC_SP`.
|
|
52
|
+
- `adjustMonetaryValues: true` (usa/cps/ipumsi) adds a CPI-adjusted copy of a dollar variable, for example `INCWAGE_CPIU_2010`. Check `variables.csv` for its label and base year.
|
|
53
|
+
- `dataQualityFlags: true` adds a `Q<VAR>` column showing whether the value was allocated or imputed.
|
|
54
|
+
- Technical variables are added automatically (YEAR, SERIAL, PERNUM and the weights, plus SAMPLE/CBSERIAL/CLUSTER/STRATA/GQ for USA, plus MONTH/CPSID/ASECFLAG for CPS, STRATA/PSU/SAMPWEIGHT/ASTATFLG/CSTATFLG for NHIS, PSUANN/STRATANN/PSUPLD/STRATAPLD/PERWEIGHT/SAQWEIGHT/DIABWEIGHT for MEPS).
|
|
55
|
+
|
|
56
|
+
## Timing and limits
|
|
57
|
+
- A small CPS, NHIS or MEPS extract finishes in about 10–60 s (one NHIS or MEPS year: 11 s in October 2026). ACS 1-year extracts take minutes. Multi-year ACS 5-year or full-count extracts can take 30+ minutes.
|
|
58
|
+
- If the fetch times out (after the spec's `max_wait_s`, default 600), the extract keeps running at IPUMS. The next `fetch` with the same spec collects it.
|
|
59
|
+
- Extract files expire after about 72 hours. The definition saved in `data/ipums/extract_spec.json` can be resubmitted at any time (`fetch` with `force: true`).
|
|
60
|
+
- On 429 or 5xx errors the client backs off and retries automatically.
|
|
61
|
+
|
|
62
|
+
## Collections that support microdata extracts through the API
|
|
63
|
+
`usa`, `cps`, `ipumsi`, `atus`, `ahtus`, `mtus`, `nhis`, `meps`, `dhs`. Case selection and monetary adjustment are
|
|
64
|
+
available for usa, cps and ipumsi only (`fetch` refuses them for the others with `invalid_request`). The API also refuses data quality flags for NHIS and attached characteristics for MEPS; it rectangularizes NHIS on person or injury records (`P`, `I`) and MEPS on person or round records (`P`, `R`), and both allow hierarchical extracts (developer.ipums.org, checked 2026-10). NHIS and MEPS
|
|
65
|
+
variable names, weights and design variables are in `variables.md` sections 8-9. An unknown variable name returns
|
|
66
|
+
HTTP 400 listing every invalid name ("Invalid variable name: EMPSTAT"), and a variable outside the chosen samples
|
|
67
|
+
says so ("PARTWEIGHT: This variable is not available in any of the samples currently selected"): fix and resubmit.
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# Report patterns for IPUMS data products
|
|
2
|
+
|
|
3
|
+
Use this file to shape the analysis for every product, and the layout when a report or paper is
|
|
4
|
+
built. The questions and conventions apply to quick answers too; the publication structure applies only
|
|
5
|
+
in report or paper mode. The aim is a product a careful reviewer would recognize: precise populations,
|
|
6
|
+
weighted estimates, a lead summary with numbers, numbered exhibits that each carry notes and a source
|
|
7
|
+
line, exhibit data shipped alongside, and limitations stated plainly.
|
|
8
|
+
|
|
9
|
+
## Typical questions and how they are framed
|
|
10
|
+
|
|
11
|
+
These recur in published work by statistical and policy agencies (Census Bureau, BLS, GAO, CRS,
|
|
12
|
+
state agencies, research institutes), so there is usually a precedent to follow and benchmark against.
|
|
13
|
+
|
|
14
|
+
- **Employment of immigrants** (ACS, CPS ASEC): people ages 25 to 54, split by sex, by nativity and
|
|
15
|
+
citizenship, by education and by birthplace. Measures: weighted population counts (millions),
|
|
16
|
+
employment rates (employed ÷ population) and average annual earnings of workers. Headline style:
|
|
17
|
+
"In 2018, 46 million people living in the United States, or 14 percent of the population, had been
|
|
18
|
+
born in other countries." Public IPUMS data **cannot** identify legal status (published work that
|
|
19
|
+
reports it imputes it), so report nativity and citizenship (naturalized vs. noncitizen) instead, and
|
|
20
|
+
say so.
|
|
21
|
+
- **Public-sector vs. private-sector pay** (CPS): full-time, full-year civilian workers, in 5 education
|
|
22
|
+
groups (high school diploma or less; some college; bachelor's degree; master's degree; professional
|
|
23
|
+
degree or doctorate). Measures: average hourly wages (annual wages ÷ annual hours), then
|
|
24
|
+
regression-adjusted differences that control for education, experience, occupation, employer size,
|
|
25
|
+
region, veteran status and demographics. With IPUMS alone: raw wage gaps, and optionally an OLS on
|
|
26
|
+
log hourly wage with controls. Benefits are **not** in IPUMS, so say so.
|
|
27
|
+
- **Veterans' labor market outcomes** (CPS, ACS): for example, men ages 22 to 44 who left active duty
|
|
28
|
+
after September 2001, compared with men who never served. Measures: labor force participation rate,
|
|
29
|
+
unemployment rate (unemployed ÷ labor force) and earnings, by age group and by disability rating.
|
|
30
|
+
- Other recurring uses of the same data:
|
|
31
|
+
- labor force projections by age and sex (CPS monthly);
|
|
32
|
+
- minimum wage analyses that count affected workers by hourly wage (CPS outgoing rotation groups: EARNWEEK, HOURWAGE, EARNWT);
|
|
33
|
+
- health insurance coverage by source (ACS HCOV*, CPS ASEC);
|
|
34
|
+
- distribution of household income and poverty (CPS ASEC);
|
|
35
|
+
- immigration surges (ACS and CPS: education and labor force participation of recent arrivals, using YRIMMIG / YRSUSA1);
|
|
36
|
+
- state-level profiles (ACS: large samples with STATEFIP and PUMA detail).
|
|
37
|
+
- **Health insurance and access by group** (NHIS): uninsured at the interview, private vs. public coverage,
|
|
38
|
+
no usual place of care, care delayed or forgone because of cost, by age (0-17, 18-64, 65+), poverty ratio,
|
|
39
|
+
race and Hispanic origin, nativity. Benchmark the headline against NCHS's annual Health Insurance Coverage
|
|
40
|
+
report (2024: 8.2% of all ages and 11.6% of adults 18-64 uninsured). NHIS has no state identifier in the
|
|
41
|
+
public files: state estimates come from the ACS.
|
|
42
|
+
- **Health care spending** (MEPS): mean and median spending per person, the share with any spending,
|
|
43
|
+
out-of-pocket spending and payer shares, by age, insurance type (`COVERTYPE`) and income group (`POVCAT`).
|
|
44
|
+
Benchmark against AHRQ's MEPS-HC summary tables or Statistical Briefs (2022: $6,765 per person). Name the dollar
|
|
45
|
+
year; per-person means hide extreme concentration, so pair them with medians or the top-percentile share.
|
|
46
|
+
|
|
47
|
+
## Conventions to follow
|
|
48
|
+
|
|
49
|
+
**Population and definitions**
|
|
50
|
+
- State the universe exactly: age range, civilian vs. all, noninstitutional (USA GQ 1, 2 or 5 = households), years and geography.
|
|
51
|
+
- The prime-age default is 25–54. It avoids school-age and retirement-age confounding.
|
|
52
|
+
- **Employment rate** = employed ÷ population. **Unemployment rate** = unemployed ÷ labor force. **Participation rate** = labor force ÷ population.
|
|
53
|
+
- **Earnings:** say whose earnings are averaged. Usually it is workers with positive wages, sometimes full-time, full-year workers only. Also say whether self-employment income is included (INCWAGE excludes it; INCEARN in the ACS includes it).
|
|
54
|
+
|
|
55
|
+
**Estimation**
|
|
56
|
+
- Always weight. In the USA collection use PERWT (HHWT for household measures). In the CPS ASEC use ASECWT, in CPS basic monthly use WTFINL (COMPWT for labor-force levels that match BLS), and in the CPS outgoing rotation groups use EARNWT. In NHIS (2019 on) use SAMPWEIGHT on sample adults or sample children; in MEPS use PERWEIGHT (SAQWEIGHT for questionnaire items).
|
|
57
|
+
- Report the unweighted cell size internally. Suppress or footnote cells with fewer than about 100 respondents; unstable estimates are not published.
|
|
58
|
+
- **Dollars:** convert to a single year's dollars and name the year and index ("in 2023 dollars, using the CPI-U-RS"; the PCE price index is a common choice for earnings and income), with CPI99 or `adjustMonetaryValues`. CPS ASEC income is for the **previous calendar year** (ASEC 2023 means 2022 income). ACS income is for the past 12 months, and IPUMS expresses it in survey-year dollars.
|
|
59
|
+
- **Top-coding:** INCWAGE is top-coded, which mostly matters for means at high education levels. Mention it.
|
|
60
|
+
- **Sampling error:** for differences that drive conclusions, note whether they are large relative to sampling error. ACS replicate weights (REPWTP, 80) and CPS ASEC replicate weights (REPWTP, 160) give proper standard errors. CPS basic monthly files have none (REPWTP is ASEC-only) and no design variables: an SE there can only be approximate (clustered by household, `SERIAL` within month). This server does not compute standard errors yet: report which weight was used and that standard errors were not computed.
|
|
61
|
+
|
|
62
|
+
**Publication structure (mirrored in `build_report`, `"mode": "report"`; only when a report is asked for)**
|
|
63
|
+
|
|
64
|
+
The report layout, the **exhibit chartbook**, runs in this order:
|
|
65
|
+
1. **Cover:** "Month Year", the title (with the period for trend reports), and a hero chart that has its own title and unit label.
|
|
66
|
+
2. **At a Glance** (one page of about 300–420 words):
|
|
67
|
+
- one framing paragraph saying what was examined, the population, the period and the dimensions;
|
|
68
|
+
- 3–5 bullets, each opening with a **bold lead phrase.** that names a section, followed by 2–4 sentences with the key numbers.
|
|
69
|
+
3. **Contents:** the Introduction subsections, each section with an "Exhibits a–b" sub-line, the appendixes, and About This Document when there is one.
|
|
70
|
+
4. **List of Exhibits:** grouped under section headings, each entry "N. Title, years".
|
|
71
|
+
5. **Notes About This Report** (also titled "Notes" or "Notes and Definitions"). Short unnumbered paragraphs covering:
|
|
72
|
+
- calendar years;
|
|
73
|
+
- "Numbers in the text and exhibits may not add up to totals because of rounding.";
|
|
74
|
+
- the dollar year and deflator;
|
|
75
|
+
- the data source and universe;
|
|
76
|
+
- key definitions;
|
|
77
|
+
- recession shading, if any;
|
|
78
|
+
- where the data are posted.
|
|
79
|
+
6. **Introduction** (or "Summary and Introduction"):
|
|
80
|
+
- a lead paragraph with the headline numbers;
|
|
81
|
+
- question-style subheadings, one per section ("How Do Demographic Factors Relate to …?");
|
|
82
|
+
- closing subsections such as "Why Focus on People Ages 25 to 54?", "What Data Were Used?" and, when updating earlier work, "What Is New in This Report?".
|
|
83
|
+
7. **Thematic sections** (2–5, unnumbered):
|
|
84
|
+
- Each opens on a full-width page with 1–3 summary paragraphs.
|
|
85
|
+
- Then comes **one exhibit per page**: "Exhibit N." with a title-case title ending in ", 1998 to 2018", a unit label above the chart, and 2–4 narrative paragraphs beside it (levels, changes, comparisons, "(see Exhibit 5)").
|
|
86
|
+
- Below the chart: the data source line ("Data source: `<organization>`, using data from …", or "Data source: Analyst calculations using data from …" when no organization is named), then the notes, then lettered notes ("a.").
|
|
87
|
+
8. **Appendix A: Data and Methods** (or "Appendix: Estimating …"): data, sample, universe, weights, imputations, reliability, comparison with other estimates. Sometimes followed by **Appendix B: Definitions**, an alphabetical glossary.
|
|
88
|
+
9. **About This Document** (optional), in this order:
|
|
89
|
+
- who requested it, or what question prompted it;
|
|
90
|
+
- who prepared it, with what data and tools;
|
|
91
|
+
- the reviewers, if any;
|
|
92
|
+
- the date.
|
|
93
|
+
10. **Footnotes** are numbered in each part (restarting in each section), and cite publications as: Organization, *Title* (Month Year), URL.
|
|
94
|
+
|
|
95
|
+
Other variants, not built by `build_report`:
|
|
96
|
+
- **Long-form**:
|
|
97
|
+
- a Summary with Figure S-1 / Table S-1;
|
|
98
|
+
- "Chapter N:" sections in two-column prose with embedded "Figure 1-1." / "Table 2-3.";
|
|
99
|
+
- Boxes;
|
|
100
|
+
- a List of Tables and Figures at the back.
|
|
101
|
+
- **Hybrid**: full-width pages each with a heading, a paragraph and unnumbered charts; sources pooled in an appendix.
|
|
102
|
+
- Use `"mode": "paper"` when a prose-led product is wanted.
|
|
103
|
+
|
|
104
|
+
**Reading PDFs from sites that block scripts:** some publishers refuse scripts and the in-app browser (HTTP 403, bot-check slider). Fetch `https://web.archive.org/web/<year>/<full pdf url>` and extract the text with pypdf. If the plain year URL returns HTML, use a CDX timestamp with `id_` (for example `https://web.archive.org/web/20240430034127id_/<url>`). Cite the original URL.
|
|
105
|
+
|
|
106
|
+
**Data files:** the codebook, the extract definition (`data/ipums/extract_spec.json`), `reproduce/ipums/`
|
|
107
|
+
(written by `build_report`: the request as `spec.json` and `REPRODUCE.md`, which says how to request the same
|
|
108
|
+
extract again), all attached to the page with Download buttons, and each exhibit's CSV ship with the product,
|
|
109
|
+
so every number can be traced and the data requested again. The extract CSV (microdata) stays local: the
|
|
110
|
+
IPUMS terms forbid redistributing it, so the report never links or embeds it, and `build_report` never puts
|
|
111
|
+
it in a package (`"replication": {"include_microdata": true}` is ignored).
|
|
112
|
+
|
|
113
|
+
**Chart forms**
|
|
114
|
+
- Trends over time: line charts with at most 4–5 series. Facet (one exhibit per sex) rather than drawing 8 lines.
|
|
115
|
+
- Group comparisons at a point in time: grouped columns, or horizontal bars when the category labels are long (education, birthplace).
|
|
116
|
+
- Composition (wages + benefits, population by status): stacked columns.
|
|
117
|
+
- No dual axes, no pie charts, no 3-D.
|