data-analyst-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -2
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +716 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +24 -0
- package/dist/report/scan_secrets.js +265 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +247 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +54 -4
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +170 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,634 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DOL API v4 adapter: dataset paths, filter_object building, case-tolerant name search, key-only needs.
|
|
3
|
+
*
|
|
4
|
+
* - Datasets are addressed as AGENCY/api_url (OSHA/inspection) or by catalog alias (osha_inspection); the
|
|
5
|
+
* catalog was built from the keyless Datasets API (https://apiprod.dol.gov/v4/datasets).
|
|
6
|
+
* - filter_object: DOL takes one JSON object of {field, operator, value} conditions joined by {"and": [...]}
|
|
7
|
+
* or {"or": [...]}; operators eq, neq, gt, lt, in, not_in, like (DOL API User Guide). The pack builds it
|
|
8
|
+
* from `filters` ({field: value} eq, lists -> in, {"gt": ..} etc.), `search` and `state`, ANDed with any
|
|
9
|
+
* raw `filter_object`.
|
|
10
|
+
* - search: DOL's "like" is case-sensitive and has no implicit wildcards, and there is no case-insensitive
|
|
11
|
+
* operator, so {field: text} becomes an OR of %text% in the spelling given, UPPER, lower and Title case.
|
|
12
|
+
* - sort: defaults to the dataset's date field, descending (newest first), as listed in the catalog.
|
|
13
|
+
* - Empty bodies (no match) count as zero rows. The key goes in the X-API-KEY query parameter.
|
|
14
|
+
* - keyOnlyNeeds: everything except the keyless catalog endpoints (datasets, dataset_info) and the OUI files
|
|
15
|
+
* needs a key.
|
|
16
|
+
* - Field names are checked against DOL's keyless field dictionary. checkSpec is synchronous here, so it checks
|
|
17
|
+
* the names of a dataset whose dictionary this process has already read; fetch reads the dictionary first and
|
|
18
|
+
* refuses unknown names before any data request.
|
|
19
|
+
*
|
|
20
|
+
* Unemployment insurance files of the Office of Unemployment Insurance (oui.doleta.gov, keyless, fixed names
|
|
21
|
+
* refreshed in place; endpoints with "host": "oui"):
|
|
22
|
+
* - ui_trust_fund: ETA 2112 (UI financial transaction summary), /unemploy/csv/ar2112.csv, one row per state and
|
|
23
|
+
* month since 1979, columns coded c1..c136, renamed from the data map (ET Handbook 402, table ar2112;
|
|
24
|
+
* endpoints.json "columns"): <line item>_<account>, account total, clearing, utf or bpa. The whole file (23 MB)
|
|
25
|
+
* is downloaded once per fetch and filtered here by state, month and fields.
|
|
26
|
+
* - ui_data_summary: the UI Data Summary workbook (SummaryTables.xlsx, latest calendar-year quarter). Read from row
|
|
27
|
+
* 1 so the quarter in the title row is kept (cyq); the header row is found by its first cell ("State").
|
|
28
|
+
* These go to oui.doleta.gov with a client that has no credentials: the DOL API key is never sent there.
|
|
29
|
+
*/
|
|
30
|
+
import fs from "node:fs";
|
|
31
|
+
import path from "node:path";
|
|
32
|
+
import { PACKS_DIR } from "../core/env.js";
|
|
33
|
+
import * as gr from "../core/generic-rest.js";
|
|
34
|
+
import { Client, DataError } from "../core/http.js";
|
|
35
|
+
import { readJson } from "../core/policy.js";
|
|
36
|
+
import { parsePyJson, pyFloatRepr, pyQuote, pyStr, pyUrlencode } from "../core/py.js";
|
|
37
|
+
import { readCsv, sheetRecords } from "../core/tabular.js";
|
|
38
|
+
import { pyRepr, state, StateError } from "./_dol_states.js";
|
|
39
|
+
const OUI = ["ui_trust_fund", "ui_data_summary"];
|
|
40
|
+
const KEYLESS = ["datasets", "dataset_info", ...OUI];
|
|
41
|
+
const OUI_BASE = "https://oui.doleta.gov";
|
|
42
|
+
const fmt = (n) => n.toLocaleString("en-US");
|
|
43
|
+
const cmp = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
44
|
+
const blank = (v) => v === null || v === undefined || v === "";
|
|
45
|
+
const isObj = (v) => !!v && typeof v === "object" && !Array.isArray(v);
|
|
46
|
+
let EPS = null;
|
|
47
|
+
function eps() {
|
|
48
|
+
EPS ??= readJson(path.join(PACKS_DIR, "dol"), "endpoints.json");
|
|
49
|
+
return EPS;
|
|
50
|
+
}
|
|
51
|
+
function entry(ds) {
|
|
52
|
+
const cat = eps().catalog;
|
|
53
|
+
const k = pyStr(ds);
|
|
54
|
+
if (typeof ds === "string" && ds in cat)
|
|
55
|
+
return cat[ds];
|
|
56
|
+
for (const c of Object.values(cat))
|
|
57
|
+
if (String(c.id).toLowerCase() === k.toLowerCase())
|
|
58
|
+
return c;
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
function keyOnlyNeeds(spec) {
|
|
62
|
+
const reqs = (spec.requests || []).filter((r) => !KEYLESS.includes(r.endpoint || "query"));
|
|
63
|
+
if (!reqs.length)
|
|
64
|
+
return [];
|
|
65
|
+
const names = [...new Set(reqs.map((r) => (r.name === undefined ? "?" : pyStr(r.name))))].sort(cmp);
|
|
66
|
+
return [`data queries (${names.join(", ")}); only the dataset catalog (datasets, dataset_info) is keyless`];
|
|
67
|
+
}
|
|
68
|
+
const FIELDS = new Map();
|
|
69
|
+
/** Column names of one dataset from DOL's keyless field dictionary, or null when it cannot be read (then no name is
|
|
70
|
+
* refused: the check is skipped, and a wrong name surfaces as the API's own HTTP 400). */
|
|
71
|
+
async function datasetFields(pack, datasetId) {
|
|
72
|
+
const key = pyStr(datasetId);
|
|
73
|
+
if (FIELDS.has(key))
|
|
74
|
+
return FIELDS.get(key);
|
|
75
|
+
let cols = new Set();
|
|
76
|
+
try {
|
|
77
|
+
const n = Number(datasetId);
|
|
78
|
+
if (!Number.isInteger(n))
|
|
79
|
+
throw new TypeError("not an id");
|
|
80
|
+
const doc = await gr.clientFor(pack, undefined, { retries: 0, timeout: 10 }).getJson(`/datasets/${n}`, null, { useKey: false });
|
|
81
|
+
const meta = gr.dig(doc, "dataset.dataset_metadatum") || [];
|
|
82
|
+
cols = new Set(meta.filter((m) => m?.column_name).map((m) => pyStr(m.column_name)));
|
|
83
|
+
}
|
|
84
|
+
catch (e) {
|
|
85
|
+
if (!(e instanceof DataError || e instanceof TypeError || e instanceof SyntaxError))
|
|
86
|
+
throw e;
|
|
87
|
+
cols = new Set();
|
|
88
|
+
}
|
|
89
|
+
FIELDS.set(key, cols.size ? cols : null);
|
|
90
|
+
return FIELDS.get(key);
|
|
91
|
+
}
|
|
92
|
+
/** Every field a query request names: filters, search, sort_by and fields. */
|
|
93
|
+
function namedFields(p) {
|
|
94
|
+
const names = [...(isObj(p.filters) ? Object.keys(p.filters) : []), ...(isObj(p.search) ? Object.keys(p.search) : [])];
|
|
95
|
+
if (p.sort_by)
|
|
96
|
+
names.push(pyStr(p.sort_by));
|
|
97
|
+
const f = p.fields;
|
|
98
|
+
if (f)
|
|
99
|
+
names.push(...(Array.isArray(f) ? f : pyStr(f).split(",")).map((x) => pyStr(x).trim()).filter(Boolean));
|
|
100
|
+
return [...new Set(names)];
|
|
101
|
+
}
|
|
102
|
+
/** The field-name problem of one request, given the dataset's columns (null: none known). */
|
|
103
|
+
function fieldProblem(r, e, named, cols) {
|
|
104
|
+
const bad = named.filter((f) => cols && !cols.has(f));
|
|
105
|
+
if (!bad.length || !cols)
|
|
106
|
+
return null;
|
|
107
|
+
return `request ${pyRepr(r.name ?? null)}: ${bad.map(pyRepr).join(", ")} not a field of ${e.id} (its fields: ` +
|
|
108
|
+
`${[...cols].sort(cmp).join(", ")}). Field names are DOL's own (dataset_info lists them with meanings); ` +
|
|
109
|
+
'if the API does have the field and the dictionary lacks it, set "check_fields": false on the request';
|
|
110
|
+
}
|
|
111
|
+
function fieldCheck(r) {
|
|
112
|
+
if ((r.endpoint || "query") !== "query")
|
|
113
|
+
return null;
|
|
114
|
+
const p = r.params || {};
|
|
115
|
+
const ds = p.dataset || r.dataset;
|
|
116
|
+
const e = ds ? entry(ds) : null;
|
|
117
|
+
const named = r.check_fields ?? true ? namedFields(p) : [];
|
|
118
|
+
return named.length && e && e.dataset_id ? [e, named] : null;
|
|
119
|
+
}
|
|
120
|
+
function checkSpec(spec, _pack) {
|
|
121
|
+
let probs = [];
|
|
122
|
+
for (const r of spec.requests || []) {
|
|
123
|
+
if (r.endpoint && OUI.includes(r.endpoint)) {
|
|
124
|
+
probs = probs.concat(checkOui(r));
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
if ((r.endpoint || "query") !== "query")
|
|
128
|
+
continue;
|
|
129
|
+
const p = r.params || {};
|
|
130
|
+
const n = pyRepr(r.name ?? null);
|
|
131
|
+
const ds = p.dataset || r.dataset;
|
|
132
|
+
const e = ds ? entry(ds) : null;
|
|
133
|
+
if (ds && !e && !pyStr(ds).includes("/")) {
|
|
134
|
+
probs.push(`request ${n}: unknown dataset ${pyRepr(ds)}: use a catalog alias (request_schema search) or AGENCY/api_url`);
|
|
135
|
+
}
|
|
136
|
+
if (p.state && !(e || {}).state_field) {
|
|
137
|
+
probs.push(`request ${n}: 'state' is only known for datasets with a state field ` +
|
|
138
|
+
"(osha_inspection, whd_enforcement, eta_ui_state_weekly_claims); use filters instead");
|
|
139
|
+
}
|
|
140
|
+
const fo = p.filter_object;
|
|
141
|
+
if (typeof fo === "string") {
|
|
142
|
+
try {
|
|
143
|
+
JSON.parse(fo);
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
probs.push(`request ${n}: filter_object is not valid JSON`);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
if (r.max_rows && Number(r.max_rows) > 100000)
|
|
150
|
+
probs.push(`request ${n}: max_rows above 100,000: narrow the filters`);
|
|
151
|
+
// field names: DOL's field dictionary is keyless; a dictionary already read is checked here (fetch reads it)
|
|
152
|
+
const fc = fieldCheck(r);
|
|
153
|
+
if (fc && FIELDS.has(pyStr(fc[0].dataset_id))) {
|
|
154
|
+
const prob = fieldProblem(r, fc[0], fc[1], FIELDS.get(pyStr(fc[0].dataset_id)));
|
|
155
|
+
if (prob)
|
|
156
|
+
probs.push(prob);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
return probs;
|
|
160
|
+
}
|
|
161
|
+
function variants(text) {
|
|
162
|
+
const out = [];
|
|
163
|
+
const title = text.replace(/\p{L}+/gu, (w) => w[0].toUpperCase() + w.slice(1).toLowerCase());
|
|
164
|
+
for (const v of [text, text.toUpperCase(), text.toLowerCase(), title]) {
|
|
165
|
+
const w = v.includes("%") ? v : `%${v}%`;
|
|
166
|
+
if (!out.includes(w))
|
|
167
|
+
out.push(w);
|
|
168
|
+
}
|
|
169
|
+
return out;
|
|
170
|
+
}
|
|
171
|
+
/** json.dumps(obj, separators=(",", ":")) (ensure_ascii). */
|
|
172
|
+
function compactJson(v) {
|
|
173
|
+
if (v === null || v === undefined)
|
|
174
|
+
return "null";
|
|
175
|
+
if (v === true)
|
|
176
|
+
return "true";
|
|
177
|
+
if (v === false)
|
|
178
|
+
return "false";
|
|
179
|
+
if (typeof v === "number")
|
|
180
|
+
return Number.isFinite(v) ? String(v) : v > 0 ? "Infinity" : v < 0 ? "-Infinity" : "NaN";
|
|
181
|
+
if (typeof v === "string")
|
|
182
|
+
return JSON.stringify(v).replace(/[\u0080-\uffff]/g, (c) => "\\u" + c.charCodeAt(0).toString(16).padStart(4, "0"));
|
|
183
|
+
if (Array.isArray(v))
|
|
184
|
+
return "[" + v.map(compactJson).join(",") + "]";
|
|
185
|
+
return "{" + Object.entries(v).map(([k, x]) => `${compactJson(k)}:${compactJson(x)}`).join(",") + "}";
|
|
186
|
+
}
|
|
187
|
+
class KeyErr extends Error {
|
|
188
|
+
}
|
|
189
|
+
function buildFilter(params, e) {
|
|
190
|
+
const conds = [];
|
|
191
|
+
const filters = params.filters || {};
|
|
192
|
+
delete params.filters;
|
|
193
|
+
for (const [field, v] of Object.entries(filters)) {
|
|
194
|
+
if (isObj(v))
|
|
195
|
+
for (const [op, val] of Object.entries(v))
|
|
196
|
+
conds.push({ field, operator: op, value: val });
|
|
197
|
+
else if (Array.isArray(v))
|
|
198
|
+
conds.push({ field, operator: "in", value: [...v] });
|
|
199
|
+
else
|
|
200
|
+
conds.push({ field, operator: "eq", value: v });
|
|
201
|
+
}
|
|
202
|
+
const search = params.search || {};
|
|
203
|
+
delete params.search;
|
|
204
|
+
for (const [field, text] of Object.entries(search)) {
|
|
205
|
+
const alts = variants(pyStr(text)).map((w) => ({ field, operator: "like", value: w }));
|
|
206
|
+
conds.push(alts.length === 1 ? alts[0] : { or: alts });
|
|
207
|
+
}
|
|
208
|
+
const st = params.state;
|
|
209
|
+
delete params.state;
|
|
210
|
+
if (st) {
|
|
211
|
+
if (!e || !("state_field" in e))
|
|
212
|
+
throw new KeyErr("'state_field'");
|
|
213
|
+
conds.push({ field: e.state_field, operator: "eq", value: pyStr(st).trim().toUpperCase() });
|
|
214
|
+
}
|
|
215
|
+
const raw = params.filter_object;
|
|
216
|
+
delete params.filter_object;
|
|
217
|
+
if (raw)
|
|
218
|
+
conds.push(typeof raw === "string" ? JSON.parse(raw) : raw);
|
|
219
|
+
if (!conds.length)
|
|
220
|
+
return null;
|
|
221
|
+
return compactJson(conds.length === 1 ? conds[0] : { and: conds });
|
|
222
|
+
}
|
|
223
|
+
/** [endpoint with the path filled in, query params, notes] for one request. */
|
|
224
|
+
function prepare(req, ep, params0) {
|
|
225
|
+
const params = { ...params0 };
|
|
226
|
+
const kind = req.endpoint || "query";
|
|
227
|
+
const notes = [];
|
|
228
|
+
if (kind === "dataset_info") {
|
|
229
|
+
let v = pyStr(params.dataset_id === undefined ? "None" : params.dataset_id);
|
|
230
|
+
if (!/^\d+$/.test(v)) {
|
|
231
|
+
const e = entry(v) || entry((req.params || {}).dataset_id ?? "");
|
|
232
|
+
if (!e)
|
|
233
|
+
throw new KeyErr(`dataset_id ${pyRepr(v)} is not a numeric id or catalog alias`);
|
|
234
|
+
v = pyStr(e.dataset_id);
|
|
235
|
+
}
|
|
236
|
+
return [{ ...ep, path: `/datasets/${v}`, params: {} }, {}, notes];
|
|
237
|
+
}
|
|
238
|
+
if (kind !== "query")
|
|
239
|
+
return [ep, params, notes];
|
|
240
|
+
const raw = (req.params || {}).dataset || req.dataset;
|
|
241
|
+
const e = entry(raw) || entry(params.dataset ?? "");
|
|
242
|
+
const ds = e ? String(e.id) : pyStr(params.dataset === undefined ? "None" : params.dataset);
|
|
243
|
+
delete params.dataset;
|
|
244
|
+
const i = ds.indexOf("/");
|
|
245
|
+
const [agency, endpoint] = i < 0 ? [ds, ""] : [ds.slice(0, i), ds.slice(i + 1)];
|
|
246
|
+
if ("sort_by" in params && blank(params.sort_by)) { // explicit: the API's own order
|
|
247
|
+
delete params.sort_by;
|
|
248
|
+
delete params.sort;
|
|
249
|
+
}
|
|
250
|
+
else if (e && e.sort_by && !params.sort_by) {
|
|
251
|
+
params.sort_by = e.sort_by;
|
|
252
|
+
if (!("sort" in params))
|
|
253
|
+
params.sort = "desc";
|
|
254
|
+
}
|
|
255
|
+
else if (params.sort_by) {
|
|
256
|
+
if (!("sort" in params))
|
|
257
|
+
params.sort = "desc";
|
|
258
|
+
}
|
|
259
|
+
const fo = buildFilter(params, e);
|
|
260
|
+
if (fo) {
|
|
261
|
+
params.filter_object = fo;
|
|
262
|
+
notes.push(`${req.name}: filter_object ${fo}`);
|
|
263
|
+
}
|
|
264
|
+
if (e && e.columns && Object.keys(e.columns).length) {
|
|
265
|
+
notes.push(`${req.name}: column meanings: ` + Object.entries(e.columns).map(([k, v]) => `${k} = ${pyStr(v)}`).join("; "));
|
|
266
|
+
}
|
|
267
|
+
return [{ ...ep, path: `/get/${pyQuote(agency)}/${pyQuote(endpoint)}/json`, params: {} }, params, notes];
|
|
268
|
+
}
|
|
269
|
+
/** Client wrapper: an empty 200/204 body means no rows. */
|
|
270
|
+
function emptyOk(c) {
|
|
271
|
+
const w = Object.create(c);
|
|
272
|
+
w.request = async (method, p = "", params, o = {}) => {
|
|
273
|
+
const r = await c.request(method, p, params, { jsonBody: o.jsonBody, okStatuses: [200, 204] });
|
|
274
|
+
if (r.status === 204 || !r.body.toString("latin1").trim())
|
|
275
|
+
r.body = Buffer.from('{"data": [], "datasets": []}');
|
|
276
|
+
return r;
|
|
277
|
+
};
|
|
278
|
+
return w;
|
|
279
|
+
}
|
|
280
|
+
async function fetch(spec, pack, _route) {
|
|
281
|
+
// the field dictionary first (keyless): unknown names are refused before any data request
|
|
282
|
+
const probs0 = [];
|
|
283
|
+
for (const r of spec.requests) {
|
|
284
|
+
const fc = fieldCheck(r);
|
|
285
|
+
if (!fc)
|
|
286
|
+
continue;
|
|
287
|
+
const prob = fieldProblem(r, fc[0], fc[1], await datasetFields(pack, fc[0].dataset_id));
|
|
288
|
+
if (prob)
|
|
289
|
+
probs0.push(prob);
|
|
290
|
+
}
|
|
291
|
+
if (probs0.length)
|
|
292
|
+
throw new DataError(`${pack.id}: the request has problems: ${probs0.join("; ")}`, "spec");
|
|
293
|
+
const client = emptyOk(gr.clientFor(pack));
|
|
294
|
+
const tables = {};
|
|
295
|
+
let notes = [];
|
|
296
|
+
const files = new Map();
|
|
297
|
+
for (const r of spec.requests) {
|
|
298
|
+
const [ep, params, probs] = gr.resolve(pack, r);
|
|
299
|
+
if (probs.length)
|
|
300
|
+
throw new DataError(probs.join("; "), "bad_request");
|
|
301
|
+
if (r.endpoint && OUI.includes(r.endpoint)) {
|
|
302
|
+
const [rows, n] = await fetchOui(pack, r, ep, params, files);
|
|
303
|
+
tables[r.name] = rows;
|
|
304
|
+
notes = notes.concat(n);
|
|
305
|
+
continue;
|
|
306
|
+
}
|
|
307
|
+
let ep2, q, n;
|
|
308
|
+
try {
|
|
309
|
+
[ep2, q, n] = prepare(r, ep, params);
|
|
310
|
+
}
|
|
311
|
+
catch (e) {
|
|
312
|
+
if (e instanceof KeyErr || e instanceof SyntaxError)
|
|
313
|
+
throw new DataError(`request ${pyRepr(r.name)}: ${e.message}`, "bad_request");
|
|
314
|
+
throw e;
|
|
315
|
+
}
|
|
316
|
+
const [rows, meta] = await gr.fetchRequest(client, ep2, q, r.max_rows);
|
|
317
|
+
if (meta.truncated)
|
|
318
|
+
notes.push(`${r.name}: stopped at ${fmt(rows.length)} rows (max_rows); narrow the filters or raise max_rows`);
|
|
319
|
+
if (!rows.length && !r.allow_empty) {
|
|
320
|
+
throw new DataError(`request ${pyRepr(r.name)} returned no rows: DOL 'eq' and 'like' are exact and ` +
|
|
321
|
+
"case-sensitive; check the values (or use search) and the field names (dataset_info)", "bad_request");
|
|
322
|
+
}
|
|
323
|
+
tables[r.name] = rows;
|
|
324
|
+
notes = notes.concat(n);
|
|
325
|
+
}
|
|
326
|
+
return [tables, notes];
|
|
327
|
+
}
|
|
328
|
+
/** urllib.parse.urlencode(q) without doseq: a list is sent as its Python str(). */
|
|
329
|
+
function urlencodeNoSeq(q) {
|
|
330
|
+
return pyUrlencode(Object.entries(q).map(([k, v]) => [k, Array.isArray(v) || isObj(v) ? pyRepr(v) : v]));
|
|
331
|
+
}
|
|
332
|
+
function manualItems(spec, pack) {
|
|
333
|
+
const base = String(pack.endpoints.base_url).replace(/\/+$/, "");
|
|
334
|
+
const items = [];
|
|
335
|
+
for (const r of spec.requests || []) {
|
|
336
|
+
const [ep, params, probs] = gr.resolve(pack, r);
|
|
337
|
+
if (probs.length)
|
|
338
|
+
continue;
|
|
339
|
+
if (r.endpoint && OUI.includes(r.endpoint)) {
|
|
340
|
+
items.push({ name: `${r.name}#1`, request: r.name, url: ep.path, filename: String(ep.path).split("/").pop(), needs_key: false,
|
|
341
|
+
method: "GET", body: null, note: "the whole file (ar2112.csv is about 23 MB); the ingest filters it as the spec says" });
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
let ep2, q;
|
|
345
|
+
try {
|
|
346
|
+
[ep2, q] = prepare(r, ep, params);
|
|
347
|
+
}
|
|
348
|
+
catch (e) {
|
|
349
|
+
if (e instanceof KeyErr || e instanceof SyntaxError)
|
|
350
|
+
continue;
|
|
351
|
+
throw e;
|
|
352
|
+
}
|
|
353
|
+
const keyed = !KEYLESS.includes(r.endpoint || "query");
|
|
354
|
+
const pg = ep2.paging || { type: "none" };
|
|
355
|
+
const size = Number(pg.size || 1000);
|
|
356
|
+
let pages = 1;
|
|
357
|
+
if (pg.type === "offset")
|
|
358
|
+
pages = Math.min(25, Math.max(1, Math.ceil(Number(r.max_rows || ep2.max_rows || size) / size)));
|
|
359
|
+
for (let i = 0; i < pages; i++) {
|
|
360
|
+
const qq = { ...q };
|
|
361
|
+
if (pg.type === "offset")
|
|
362
|
+
[qq[pg.limit_param], qq[pg.offset_param]] = [size, i * size];
|
|
363
|
+
let url = `${base}${ep2.path}` + (Object.keys(qq).length ? "?" + urlencodeNoSeq(qq) : "");
|
|
364
|
+
if (keyed)
|
|
365
|
+
url += (url.includes("?") ? "&" : "?") + "X-API-KEY={API_KEY}";
|
|
366
|
+
items.push({ name: `${r.name}#${i + 1}`, request: r.name, url, filename: pages === 1 ? `${r.name}.json` : `${r.name}_${i + 1}.json`,
|
|
367
|
+
needs_key: keyed, method: "GET", body: null,
|
|
368
|
+
note: pages === 1 ? "" : `page ${i + 1} of up to ${pages}; stop when "data" comes back empty` });
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
return items;
|
|
372
|
+
}
|
|
373
|
+
function parseUpload(pack, req, files) {
|
|
374
|
+
const [ep, params] = gr.resolve(pack, req);
|
|
375
|
+
const sorted = [...files].sort(cmp);
|
|
376
|
+
if (req.endpoint && OUI.includes(req.endpoint)) {
|
|
377
|
+
const p = sorted[sorted.length - 1];
|
|
378
|
+
return tidyOui(req, ep, params, fs.readFileSync(p), path.basename(p))[0];
|
|
379
|
+
}
|
|
380
|
+
let rows = [];
|
|
381
|
+
for (const p of sorted) {
|
|
382
|
+
const data = fs.readFileSync(p);
|
|
383
|
+
if (!data.toString("latin1").trim())
|
|
384
|
+
continue;
|
|
385
|
+
const doc = parsePyJson(data.toString("utf8").replace(/^\ufeff/, ""));
|
|
386
|
+
rows = rows.concat(gr.dig(doc, ep.records ?? "data") || []);
|
|
387
|
+
}
|
|
388
|
+
return gr.clean(rows, ep);
|
|
389
|
+
}
|
|
390
|
+
// ---------------------------------------------------------------------- OUI unemployment insurance files
|
|
391
|
+
class OuiError extends Error {
|
|
392
|
+
}
|
|
393
|
+
function usps(x) {
|
|
394
|
+
const s = pyStr(x).trim();
|
|
395
|
+
if (/^[A-Za-z]{2}$/.test(s))
|
|
396
|
+
return s.toUpperCase();
|
|
397
|
+
try {
|
|
398
|
+
return state(s, "usps");
|
|
399
|
+
}
|
|
400
|
+
catch (e) {
|
|
401
|
+
if (e instanceof StateError)
|
|
402
|
+
throw new OuiError(`unknown state ${pyRepr(x)}: use a two-letter code (WY)`);
|
|
403
|
+
throw e;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
function states(v) {
|
|
407
|
+
const vals = typeof v === "string" ? [v] : Array.isArray(v) ? v : [];
|
|
408
|
+
return new Set(vals.filter((x) => pyStr(x).trim()).map(usps));
|
|
409
|
+
}
|
|
410
|
+
function bound(v, end) {
|
|
411
|
+
if (blank(v))
|
|
412
|
+
return null;
|
|
413
|
+
const s = pyStr(v).trim();
|
|
414
|
+
if (/^\d{4}-\d{2}-\d{2}$/.test(s))
|
|
415
|
+
return s;
|
|
416
|
+
if (/^\d{4}-\d{2}$/.test(s))
|
|
417
|
+
return s + (end ? "-31" : "-01");
|
|
418
|
+
if (/^\d{4}$/.test(s))
|
|
419
|
+
return s + (end ? "-12-31" : "-01-01");
|
|
420
|
+
throw new OuiError(`${end ? "end" : "start"} ${pyRepr(v)}: use YYYY-MM or YYYY-MM-DD`);
|
|
421
|
+
}
|
|
422
|
+
function fieldList(f) {
|
|
423
|
+
return Array.isArray(f) ? f.map((x) => pyStr(x)) : pyStr(f).split(",");
|
|
424
|
+
}
|
|
425
|
+
function checkOui(r) {
|
|
426
|
+
const p = r.params || {}, name = pyRepr(r.name ?? null), probs = [];
|
|
427
|
+
if (r.endpoint !== "ui_trust_fund")
|
|
428
|
+
return probs;
|
|
429
|
+
try {
|
|
430
|
+
states(p.state);
|
|
431
|
+
}
|
|
432
|
+
catch (e) {
|
|
433
|
+
if (!(e instanceof OuiError))
|
|
434
|
+
throw e;
|
|
435
|
+
probs.push(`request ${name}: ${e.message}`);
|
|
436
|
+
}
|
|
437
|
+
for (const k of ["start", "end"]) {
|
|
438
|
+
try {
|
|
439
|
+
bound(p[k], k === "end");
|
|
440
|
+
}
|
|
441
|
+
catch (e) {
|
|
442
|
+
if (!(e instanceof OuiError))
|
|
443
|
+
throw e;
|
|
444
|
+
probs.push(`request ${name}: ${e.message}`);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
if (p.fields) {
|
|
448
|
+
const cols = eps().endpoints.ui_trust_fund.columns;
|
|
449
|
+
const vals = new Set(Object.values(cols));
|
|
450
|
+
const bad = fieldList(p.fields).map((x) => x.trim()).filter((x) => x && !(x in cols) && !vals.has(x) && x !== "st" && x !== "rptdate");
|
|
451
|
+
if (bad.length) {
|
|
452
|
+
probs.push(`request ${name}: unknown field(s) ${bad.map(pyRepr).join(", ")}: ui_trust_fund columns are ` +
|
|
453
|
+
"<line item>_<account> names such as balance_close_utf, dep_ui_contributions_total, " +
|
|
454
|
+
"dis_ui_benefits_total, dep_treasury_interest_utf (endpoints.json 'columns' lists all), or the file's c-codes");
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
return probs;
|
|
458
|
+
}
|
|
459
|
+
/** One OUI file (downloaded once per fetch, keyless) filtered and renamed as the request says. */
|
|
460
|
+
async function fetchOui(pack, r, ep, params, files) {
|
|
461
|
+
const url = ep.path;
|
|
462
|
+
if (!files.has(url)) {
|
|
463
|
+
const L = pack.policy.limits;
|
|
464
|
+
const c = new Client(OUI_BASE, { name: "dol (oui.doleta.gov)", auth: {}, rps: L.rps || 2, burst: L.burst || 4, timeout: 120,
|
|
465
|
+
deadline: 600 });
|
|
466
|
+
try {
|
|
467
|
+
files.set(url, (await c.request("GET", url)).body);
|
|
468
|
+
}
|
|
469
|
+
catch (e) {
|
|
470
|
+
if (!(e instanceof DataError))
|
|
471
|
+
throw e;
|
|
472
|
+
throw new DataError(`request ${pyRepr(r.name)} (${r.endpoint}): ${e.message}`, e.kind, e.status);
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
let out;
|
|
476
|
+
try {
|
|
477
|
+
out = tidyOui(r, ep, params, files.get(url), url.split("/").pop());
|
|
478
|
+
}
|
|
479
|
+
catch (e) {
|
|
480
|
+
if (e instanceof OuiError)
|
|
481
|
+
throw new DataError(e.message, "bad_request");
|
|
482
|
+
throw e;
|
|
483
|
+
}
|
|
484
|
+
if (!out[0].length && !r.allow_empty) {
|
|
485
|
+
throw new DataError(`request ${pyRepr(r.name)} returned no rows: check state and the month range (the file runs ` +
|
|
486
|
+
"from 1979-01 to the latest month each state has reported)", "bad_request");
|
|
487
|
+
}
|
|
488
|
+
return out;
|
|
489
|
+
}
|
|
490
|
+
const TRUE = (v) => v === true || v === 1 || v === "true" || v === "True" || v === "1";
|
|
491
|
+
export function tidyOui(req, ep, params, body, fname) {
|
|
492
|
+
if (req.endpoint === "ui_data_summary")
|
|
493
|
+
return dataSummary(req, ep, params, body);
|
|
494
|
+
const text = new TextDecoder("utf-8").decode(body);
|
|
495
|
+
const first = text.trim() ? text.slice(0, 200).split(/\r\n|[\n\r\v\f\x1c\x1d\x1e\x85\u2028\u2029]/)[0] : "";
|
|
496
|
+
if ((!first.includes('"st"') && !first.startsWith("st,")) || !first.includes("rptdate")) {
|
|
497
|
+
throw new DataError(`${fname}: expected the ETA 2112 CSV (header st, rptdate, c1, ...), got something else (an HTML page?)`, "api");
|
|
498
|
+
}
|
|
499
|
+
const cols = ep.columns || {};
|
|
500
|
+
const sts = states(params.state);
|
|
501
|
+
const lo = bound(params.start, false), hi = bound(params.end, true);
|
|
502
|
+
let want = null;
|
|
503
|
+
if (params.fields)
|
|
504
|
+
want = new Set(fieldList(params.fields).map((x) => cols[x.trim()] ?? x.trim()));
|
|
505
|
+
const missing = new Set(ep.missing || [""]);
|
|
506
|
+
let rows = [];
|
|
507
|
+
for (const rec of readCsv(text)) {
|
|
508
|
+
const st = (rec.st ?? ""), d = (rec.rptdate ?? "");
|
|
509
|
+
if ((sts.size && !sts.has(st)) || (lo && d < lo) || (hi && d > hi))
|
|
510
|
+
continue;
|
|
511
|
+
const out = { st, rptdate: d };
|
|
512
|
+
for (const [k, v] of Object.entries(rec)) {
|
|
513
|
+
if (k === "st" || k === "rptdate")
|
|
514
|
+
continue;
|
|
515
|
+
const name = cols[k] ?? k;
|
|
516
|
+
if (want === null || want.has(name))
|
|
517
|
+
out[name] = v === null || v === undefined || missing.has(String(v).trim()) ? "" : v;
|
|
518
|
+
}
|
|
519
|
+
rows.push(out);
|
|
520
|
+
}
|
|
521
|
+
const notes = [];
|
|
522
|
+
if (TRUE(params.latest)) {
|
|
523
|
+
const last = new Map();
|
|
524
|
+
for (const x of rows) {
|
|
525
|
+
const cur = last.get(x.st) ?? "";
|
|
526
|
+
last.set(x.st, x.rptdate > cur ? x.rptdate : cur);
|
|
527
|
+
}
|
|
528
|
+
rows = rows.filter((x) => x.rptdate === last.get(x.st));
|
|
529
|
+
const top = [...last.values()].reduce((m, v) => (v > m ? v : m), "");
|
|
530
|
+
const lag = [...last].filter(([, d]) => d !== top).map(([s, d]) => `${s} ${d.slice(0, 7)}`).sort(cmp);
|
|
531
|
+
if (lag.length) {
|
|
532
|
+
notes.push(`${req.name}: latest month per state; the newest month in the file is ${top.slice(0, 7)}, these ` +
|
|
533
|
+
`states have not reported it yet: ${lag.join(", ")}`);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
if (rows.length) {
|
|
537
|
+
const ds = rows.map((x) => x.rptdate);
|
|
538
|
+
const mn = ds.reduce((m, v) => (v < m ? v : m)), mx = ds.reduce((m, v) => (v > m ? v : m));
|
|
539
|
+
notes.push(`${req.name}: ETA 2112 (${fname}), ${fmt(rows.length)} state-months ${mn.slice(0, 7)} to ${mx.slice(0, 7)}; amounts in ` +
|
|
540
|
+
"dollars; the state's trust fund balance at month end is balance_close_utf (c87); " +
|
|
541
|
+
"<item>_total is the sum of the clearing, utf and bpa accounts (column C of the report)");
|
|
542
|
+
}
|
|
543
|
+
return [rows, notes];
|
|
544
|
+
}
|
|
545
|
+
function summaryCol(h) {
|
|
546
|
+
let s = h.replaceAll("*", "").trim();
|
|
547
|
+
let unit = "";
|
|
548
|
+
if (s.includes("(000)"))
|
|
549
|
+
[s, unit] = [s.replaceAll("(000)", ""), "thousands"];
|
|
550
|
+
s = s.replaceAll("(%)", " pct").replaceAll("%", " pct ");
|
|
551
|
+
const slug = s.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "");
|
|
552
|
+
return [slug, unit].filter(Boolean).join("_");
|
|
553
|
+
}
|
|
554
|
+
/** A number as the workbook stores it ('319139.59999999998') in its shortest form ('319139.6'). Text is kept as is. */
|
|
555
|
+
function num(v) {
|
|
556
|
+
if (!/^-?\d+(\.\d+)?([eE][-+]?\d+)?$/.test(v || ""))
|
|
557
|
+
return v;
|
|
558
|
+
const f = Number(v);
|
|
559
|
+
if (Number.isInteger(f) && !v.toLowerCase().includes("e") && !v.includes("."))
|
|
560
|
+
return BigInt(f).toString();
|
|
561
|
+
return pyFloatRepr(f);
|
|
562
|
+
}
|
|
563
|
+
/** SummaryTables.xlsx, one sheet: the quarter from the title row, the header found by its first cell. */
|
|
564
|
+
function dataSummary(req, ep, params, body) {
|
|
565
|
+
const e2 = gr.withRecordParams(ep, params);
|
|
566
|
+
const rows = gr.clean(sheetRecords(e2, body), e2);
|
|
567
|
+
const keys = [];
|
|
568
|
+
for (const r of rows)
|
|
569
|
+
for (const k of Object.keys(r))
|
|
570
|
+
if (!keys.includes(k))
|
|
571
|
+
keys.push(k);
|
|
572
|
+
const head = keys.map((k) => (/^col\d+$/.test(k) ? "" : k));
|
|
573
|
+
const grid = [head, ...rows.map((r) => keys.map((k) => pyStr(k in r ? r[k] : "").trim()))];
|
|
574
|
+
const cyq = head.find((c) => /^\d{4}\.[1-4]$/.test(c)) ?? "";
|
|
575
|
+
const li = grid.findIndex((g) => g.length && g[0] === "State");
|
|
576
|
+
if (li < 0) {
|
|
577
|
+
throw new DataError(`SummaryTables.xlsx sheet ${pyRepr(e2.sheet ?? null)}: the layout changed (no 'State' header row); ` +
|
|
578
|
+
"open the file and check", "api");
|
|
579
|
+
}
|
|
580
|
+
const names = ["state", ...grid[li].slice(1).filter(Boolean).map(summaryCol)];
|
|
581
|
+
const data = [];
|
|
582
|
+
const foot = [];
|
|
583
|
+
for (const g of grid.slice(li + 1)) {
|
|
584
|
+
if (!g.slice(1, names.length).some(Boolean)) {
|
|
585
|
+
if (g.length && g[0])
|
|
586
|
+
foot.push(g[0]);
|
|
587
|
+
continue;
|
|
588
|
+
}
|
|
589
|
+
const vals = [g[0], ...g.slice(1, names.length).map(num)];
|
|
590
|
+
const d = {};
|
|
591
|
+
for (let i = 0; i < Math.min(names.length, vals.length); i++)
|
|
592
|
+
d[names[i]] = vals[i];
|
|
593
|
+
data.push(d);
|
|
594
|
+
}
|
|
595
|
+
const starred = foot.find((f) => f.startsWith("*"));
|
|
596
|
+
const star = starred !== undefined ? starred.replace(/^\*+/, "").trim() : "";
|
|
597
|
+
let quarterEnd = "";
|
|
598
|
+
if (cyq) {
|
|
599
|
+
const [y, q] = cyq.split(".");
|
|
600
|
+
quarterEnd = `${y}-${["03-31", "06-30", "09-30", "12-31"][Number(q) - 1]}`;
|
|
601
|
+
}
|
|
602
|
+
const out = data.map((d) => {
|
|
603
|
+
let name = d.state;
|
|
604
|
+
const flagged = name.endsWith("*");
|
|
605
|
+
name = name.replace(/\*+$/, "").trim();
|
|
606
|
+
let code;
|
|
607
|
+
try {
|
|
608
|
+
code = name === "United States" ? "US" : usps(name === "Virgin Islands" ? "U.S. Virgin Islands" : name);
|
|
609
|
+
}
|
|
610
|
+
catch (e) {
|
|
611
|
+
if (!(e instanceof OuiError))
|
|
612
|
+
throw e;
|
|
613
|
+
code = "";
|
|
614
|
+
}
|
|
615
|
+
const rest = Object.fromEntries(Object.entries(d).filter(([k]) => k !== "state"));
|
|
616
|
+
return { cyq, quarter_end: quarterEnd, state: name, st: code, ...rest, note: flagged ? star : "" };
|
|
617
|
+
});
|
|
618
|
+
const notes = [`${req.name}: UI Data Summary, sheet ${pyStr(e2.sheet ?? "None")}, calendar-year quarter ${cyq}` +
|
|
619
|
+
(foot.length ? ": " + foot.join(" ") : "")];
|
|
620
|
+
return [out, notes];
|
|
621
|
+
}
|
|
622
|
+
function search(query, pack) {
|
|
623
|
+
const words = (query.toLowerCase().match(/[a-z0-9]+/g) || []).filter((w) => w.length > 1);
|
|
624
|
+
const scored = [];
|
|
625
|
+
for (const h0 of gr.searchCatalog(pack, query, 500)) {
|
|
626
|
+
const h = Object.fromEntries(Object.entries(h0).filter(([k]) => k !== "columns"));
|
|
627
|
+
const text = Object.values(h).map((v) => pyStr(v)).join(" ").toLowerCase();
|
|
628
|
+
scored.push([words.filter((w) => text.includes(w)).length, h]);
|
|
629
|
+
}
|
|
630
|
+
return scored.sort((a, b) => b[0] - a[0]).map(([, h]) => h).slice(0, 20);
|
|
631
|
+
}
|
|
632
|
+
const adapter = { keyOnlyNeeds, checkSpec, fetch, manualItems, parseUpload, search, tidyOui, fetchOui, prepare };
|
|
633
|
+
export default adapter;
|
|
634
|
+
//# sourceMappingURL=dol.js.map
|