data-analyst-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -2
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +716 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +24 -0
- package/dist/report/scan_secrets.js +265 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +247 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +54 -4
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +170 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,696 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* usda-outlook adapter: WASDE monthly CSVs (USDA OCE/WAOB), ERS farm income releases and the USDA Agricultural
|
|
3
|
+
* Baseline (ERS). Published files only, no key (verified 2026-10-30).
|
|
4
|
+
*
|
|
5
|
+
* - WASDE: one long-format CSV per report month at a predictable name,
|
|
6
|
+
* https://www.usda.gov/sites/default/files/documents/oce-wasde-report-data-YYYY-MM.csv (2021-01 on; HEAD answers
|
|
7
|
+
* Content-Length 0, so it is fetched with GET). Months with no CSV at that name: 2025-10 (no October 2025 report,
|
|
8
|
+
* lapse in federal funding) and 2026-05 / 2026-06 (reissued as corrected "v2" reports; HTTP 404 at the name).
|
|
9
|
+
* "latest" = the newest report date on the ESMIS publication page (esmis.nal.usda.gov), stepping back a month when
|
|
10
|
+
* its CSV is not posted yet (USDA posts it the day after the release). The usda.gov WASDE web pages answer scripts
|
|
11
|
+
* with HTTP 403 (Akamai "Access Denied"); the files do not, and the pack never reads those pages.
|
|
12
|
+
* - Farm income (ERS): the data-files page links one ZIP per release whose media id changes ("September 3, 2026
|
|
13
|
+
* release" -> /media/29517/september-3-2026-release.zip), so links are scraped by their text. The ZIP holds one
|
|
14
|
+
* CSV of 111 MB (cp1252: unit_desc is padded with non-breaking spaces), read row by row and filtered. Releases up
|
|
15
|
+
* to November 2017 are plain CSVs. The same page links the one-page summary workbook ("U.S. farm sector financial
|
|
16
|
+
* indicators, 2019-2026F") and the forecast archive ZIP.
|
|
17
|
+
* - Baseline (ERS): the Agricultural Baseline Database page links each edition's ZIP under "Most Recent Early-Release",
|
|
18
|
+
* "Most Recent Full-Release" and "Historic Full-Release" headings; both 2035 ZIPs share a file name and differ by
|
|
19
|
+
* media id, so the heading decides which is which.
|
|
20
|
+
*
|
|
21
|
+
* Values are as published, except: unit_desc is stripped of the padding spaces; .xlsx floats are written in their
|
|
22
|
+
* shortest exact form (369.31670200000002 -> 369.316702). Added columns: farm_income `release_date` (from the
|
|
23
|
+
* file's PublicationDate) and `estimate_type` (forecast/estimate by ERS's stated rule), farm_income_summary
|
|
24
|
+
* `section` and `unit`.
|
|
25
|
+
*/
|
|
26
|
+
import fs from "node:fs";
|
|
27
|
+
import path from "node:path";
|
|
28
|
+
import * as gr from "../core/generic-rest.js";
|
|
29
|
+
import { DataError } from "../core/http.js";
|
|
30
|
+
import { unzip } from "../core/tabular.js";
|
|
31
|
+
import { asTable, baselineEditions, basename, blank, cp1252, dateOf, day, dictReader, ERS, fiwLink, fiwReleases, fnmatch, gridRecords, noNone, NUM, num, orderedSheets, pickEdition, pickRelease, pstr, pyInt, repr, strip, utf8OrCp1252, words, } from "./_usda-outlook_read.js";
|
|
32
|
+
const WASDE_CSV = (ym) => `https://www.usda.gov/sites/default/files/documents/oce-wasde-report-data-${ym}.csv`;
|
|
33
|
+
const WASDE_HIST = "https://www.usda.gov/oce/commodity-markets/wasde/historical-wasde-report-data";
|
|
34
|
+
const ESMIS = "https://esmis.nal.usda.gov/publication/world-agricultural-supply-and-demand-estimates";
|
|
35
|
+
const FIW_PAGE = ERS + "/data-products/farm-income-and-wealth-statistics/data-files-us-and-state-level-farm-income-and-wealth-statistics";
|
|
36
|
+
const BASE_PAGE = ERS + "/data-products/agricultural-baseline-database";
|
|
37
|
+
const WASDE_FIRST = "2021-01";
|
|
38
|
+
const NO_REPORT = { "2025-10": "no October 2025 WASDE was issued (lapse in federal funding)" };
|
|
39
|
+
// known HTTP 404s at the usual CSV name (verified 2026-10-30); still requested, in case USDA posts them later
|
|
40
|
+
const WASDE_GAPS = { ...NO_REPORT,
|
|
41
|
+
"2026-05": "the May 2026 report was reissued as a corrected v2 report; no CSV at the usual name",
|
|
42
|
+
"2026-06": "the June 2026 report was reissued as a corrected v2 report; no CSV at the usual name" };
|
|
43
|
+
const WASDE_MAX_MONTHS = 72;
|
|
44
|
+
const WASDE_COLS = ["WasdeNumber", "ReportDate", "ReportTitle", "Attribute", "Commodity", "Region", "MarketYear",
|
|
45
|
+
"ProjEstFlag", "Value", "Unit"];
|
|
46
|
+
const FI_FIRST = "2014-08";
|
|
47
|
+
const YM = /^\d{4}-(0[1-9]|1[0-2])$/;
|
|
48
|
+
const has = (o, k) => Object.prototype.hasOwnProperty.call(o, k);
|
|
49
|
+
const fmtInt = (n) => n.toLocaleString("en-US");
|
|
50
|
+
// ---------------------------------------------------------------------- small helpers
|
|
51
|
+
const norm = (v) => words(pstr(v || "")).toLowerCase();
|
|
52
|
+
/** ProjEstFlag 'Proj.', '(Proj.)', 'Est.', '(Est.)', '' -> proj / est / none. */
|
|
53
|
+
export function flag(v) {
|
|
54
|
+
return pstr(v || "").replace(/[().\s]/g, "").toLowerCase() || "none";
|
|
55
|
+
}
|
|
56
|
+
function list(v) {
|
|
57
|
+
if (blank(v))
|
|
58
|
+
return [];
|
|
59
|
+
return (Array.isArray(v) ? v : [v]).filter((x) => !blank(x));
|
|
60
|
+
}
|
|
61
|
+
function match(value, wanted) {
|
|
62
|
+
return !wanted.length || new Set(wanted.map(norm)).has(norm(value));
|
|
63
|
+
}
|
|
64
|
+
function ymAdd(ym, k) {
|
|
65
|
+
const y = parseInt(ym.slice(0, 4), 10), m = parseInt(ym.slice(5, 7), 10) - 1 + k;
|
|
66
|
+
return `${String(y + Math.floor(m / 12)).padStart(4, "0")}-${String((((m % 12) + 12) % 12) + 1).padStart(2, "0")}`;
|
|
67
|
+
}
|
|
68
|
+
function ymRange(a, b) {
|
|
69
|
+
const out = [];
|
|
70
|
+
let cur = a;
|
|
71
|
+
while (cur <= b && out.length <= WASDE_MAX_MONTHS) {
|
|
72
|
+
out.push(cur);
|
|
73
|
+
cur = ymAdd(cur, 1);
|
|
74
|
+
}
|
|
75
|
+
return out;
|
|
76
|
+
}
|
|
77
|
+
/** Python _years: [lo, hi]; throws (as int() does) on a value that is not a whole number. */
|
|
78
|
+
function years(p) {
|
|
79
|
+
const int = (v) => {
|
|
80
|
+
const n = pyInt(v);
|
|
81
|
+
if (n === null)
|
|
82
|
+
throw new TypeError(`invalid literal for int(): ${repr(v)}`);
|
|
83
|
+
return n;
|
|
84
|
+
};
|
|
85
|
+
if (!blank(p.year))
|
|
86
|
+
return [int(p.year), int(p.year)];
|
|
87
|
+
return [!blank(p.start_year) ? int(p.start_year) : null, !blank(p.end_year) ? int(p.end_year) : null];
|
|
88
|
+
}
|
|
89
|
+
function inYears(y, lo, hi) {
|
|
90
|
+
const n = pyInt(pstr(y).slice(0, 4));
|
|
91
|
+
if (n === null)
|
|
92
|
+
return lo === null && hi === null;
|
|
93
|
+
return (lo === null || n >= lo) && (hi === null || n <= hi);
|
|
94
|
+
}
|
|
95
|
+
/** The index pages, each read once per fetch. */
|
|
96
|
+
class Site {
|
|
97
|
+
c;
|
|
98
|
+
pages = new Map();
|
|
99
|
+
constructor(pack) {
|
|
100
|
+
this.c = gr.clientFor(pack);
|
|
101
|
+
}
|
|
102
|
+
async page(url) {
|
|
103
|
+
if (!this.pages.has(url))
|
|
104
|
+
this.pages.set(url, (await this.c.get(url)).text());
|
|
105
|
+
return this.pages.get(url);
|
|
106
|
+
}
|
|
107
|
+
async wasdeLatest() {
|
|
108
|
+
const body = await this.page(ESMIS);
|
|
109
|
+
const ds = [...body.matchAll(/\/publication\/world-agricultural-supply-and-demand-estimates\/(\d{4}-\d{2})-\d{2}/g)].map((m) => m[1]);
|
|
110
|
+
if (!ds.length)
|
|
111
|
+
throw new DataError("no WASDE release dates found on the ESMIS publication page", "api");
|
|
112
|
+
return ds.reduce((a, b) => (b > a ? b : a));
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
// ---------------------------------------------------------------------- WASDE
|
|
116
|
+
function wasdeRows(data, label) {
|
|
117
|
+
const text = new TextDecoder("utf-8").decode(data).replace(/^/, "");
|
|
118
|
+
const [names, rows] = dictReader(text);
|
|
119
|
+
const miss = WASDE_COLS.filter((c) => !names.includes(c));
|
|
120
|
+
if (miss.length) {
|
|
121
|
+
throw new DataError(`${label} is not a WASDE data CSV (no ${miss.join(", ")} column; an HTML page in its place?)`, "api");
|
|
122
|
+
}
|
|
123
|
+
return [...rows].map(noNone);
|
|
124
|
+
}
|
|
125
|
+
const sget = (r, k) => (r.has(k) ? r.get(k) : null);
|
|
126
|
+
function wasdeFilter(rows, p) {
|
|
127
|
+
const keepRel = ["true", "1", "yes"].includes(pstr(p.include_reliability ?? "").toLowerCase());
|
|
128
|
+
const pe = list(p.proj_est);
|
|
129
|
+
const peSet = new Set(pe.map((x) => pstr(x).toLowerCase()));
|
|
130
|
+
return rows.filter((r) => {
|
|
131
|
+
if (!keepRel && String(r.get("ReportTitle") ?? "").toLowerCase().startsWith("reliability"))
|
|
132
|
+
return false;
|
|
133
|
+
if (!(match(sget(r, "Commodity"), list(p.commodity)) && match(sget(r, "Region"), list(p.region)) &&
|
|
134
|
+
match(sget(r, "Attribute"), list(p.attribute)) && match(sget(r, "ReportTitle"), list(p.report_title)) &&
|
|
135
|
+
match(sget(r, "MarketYear"), list(p.market_year))))
|
|
136
|
+
return false;
|
|
137
|
+
if (pe.length && !peSet.has(flag(sget(r, "ProjEstFlag"))))
|
|
138
|
+
return false;
|
|
139
|
+
return true;
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
const cmp = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
143
|
+
function wasdeNotes(name, rows) {
|
|
144
|
+
const notes = [];
|
|
145
|
+
const tables = [...new Set(rows.map((r) => String(r.has("ReportTitle") ? r.get("ReportTitle") : "")))].sort(cmp);
|
|
146
|
+
const units = [...new Set(rows.map((r) => String(r.has("Unit") ? r.get("Unit") : "")))].sort(cmp);
|
|
147
|
+
if (tables.length > 1 && rows.length <= 5000) {
|
|
148
|
+
const groups = new Map();
|
|
149
|
+
for (const r of rows) {
|
|
150
|
+
const key = ["Commodity", "Region", "Attribute", "MarketYear", "WasdeNumber", "AnnualQuarterFlag"].map((k) => sget(r, k));
|
|
151
|
+
const k = JSON.stringify(key);
|
|
152
|
+
if (!groups.has(k))
|
|
153
|
+
groups.set(k, [key, new Set()]);
|
|
154
|
+
groups.get(k)[1].add(sget(r, "ReportTitle"));
|
|
155
|
+
}
|
|
156
|
+
const dup = [...groups.values()].filter(([, v]) => v.size > 1);
|
|
157
|
+
if (dup.length) {
|
|
158
|
+
const [c, rg, a] = dup[0][0].map(pstr);
|
|
159
|
+
notes.push(`${name}: the same item comes from ${tables.length} report tables (${tables.slice(0, 4).join("; ")}` +
|
|
160
|
+
`${tables.length > 4 ? " ..." : ""}), e.g. ${c} / ${rg} / ${a}; units: ${units.join(", ")}. ` +
|
|
161
|
+
"Do not add or compare across tables: set report_title (U.S. tables are in U.S. units such " +
|
|
162
|
+
"as million bushels, world tables in million metric tons).");
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
if (rows.some((r) => flag(sget(r, "ProjEstFlag")) === "proj")) {
|
|
166
|
+
notes.push(`${name}: rows flagged Proj. are projections for the current marketing year and Est. are ` +
|
|
167
|
+
"estimates for the year before; they are revised in every monthly report (blank = history).");
|
|
168
|
+
}
|
|
169
|
+
return notes;
|
|
170
|
+
}
|
|
171
|
+
async function wasdeMonths(site, p) {
|
|
172
|
+
if (!blank(p.start_report)) {
|
|
173
|
+
const end = blank(p.end_report) || p.end_report === "latest" ? await site.wasdeLatest() : pstr(p.end_report);
|
|
174
|
+
return [ymRange(pstr(p.start_report), end), false];
|
|
175
|
+
}
|
|
176
|
+
if (blank(p.report) || p.report === "latest")
|
|
177
|
+
return [[await site.wasdeLatest()], true];
|
|
178
|
+
return [[pstr(p.report)], false];
|
|
179
|
+
}
|
|
180
|
+
async function fetchWasde(site, r, p, maxRows) {
|
|
181
|
+
const [months, latest] = await wasdeMonths(site, p);
|
|
182
|
+
const notes = [], missing = [];
|
|
183
|
+
let rows = [];
|
|
184
|
+
if (latest) {
|
|
185
|
+
const tried = [];
|
|
186
|
+
let done = false;
|
|
187
|
+
for (let k = 0; k < 3; k++) {
|
|
188
|
+
const ym = ymAdd(months[0], -k);
|
|
189
|
+
if (has(NO_REPORT, ym)) {
|
|
190
|
+
tried.push(`${ym} (${NO_REPORT[ym]})`);
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
let got;
|
|
194
|
+
try {
|
|
195
|
+
got = wasdeRows((await site.c.get(WASDE_CSV(ym))).body, `the ${ym} WASDE CSV`);
|
|
196
|
+
}
|
|
197
|
+
catch (e) {
|
|
198
|
+
if (!(e instanceof DataError) || !["not_found", "fixture_missing"].includes(e.kind))
|
|
199
|
+
throw e;
|
|
200
|
+
tried.push(`${ym} (${has(WASDE_GAPS, ym) ? WASDE_GAPS[ym] : "CSV not posted yet"})`);
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
if (tried.length) {
|
|
204
|
+
notes.push(`${r.name}: newest report listed is ${months[0]}, but its CSV is not available ` +
|
|
205
|
+
`(${tried.join("; ")}); used the ${ym} report`);
|
|
206
|
+
}
|
|
207
|
+
notes.push(`${r.name}: report = ${ym} (WASDE ${got.length ? pstr(got[0].get("WasdeNumber")) : "?"}, released ` +
|
|
208
|
+
`${got.length ? pstr(got[0].has("ReleaseDate") ? got[0].get("ReleaseDate") : "") : "?"})`);
|
|
209
|
+
rows = wasdeFilter(got, p);
|
|
210
|
+
done = true;
|
|
211
|
+
break;
|
|
212
|
+
}
|
|
213
|
+
if (!done) {
|
|
214
|
+
throw new DataError(`request ${repr(r.name)}: no WASDE CSV found for the latest months (${tried.join("; ")})`, "not_found");
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
else {
|
|
218
|
+
for (const ym of months) {
|
|
219
|
+
if (has(NO_REPORT, ym)) {
|
|
220
|
+
missing.push(`${ym}: ${NO_REPORT[ym]}`);
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
let got;
|
|
224
|
+
try {
|
|
225
|
+
got = wasdeRows((await site.c.get(WASDE_CSV(ym))).body, `the ${ym} WASDE CSV`);
|
|
226
|
+
}
|
|
227
|
+
catch (e) {
|
|
228
|
+
if (!(e instanceof DataError) || e.kind !== "not_found")
|
|
229
|
+
throw e;
|
|
230
|
+
missing.push(`${ym}: ${has(WASDE_GAPS, ym) ? WASDE_GAPS[ym] : "no CSV at " + WASDE_CSV(ym) + " (HTTP 404)"}`);
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
233
|
+
rows = rows.concat(wasdeFilter(got, p));
|
|
234
|
+
if (rows.length >= maxRows)
|
|
235
|
+
break;
|
|
236
|
+
}
|
|
237
|
+
if (missing.length && missing.length === months.length) {
|
|
238
|
+
throw new DataError(`request ${repr(r.name)}: no WASDE CSV for ${months.join(", ")} (${missing.join("; ")}). ` +
|
|
239
|
+
`Older or reissued months are in the consolidated CSV on ${WASDE_HIST} (a browser page; ` +
|
|
240
|
+
"download it and upload it for ingest)", "not_found");
|
|
241
|
+
}
|
|
242
|
+
if (missing.length) {
|
|
243
|
+
notes.push(`${r.name}: report month(s) skipped: ${missing.join("; ")}. The consolidated CSV on ` +
|
|
244
|
+
`${WASDE_HIST} (browser download, then ingest) may hold them (unverified)`);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
if (rows.length > maxRows) {
|
|
248
|
+
notes.push(`${r.name}: stopped at ${fmtInt(maxRows)} rows (max_rows); the reports have more. Narrow the filter ` +
|
|
249
|
+
"or raise max_rows.");
|
|
250
|
+
rows = rows.slice(0, maxRows);
|
|
251
|
+
}
|
|
252
|
+
return [rows, [...notes, ...wasdeNotes(r.name, rows)]];
|
|
253
|
+
}
|
|
254
|
+
// ---------------------------------------------------------------------- farm income
|
|
255
|
+
/**
|
|
256
|
+
* ERS (archive readme, Sept 2026): a year is first forecast in February of that year, updated in late
|
|
257
|
+
* August/early September, late November/early December and the next February, and becomes an estimate in the
|
|
258
|
+
* late August/early September release of the next year.
|
|
259
|
+
*/
|
|
260
|
+
export function estimateType(year, rel) {
|
|
261
|
+
if (!rel)
|
|
262
|
+
return "";
|
|
263
|
+
const y = pyInt(pstr(year).slice(0, 4));
|
|
264
|
+
if (y === null)
|
|
265
|
+
return "";
|
|
266
|
+
return y >= rel.y || (y === rel.y - 1 && rel.m <= 4) ? "forecast" : "estimate";
|
|
267
|
+
}
|
|
268
|
+
/** The text of a release's CSV (a ZIP with one CSV member, or the CSV itself), and the member name. */
|
|
269
|
+
function csvText(data, label) {
|
|
270
|
+
if (data[0] === 0x50 && data[1] === 0x4b && data[2] === 0x03 && data[3] === 0x04) {
|
|
271
|
+
const z = unzip(data);
|
|
272
|
+
const members = Object.keys(z).filter((n) => n.toLowerCase().endsWith(".csv"));
|
|
273
|
+
if (!members.length)
|
|
274
|
+
throw new DataError(`${label}: no CSV in the ZIP (${Object.keys(z).slice(0, 5).join(", ")})`, "api");
|
|
275
|
+
return [cp1252(z[members[0]]), members[0]];
|
|
276
|
+
}
|
|
277
|
+
let i = 0;
|
|
278
|
+
while (i < data.length && [0x20, 0x09, 0x0a, 0x0d, 0x0b, 0x0c].includes(data[i]))
|
|
279
|
+
i++;
|
|
280
|
+
if (data[i] === 0x3c)
|
|
281
|
+
throw new DataError(`${label}: an HTML page came back instead of the data file`, "api");
|
|
282
|
+
return [cp1252(data), label];
|
|
283
|
+
}
|
|
284
|
+
function farmIncomeRows(data, p, maxRows, label) {
|
|
285
|
+
const [text, member] = csvText(data, label);
|
|
286
|
+
const [names, rd] = dictReader(text, true);
|
|
287
|
+
const need = ["Year", "State", "artificialKey", "VariableDescriptionTotal", "Amount"];
|
|
288
|
+
const miss = need.filter((c) => !names.includes(c));
|
|
289
|
+
if (miss.length)
|
|
290
|
+
throw new DataError(`${label}: not an ERS farm income CSV (no ${miss.join(", ")} column)`, "api");
|
|
291
|
+
const vr = new Set(list(p.variable).map(norm));
|
|
292
|
+
const key = new Set(list(p.key).map((v) => strip(pstr(v)).toUpperCase()));
|
|
293
|
+
const st = new Set(list(p.state).map((v) => strip(pstr(v)).toUpperCase()));
|
|
294
|
+
const srch = !blank(p.search) ? norm(p.search) : "";
|
|
295
|
+
const [lo, hi] = years(p);
|
|
296
|
+
const out = [];
|
|
297
|
+
let read = 0, truncated = false, rel = null;
|
|
298
|
+
const g = (row, k, dflt = "") => (row.has(k) ? row.get(k) : dflt);
|
|
299
|
+
for (const row of rd) {
|
|
300
|
+
read++;
|
|
301
|
+
if (st.size && !st.has(strip(pstr(g(row, "State"))).toUpperCase()))
|
|
302
|
+
continue;
|
|
303
|
+
if (key.size && !key.has(strip(pstr(g(row, "artificialKey"))).toUpperCase()))
|
|
304
|
+
continue;
|
|
305
|
+
const d = norm(g(row, "VariableDescriptionTotal", null));
|
|
306
|
+
if (vr.size && !vr.has(d))
|
|
307
|
+
continue;
|
|
308
|
+
if (srch && !d.includes(srch))
|
|
309
|
+
continue;
|
|
310
|
+
if (!inYears(g(row, "Year", null), lo, hi))
|
|
311
|
+
continue;
|
|
312
|
+
if (out.length >= maxRows) {
|
|
313
|
+
truncated = true;
|
|
314
|
+
break;
|
|
315
|
+
}
|
|
316
|
+
rel = rel || dateOf(String(g(row, "PublicationDate") || ""));
|
|
317
|
+
const rec = new Map([...row].map(([k, v]) => [k, v === null ? "" : k === "unit_desc" ? strip(String(v)) : v]));
|
|
318
|
+
rec.set("unit_desc", strip(String(rec.has("unit_desc") ? rec.get("unit_desc") : "").replace(/ /g, " ")));
|
|
319
|
+
rec.set("release_date", rel ? rel.iso : "");
|
|
320
|
+
rec.set("estimate_type", estimateType(g(row, "Year", null), rel));
|
|
321
|
+
out.push(rec);
|
|
322
|
+
}
|
|
323
|
+
return [out, { member, read, truncated, release: rel }];
|
|
324
|
+
}
|
|
325
|
+
async function fetchFarmIncome(site, r, p, maxRows) {
|
|
326
|
+
const rel = pickRelease(fiwReleases(await site.page(FIW_PAGE)), p.release);
|
|
327
|
+
const data = (await site.c.get(rel.url)).body;
|
|
328
|
+
const [rows, info] = farmIncomeRows(data, p, maxRows, basename(rel.url.split("?")[0]));
|
|
329
|
+
const notes = [`${r.name}: ERS farm income release of ${rel.date.iso} (${rel.label}; ${info.member}); ` +
|
|
330
|
+
"amounts are nominal (ChainType_GDP_Deflator, latest year = 100, deflates them); estimate_type follows " +
|
|
331
|
+
"ERS's rule (forecast until the late-August/September release of the next year)"];
|
|
332
|
+
const fc = [...new Set(rows.filter((x) => x.get("estimate_type") === "forecast").map((x) => String(x.get("Year"))))].sort(cmp);
|
|
333
|
+
if (fc.length) {
|
|
334
|
+
notes.push(`${r.name}: forecast year(s) ${fc.join(", ")} (ERS marks them F); a forecast is revised in ` +
|
|
335
|
+
"the next releases");
|
|
336
|
+
}
|
|
337
|
+
if (info.truncated) {
|
|
338
|
+
notes.push(`${r.name}: stopped at ${fmtInt(maxRows)} rows (max_rows); the release has more matching rows. ` +
|
|
339
|
+
"Narrow variable, state or years.");
|
|
340
|
+
}
|
|
341
|
+
return [rows, notes];
|
|
342
|
+
}
|
|
343
|
+
function archiveRows(data, p, label) {
|
|
344
|
+
if (data[0] === 0x50 && data[1] === 0x4b && data[2] === 0x03 && data[3] === 0x04) {
|
|
345
|
+
const z = unzip(data);
|
|
346
|
+
const names = Object.keys(z).filter((n) => n.toLowerCase().endsWith(".csv"));
|
|
347
|
+
if (!names.length)
|
|
348
|
+
throw new DataError(`${label}: no CSV in the ZIP`, "api");
|
|
349
|
+
data = z[names[0]];
|
|
350
|
+
}
|
|
351
|
+
const [fieldnames, rd] = dictReader(cp1252(data));
|
|
352
|
+
if (!fieldnames.includes("release_number")) {
|
|
353
|
+
throw new DataError(`${label}: not the ERS farm income forecast archive (no release_number column)`, "api");
|
|
354
|
+
}
|
|
355
|
+
const items = new Set(list(p.item).map((x) => pstr(x).toLowerCase()));
|
|
356
|
+
const [lo, hi] = years(p);
|
|
357
|
+
const out = [];
|
|
358
|
+
for (const x of rd) {
|
|
359
|
+
if ((!items.size || items.has(String(x.get("item") ?? "").toLowerCase())) &&
|
|
360
|
+
inYears(x.has("year") ? x.get("year") : null, lo, hi))
|
|
361
|
+
out.push(noNone(x));
|
|
362
|
+
}
|
|
363
|
+
return out;
|
|
364
|
+
}
|
|
365
|
+
/** The one-sheet 'U.S. farm sector financial indicators' table: header row = the row starting with years. */
|
|
366
|
+
function summaryRows(data) {
|
|
367
|
+
const grid = orderedSheets(data)[0]?.[1] ?? [];
|
|
368
|
+
const s = (v) => strip(v === null || v === undefined ? "" : String(v));
|
|
369
|
+
const hi = grid.findIndex((row) => row.slice(1).filter((v) => /^\d{4}F?$/.test(s(v))).length >= 3);
|
|
370
|
+
if (hi < 0)
|
|
371
|
+
throw new DataError("the farm sector indicators workbook has no row of years", "api");
|
|
372
|
+
const cols = grid[hi].slice(1).filter((v) => !blank(v)).map((v) => strip(String(v)));
|
|
373
|
+
const notes = [], out = [];
|
|
374
|
+
const asofRow = grid.slice(0, hi).find((row) => row.length > 2 && String(row[1] || "").startsWith("Data as of"));
|
|
375
|
+
const asof = asofRow ? asofRow[2] : null;
|
|
376
|
+
if (asof && /^\d+$/.test(String(asof))) {
|
|
377
|
+
const t = new Date(Date.UTC(1899, 11, 30) + parseInt(String(asof), 10) * 86400000);
|
|
378
|
+
notes.push(`data as of ${day(t.getUTCFullYear(), t.getUTCMonth() + 1, t.getUTCDate()).iso}`);
|
|
379
|
+
}
|
|
380
|
+
let section = "", unit = "";
|
|
381
|
+
for (const row of grid.slice(hi + 1)) {
|
|
382
|
+
const label = row.length ? strip(String(row[0] || "")) : "";
|
|
383
|
+
const vals = Array.from({ length: cols.length }, (_, k) => (k + 1 < row.length ? row[k + 1] : null));
|
|
384
|
+
const texts = vals.filter((v) => !blank(v) && !NUM.test(s(v)) && !["NA", ""].includes(s(v))).map(s);
|
|
385
|
+
const nums = vals.filter((v) => !blank(v) && (NUM.test(s(v)) || s(v) === "NA"));
|
|
386
|
+
if (!label && !nums.length && !texts.length)
|
|
387
|
+
continue;
|
|
388
|
+
if (texts.length && !nums.length) {
|
|
389
|
+
unit = texts[0];
|
|
390
|
+
if (label)
|
|
391
|
+
section = label;
|
|
392
|
+
continue;
|
|
393
|
+
}
|
|
394
|
+
if (label && !nums.length) {
|
|
395
|
+
if (label.startsWith("Note:") || /^\d+\//.test(label))
|
|
396
|
+
notes.push(label);
|
|
397
|
+
else
|
|
398
|
+
section = label;
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
const rec = new Map([["section", section], ["item", label], ["unit", unit]]);
|
|
402
|
+
cols.forEach((c, k) => rec.set(c, num(vals[k])));
|
|
403
|
+
out.push(rec);
|
|
404
|
+
}
|
|
405
|
+
return [out, notes];
|
|
406
|
+
}
|
|
407
|
+
function xlsxHeader(grid, headerRow) {
|
|
408
|
+
if (!blank(headerRow))
|
|
409
|
+
return pyInt(headerRow) - 1;
|
|
410
|
+
const i = grid.findIndex((row) => row.filter((v) => !blank(v)).length >= 3);
|
|
411
|
+
return i < 0 ? 0 : i;
|
|
412
|
+
}
|
|
413
|
+
/** The member of a baseline ZIP the request names, as rows; z = {member name: bytes} in the ZIP's order. */
|
|
414
|
+
function baselineMemberRows(z, p, label) {
|
|
415
|
+
const glob = pstr(p.member || "Projections*.csv").toLowerCase();
|
|
416
|
+
const names = Object.keys(z).filter((n) => !n.endsWith("/"));
|
|
417
|
+
const hit = names.filter((n) => fnmatch(n.toLowerCase(), glob) || fnmatch(basename(n).toLowerCase(), glob));
|
|
418
|
+
if (hit.length !== 1) {
|
|
419
|
+
throw new DataError(`${label}: member ${repr(p.member || "Projections*.csv")} matches ${hit.length} files; members: ` +
|
|
420
|
+
`${names.join("; ")}`, "bad_request");
|
|
421
|
+
}
|
|
422
|
+
const m = hit[0];
|
|
423
|
+
if (m.toLowerCase().endsWith(".csv")) {
|
|
424
|
+
const [, rd] = dictReader(utf8OrCp1252(z[m]));
|
|
425
|
+
const com = list(p.commodity), att = list(p.attribute), yr = list(p.year);
|
|
426
|
+
const g = (x, k) => (x.has(k) ? x.get(k) : null);
|
|
427
|
+
return [[...rd].filter((x) => match(g(x, "commodity"), com) && match(g(x, "attribute"), att) && match(g(x, "year"), yr))
|
|
428
|
+
.map(noNone), m];
|
|
429
|
+
}
|
|
430
|
+
if (m.toLowerCase().endsWith(".xlsx")) {
|
|
431
|
+
const sheets = orderedSheets(z[m]);
|
|
432
|
+
const want = p.sheet || 1;
|
|
433
|
+
let grid;
|
|
434
|
+
if (/^\d+$/.test(pstr(want))) {
|
|
435
|
+
const i = parseInt(pstr(want), 10) - 1;
|
|
436
|
+
grid = sheets[i < 0 ? sheets.length + i : i]?.[1];
|
|
437
|
+
}
|
|
438
|
+
else
|
|
439
|
+
grid = sheets.find(([n]) => n === want)?.[1];
|
|
440
|
+
if (!grid)
|
|
441
|
+
throw new DataError(`${m}: no sheet ${repr(want)} (sheets: ${sheets.map(([n]) => n).join(", ")})`, "bad_request");
|
|
442
|
+
const h = xlsxHeader(grid, p.header_row);
|
|
443
|
+
return [gridRecords(grid, h + 1).map((x) => new Map([...x].map(([k, v]) => [k, num(v)]))), m];
|
|
444
|
+
}
|
|
445
|
+
const ext = /\.[^./\\]*$/.exec(basename(m));
|
|
446
|
+
throw new DataError(`${m}: only the CSV and .xlsx members are read (this is ${ext ? ext[0] : ""})`, "bad_request");
|
|
447
|
+
}
|
|
448
|
+
function zipMembers(data, what) {
|
|
449
|
+
try {
|
|
450
|
+
return unzip(data);
|
|
451
|
+
}
|
|
452
|
+
catch {
|
|
453
|
+
throw new DataError(`${what}: not a ZIP file (an HTML page in its place?)`, "api");
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
async function fetchBaseline(site, r, p) {
|
|
457
|
+
const ed = pickEdition(baselineEditions(await site.page(BASE_PAGE)), p.edition);
|
|
458
|
+
const data = (await site.c.get(ed.url)).body;
|
|
459
|
+
const z = zipMembers(data, ed.url);
|
|
460
|
+
const [rows, m] = baselineMemberRows(z, p, basename(ed.url.split("?")[0]));
|
|
461
|
+
const notes = [`${r.name}: USDA Agricultural Projections to ${ed.to_year} (${ed.kind} release` +
|
|
462
|
+
`${ed.updated ? ", last updated " + ed.updated : ""}), ${m}. A conditional baseline under ` +
|
|
463
|
+
"current law and policy, not a forecast; the first years follow the WASDE of the November before"];
|
|
464
|
+
if (ed.kind === "early") {
|
|
465
|
+
notes.push(`${r.name}: early-release tables (before the full report); the full release in February ` +
|
|
466
|
+
"can differ and adds farm income, trade and horticulture tables");
|
|
467
|
+
}
|
|
468
|
+
return [rows, notes];
|
|
469
|
+
}
|
|
470
|
+
// ---------------------------------------------------------------------- hooks
|
|
471
|
+
function nowYm() {
|
|
472
|
+
const t = new Date();
|
|
473
|
+
return `${String(t.getFullYear()).padStart(4, "0")}-${String(t.getMonth() + 1).padStart(2, "0")}`;
|
|
474
|
+
}
|
|
475
|
+
function checkSpec(spec) {
|
|
476
|
+
const probs = [];
|
|
477
|
+
const now = nowYm();
|
|
478
|
+
for (const r of spec.requests || []) {
|
|
479
|
+
const p = r.params || {}, n = repr(r.name), e = r.endpoint;
|
|
480
|
+
if (e === "wasde") {
|
|
481
|
+
for (const k of ["report", "start_report", "end_report"]) {
|
|
482
|
+
const v = p[k];
|
|
483
|
+
if (blank(v) || (k !== "start_report" && v === "latest"))
|
|
484
|
+
continue;
|
|
485
|
+
const sv = pstr(v);
|
|
486
|
+
if (!YM.test(sv))
|
|
487
|
+
probs.push(`request ${n}: ${k} must be YYYY-MM such as 2026-10` + (k !== "start_report" ? ' or "latest"' : ""));
|
|
488
|
+
else if (sv < WASDE_FIRST) {
|
|
489
|
+
probs.push(`request ${n}: ${k} ${sv}: the monthly WASDE CSVs start with ${WASDE_FIRST}; earlier ` +
|
|
490
|
+
`reports are in the consolidated CSV on ${WASDE_HIST} (browser download, then ingest)`);
|
|
491
|
+
}
|
|
492
|
+
else if (sv > now)
|
|
493
|
+
probs.push(`request ${n}: ${k} ${sv} is in the future`);
|
|
494
|
+
}
|
|
495
|
+
if (!blank(p.report) && !blank(p.start_report))
|
|
496
|
+
probs.push(`request ${n}: give report or start_report/end_report, not both`);
|
|
497
|
+
if (!blank(p.end_report) && blank(p.start_report))
|
|
498
|
+
probs.push(`request ${n}: end_report needs start_report`);
|
|
499
|
+
const s = pstr(p.start_report || ""), t = pstr(p.end_report || now);
|
|
500
|
+
const t2 = t !== "latest" ? t : now;
|
|
501
|
+
if (YM.test(s) && YM.test(t2)) {
|
|
502
|
+
if (ymRange(s, t2).length > WASDE_MAX_MONTHS)
|
|
503
|
+
probs.push(`request ${n}: a range of more than ${WASDE_MAX_MONTHS} reports; split it`);
|
|
504
|
+
else if (t !== "latest" && s > t)
|
|
505
|
+
probs.push(`request ${n}: end_report is before start_report`);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
if (e === "farm_income") {
|
|
509
|
+
if (!["variable", "key", "search"].some((k) => !(blank(p[k]) || (Array.isArray(p[k]) && !p[k].length)))) {
|
|
510
|
+
probs.push(`request ${n}: farm_income needs variable, key or search (the release holds about ` +
|
|
511
|
+
'470,000 rows); e.g. "variable": "Net farm income"');
|
|
512
|
+
}
|
|
513
|
+
for (const s of list(p.state))
|
|
514
|
+
if (!/^[A-Za-z]{2}$/.test(pstr(s)))
|
|
515
|
+
probs.push(`request ${n}: state ${repr(s)} must be US or a two-letter state code`);
|
|
516
|
+
const rel = pstr(p.release || "latest");
|
|
517
|
+
if (rel !== "latest" && !/^\d{4}-\d{2}(-\d{2})?$/.test(rel))
|
|
518
|
+
probs.push(`request ${n}: release must be "latest", YYYY-MM-DD or YYYY-MM`);
|
|
519
|
+
else if (rel !== "latest" && rel.slice(0, 7) < FI_FIRST) {
|
|
520
|
+
probs.push(`request ${n}: release files on the ERS page start with August 2014; older vintages ` +
|
|
521
|
+
"of net farm income, net cash income and cash receipts are in farm_income_archive");
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
if (e === "farm_income" || e === "farm_income_archive") {
|
|
525
|
+
try {
|
|
526
|
+
years(p);
|
|
527
|
+
}
|
|
528
|
+
catch {
|
|
529
|
+
probs.push(`request ${n}: year, start_year and end_year are four-digit years`);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
if (e === "baseline") {
|
|
533
|
+
const ed = pstr(p.edition || "latest").toLowerCase();
|
|
534
|
+
if (!["latest", "early"].includes(ed) && !/^20\d\d$/.test(ed)) {
|
|
535
|
+
probs.push(`request ${n}: edition must be "latest", "early" or the last projection year (e.g. 2035)`);
|
|
536
|
+
}
|
|
537
|
+
if (!blank(p.header_row) && !/^\d+$/.test(pstr(p.header_row)))
|
|
538
|
+
probs.push(`request ${n}: header_row is a 1-based row number`);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
return probs;
|
|
542
|
+
}
|
|
543
|
+
async function fetch(spec, pack) {
|
|
544
|
+
const site = new Site(pack);
|
|
545
|
+
const tables = {};
|
|
546
|
+
let notes = [];
|
|
547
|
+
const defaultMax = Number(pack.policy.limits?.max_rows || 50000);
|
|
548
|
+
for (const r of spec.requests) {
|
|
549
|
+
const [, params, probs] = gr.resolve(pack, r);
|
|
550
|
+
if (probs.length)
|
|
551
|
+
throw new DataError(probs.join("; "), "bad_request");
|
|
552
|
+
const e = r.endpoint;
|
|
553
|
+
const mx = Math.trunc(Number(r.max_rows || defaultMax));
|
|
554
|
+
let rows, nn;
|
|
555
|
+
try {
|
|
556
|
+
if (e === "wasde")
|
|
557
|
+
[rows, nn] = await fetchWasde(site, r, params, mx);
|
|
558
|
+
else if (e === "farm_income")
|
|
559
|
+
[rows, nn] = await fetchFarmIncome(site, r, params, mx);
|
|
560
|
+
else if (e === "farm_income_summary") {
|
|
561
|
+
const a = fiwLink(await site.page(FIW_PAGE), "U.S. farm sector financial indicators");
|
|
562
|
+
if (!a)
|
|
563
|
+
throw new DataError("no 'U.S. farm sector financial indicators' workbook on the data-files page", "api");
|
|
564
|
+
let extra;
|
|
565
|
+
[rows, extra] = summaryRows((await site.c.get(a.url)).body);
|
|
566
|
+
nn = [`${r.name}: ${a.text} (${a.url}); ${extra.slice(0, 1).join("; ")}; values in the unit column ` +
|
|
567
|
+
"(billion dollars or percent); F = forecast"];
|
|
568
|
+
}
|
|
569
|
+
else if (e === "farm_income_archive") {
|
|
570
|
+
const a = fiwLink(await site.page(FIW_PAGE), "ERS Farm Income and Wealth Statistics Data Archive");
|
|
571
|
+
if (!a)
|
|
572
|
+
throw new DataError("no data archive ZIP on the data-files page", "api");
|
|
573
|
+
rows = archiveRows((await site.c.get(a.url)).body, params, basename(a.url));
|
|
574
|
+
nn = [`${r.name}: ERS archive of past forecasts and estimates (${a.url}); release_number 1-4 are ` +
|
|
575
|
+
"forecasts, 5 on estimates; ERS says it is a research file, not an official record"];
|
|
576
|
+
}
|
|
577
|
+
else if (e === "baseline")
|
|
578
|
+
[rows, nn] = await fetchBaseline(site, r, params);
|
|
579
|
+
else
|
|
580
|
+
throw new DataError(`unknown endpoint ${repr(e)}`, "bad_request");
|
|
581
|
+
}
|
|
582
|
+
catch (err) {
|
|
583
|
+
if (!(err instanceof DataError))
|
|
584
|
+
throw err;
|
|
585
|
+
throw new DataError(`request ${repr(r.name)} (${pstr(e)}): ${err.message}`, err.kind, err.status);
|
|
586
|
+
}
|
|
587
|
+
if (!rows.length && !r.allow_empty) {
|
|
588
|
+
throw new DataError(`request ${repr(r.name)} returned no rows: check the filter values against the ` +
|
|
589
|
+
"published names (case is ignored; see source.md)", "bad_request");
|
|
590
|
+
}
|
|
591
|
+
notes = notes.concat(nn);
|
|
592
|
+
tables[r.name] = asTable(rows);
|
|
593
|
+
}
|
|
594
|
+
return [tables, notes];
|
|
595
|
+
}
|
|
596
|
+
// ---------------------------------------------------------------------- manual route
|
|
597
|
+
/**
|
|
598
|
+
* Download links. Python read the index pages here when it could (latest WASDE month, release and edition links)
|
|
599
|
+
* and fell back to the pages themselves when it could not; this hook is synchronous, so it always gives the
|
|
600
|
+
* fallback: WASDE months from the request (this month for "latest"), and the ERS pages for the other files.
|
|
601
|
+
*/
|
|
602
|
+
function manualItems(spec, pack) {
|
|
603
|
+
const items = [];
|
|
604
|
+
for (const r of spec.requests || []) {
|
|
605
|
+
const [, params, probs] = gr.resolve(pack, r);
|
|
606
|
+
if (probs.length)
|
|
607
|
+
continue;
|
|
608
|
+
const e = r.endpoint, n = r.name;
|
|
609
|
+
if (e === "wasde") {
|
|
610
|
+
const rep = params.report;
|
|
611
|
+
let months = [!blank(rep) && rep !== "latest" ? pstr(rep) : nowYm()];
|
|
612
|
+
if (params.start_report) {
|
|
613
|
+
const end = blank(params.end_report) || params.end_report === "latest" ? nowYm() : pstr(params.end_report);
|
|
614
|
+
months = ymRange(pstr(params.start_report), end);
|
|
615
|
+
}
|
|
616
|
+
for (const ym of months) {
|
|
617
|
+
if (has(NO_REPORT, ym))
|
|
618
|
+
continue;
|
|
619
|
+
const url = WASDE_CSV(ym);
|
|
620
|
+
items.push({ name: `${n}#${ym}`, request: n, url, filename: `${n}_${ym}.csv`, alt_filenames: [basename(url)],
|
|
621
|
+
needs_key: false, method: "GET", body: null,
|
|
622
|
+
note: `the ${ym} WASDE data CSV (if it answers 404, take the previous month; for ` +
|
|
623
|
+
`older or reissued months download the consolidated CSV on ${WASDE_HIST})` });
|
|
624
|
+
}
|
|
625
|
+
continue;
|
|
626
|
+
}
|
|
627
|
+
const url = e === "baseline" ? BASE_PAGE : FIW_PAGE;
|
|
628
|
+
const note = { farm_income: "open the page and download the release ZIP (e.g. 'September 3, 2026 release'); " +
|
|
629
|
+
"upload it unchanged",
|
|
630
|
+
farm_income_summary: "open the page and download 'U.S. farm sector financial indicators' (.xlsx)",
|
|
631
|
+
farm_income_archive: "open the page and download the 'ERS Farm Income and Wealth Statistics Data Archive' ZIP",
|
|
632
|
+
baseline: "open the page and download the ZIP under 'Most Recent Full-Release' (or the edition asked for)" }[e || ""] ?? "";
|
|
633
|
+
const ext = e === "farm_income_summary" ? "xlsx" : "zip";
|
|
634
|
+
const base = basename(url.split("?")[0]);
|
|
635
|
+
items.push({ name: `${n}#1`, request: n, url, filename: `${n}.${ext}`, also_ext: [".csv", ".zip", ".xlsx"],
|
|
636
|
+
alt_filenames: base.includes(".") ? [base] : [], needs_key: false, method: "GET", body: null, note });
|
|
637
|
+
}
|
|
638
|
+
return items;
|
|
639
|
+
}
|
|
640
|
+
function parseUpload(pack, req, files) {
|
|
641
|
+
const [, p] = gr.resolve(pack, req);
|
|
642
|
+
const e = req.endpoint;
|
|
643
|
+
const mx = Math.trunc(Number(req.max_rows || pack.policy.limits?.max_rows || 50000));
|
|
644
|
+
let rows = [];
|
|
645
|
+
for (const f of [...files].sort(cmp)) {
|
|
646
|
+
const data = fs.readFileSync(f);
|
|
647
|
+
const label = path.basename(f);
|
|
648
|
+
try {
|
|
649
|
+
if (e === "wasde") {
|
|
650
|
+
let got = wasdeRows(data, label);
|
|
651
|
+
let months = [];
|
|
652
|
+
if (p.start_report)
|
|
653
|
+
months = ymRange(pstr(p.start_report), pstr(p.end_report || "9999-12"));
|
|
654
|
+
else if (!blank(p.report) && p.report !== "latest")
|
|
655
|
+
months = [pstr(p.report)];
|
|
656
|
+
const ym = (x) => {
|
|
657
|
+
const fy = pstr(x.has("ForecastYear") ? x.get("ForecastYear") : ""), fm = pstr(x.has("ForecastMonth") ? x.get("ForecastMonth") : "");
|
|
658
|
+
return /^\d+$/.test(fy) && /^\d+$/.test(fm)
|
|
659
|
+
? `${String(parseInt(fy || "0", 10)).padStart(4, "0")}-${String(parseInt(fm || "0", 10)).padStart(2, "0")}` : "";
|
|
660
|
+
};
|
|
661
|
+
if (months.length)
|
|
662
|
+
got = got.filter((x) => months.includes(ym(x)));
|
|
663
|
+
else if (got.length && got.some((x) => ym(x))) {
|
|
664
|
+
const newest = got.map(ym).reduce((a, b) => (b > a ? b : a));
|
|
665
|
+
got = got.filter((x) => ym(x) === newest);
|
|
666
|
+
}
|
|
667
|
+
rows = rows.concat(wasdeFilter(got, p));
|
|
668
|
+
}
|
|
669
|
+
else if (e === "farm_income")
|
|
670
|
+
rows = rows.concat(farmIncomeRows(data, p, mx, label)[0]);
|
|
671
|
+
else if (e === "farm_income_archive")
|
|
672
|
+
rows = rows.concat(archiveRows(data, p, label));
|
|
673
|
+
else if (e === "farm_income_summary")
|
|
674
|
+
rows = rows.concat(summaryRows(data)[0]);
|
|
675
|
+
else if (e === "baseline") {
|
|
676
|
+
if (data.subarray(0, 4).equals(Buffer.from("PK\x03\x04", "latin1")) && label.toLowerCase().endsWith(".zip")) {
|
|
677
|
+
rows = rows.concat(baselineMemberRows(unzip(data), p, label)[0]);
|
|
678
|
+
}
|
|
679
|
+
else { // the member itself (CSV or .xlsx) uploaded on its own
|
|
680
|
+
rows = rows.concat(baselineMemberRows({ [label]: data }, { ...p, member: label }, label)[0]);
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
catch (err) {
|
|
685
|
+
if (!(err instanceof DataError))
|
|
686
|
+
throw err;
|
|
687
|
+
throw new DataError(`${label}: ${err.message}`, "bad_request");
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
if (!rows.length)
|
|
691
|
+
throw new DataError(`request ${repr(req.name)}: the uploaded file(s) have no rows matching the request`, "bad_request");
|
|
692
|
+
return asTable(rows);
|
|
693
|
+
}
|
|
694
|
+
const adapter = { checkSpec, fetch, manualItems, parseUpload };
|
|
695
|
+
export default adapter;
|
|
696
|
+
//# sourceMappingURL=usda-outlook.js.map
|