data-analyst-mcp 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +262 -3
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +710 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +21 -0
- package/dist/report/scan_secrets.js +234 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +246 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +56 -6
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +165 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,1168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build a single self-contained HTML report or paper, and replication_package.zip, from report.json (a port of
|
|
3
|
+
* report/build_report.py). The HTML has no external dependencies; charts are drawn by an inline script; the
|
|
4
|
+
* AI-assistance disclaimer (guide/assets/disclaimer.txt) is always printed at the top of the page.
|
|
5
|
+
*
|
|
6
|
+
* What differs by source comes from each used source's policy.json (report.json "sources", else the data/<id>/
|
|
7
|
+
* folders): notices (Data and Methods and the page footer), redistribution (policyset.ts decides what enters the
|
|
8
|
+
* ZIP; definition_only sources get reproduce/<id>/ with download scripts, the request and REPRODUCE.md), withhold
|
|
9
|
+
* (rows of withheld keys are dropped from shared CSVs unless a "clearance" lifts them), vintage (the Data Vintage
|
|
10
|
+
* block from data/<id>/request.json), size (data files over 50 MB stay out with a note).
|
|
11
|
+
*
|
|
12
|
+
* The replication package carries the exhibit recipes (exhibits/lineage.json, written by compute) instead of an
|
|
13
|
+
* analysis script: its REPLICATION.md says how to replay them with this server's tools (rerun, build_report,
|
|
14
|
+
* audit_report). Nothing is ever written inside the server's package folder.
|
|
15
|
+
*/
|
|
16
|
+
import crypto from "node:crypto";
|
|
17
|
+
import fs from "node:fs";
|
|
18
|
+
import path from "node:path";
|
|
19
|
+
import { zipSync } from "fflate";
|
|
20
|
+
import { GUIDE_DIR, ROOT } from "../core/env.js";
|
|
21
|
+
import * as manual from "../core/manual.js";
|
|
22
|
+
import { pyJsonDumps } from "../core/py.js";
|
|
23
|
+
import * as pol from "../core/policy.js";
|
|
24
|
+
import { FMT_PATTERN, formatValue } from "../analysis/factfmt.js";
|
|
25
|
+
import * as MA from "./method_audit.js";
|
|
26
|
+
import * as PS from "./policyset.js";
|
|
27
|
+
import { knownKeys, scanText } from "./scan_secrets.js";
|
|
28
|
+
import { commas, escapeHtml, fmtG, readDict, readText, split, splitlines, str, strip, toFloat, writeDict } from "./py.js";
|
|
29
|
+
const TEMPLATE = path.join(GUIDE_DIR, "assets", "report_template.html");
|
|
30
|
+
const DISCLAIMER = path.join(GUIDE_DIR, "assets", "disclaimer.txt");
|
|
31
|
+
const EMBED_MAX = 400_000;
|
|
32
|
+
const SCRIPT_KINDS = { bash: ".sh", powershell: ".ps1" };
|
|
33
|
+
export class BuildError extends Error {
|
|
34
|
+
}
|
|
35
|
+
function fail(msg) {
|
|
36
|
+
throw new BuildError(msg);
|
|
37
|
+
}
|
|
38
|
+
// ----------------------------------------------------------------------------- text helpers
|
|
39
|
+
const fnId = (n, scope) => (scope ? `${scope}-${n}` : String(n));
|
|
40
|
+
const CITE_SRC = String.raw `\[@[A-Za-z0-9_-]+(?:\s*[;,]\s*@[A-Za-z0-9_-]+)*\]`;
|
|
41
|
+
const CITE_RE = new RegExp(CITE_SRC, "g");
|
|
42
|
+
let REF_LABELS = new Map();
|
|
43
|
+
export function citedIds(text) {
|
|
44
|
+
const out = new Set();
|
|
45
|
+
for (const g of text.match(CITE_RE) || [])
|
|
46
|
+
for (const m of g.matchAll(/@([A-Za-z0-9_-]+)/g))
|
|
47
|
+
out.add(m[1]);
|
|
48
|
+
return out;
|
|
49
|
+
}
|
|
50
|
+
export const esc = (s) => escapeHtml(s === null || s === undefined ? "" : str(s));
|
|
51
|
+
export function inlineMd(s0, scope) {
|
|
52
|
+
let s = escapeHtml(str(s0 ?? ""));
|
|
53
|
+
s = s.replace(/\*\*(.+?)\*\*/g, "<strong>$1</strong>");
|
|
54
|
+
s = s.replace(/(?<!\*)\*(?!\s)(.+?)(?<!\s)\*(?!\*)/g, "<em>$1</em>");
|
|
55
|
+
s = s.replace(/`(.+?)`/g, "<code>$1</code>");
|
|
56
|
+
s = s.replace(/\[([^\]]+)\]\((https?:\/\/[^)\s]+)\)/g, '<a href="$2">$1</a>');
|
|
57
|
+
s = s.replace(/\[\^(\d+)\]/g, (_m, n) => `<sup class="fn"><a href="#fn-${fnId(n, scope)}" id="fnref-${fnId(n, scope)}">${n}</a></sup>`);
|
|
58
|
+
return s.replace(CITE_RE, (m) => "(" + [...m.matchAll(/@([A-Za-z0-9_-]+)/g)].map((x) => `<a class="cite" href="#ref-${x[1]}">${escapeHtml(REF_LABELS.get(x[1]) ?? x[1])}</a>`).join("; ") + ")");
|
|
59
|
+
}
|
|
60
|
+
/** Paragraphs separated by blank lines; lines starting with '- ' become a list. */
|
|
61
|
+
export function blocksMd(text, scope) {
|
|
62
|
+
if (Array.isArray(text))
|
|
63
|
+
return text.map((t) => blocksMd(t, scope)).join("");
|
|
64
|
+
const out = [];
|
|
65
|
+
for (const block of split(/\n\s*\n/, strip(text ? str(text) : ""))) {
|
|
66
|
+
const lines = splitlines(block).filter((l) => strip(l));
|
|
67
|
+
if (lines.length && lines.every((l) => /^\s*(?:- |\* )/.test(l))) {
|
|
68
|
+
out.push("<ul>" + lines.map((l) => `<li>${inlineMd(l.trimStart().slice(2), scope)}</li>`).join("") + "</ul>");
|
|
69
|
+
}
|
|
70
|
+
else if (lines.length)
|
|
71
|
+
out.push(`<p>${inlineMd(lines.join(" "), scope)}</p>`);
|
|
72
|
+
}
|
|
73
|
+
return out.join("");
|
|
74
|
+
}
|
|
75
|
+
function footnotesHtml(notes, scope, heading = "Notes") {
|
|
76
|
+
if (!notes || !notes.length)
|
|
77
|
+
return "";
|
|
78
|
+
const items = notes.map((t, i) => `<li id='fn-${fnId(i + 1, scope)}'>${inlineMd(t, scope)} ` +
|
|
79
|
+
`<a href='#fnref-${fnId(i + 1, scope)}' aria-label='Back to text'>↩</a></li>`).join("");
|
|
80
|
+
const h = heading ? `<h2>${esc(heading)}</h2>` : "";
|
|
81
|
+
return `<section class='prose ${heading ? "footnotes" : "footnotes part-notes"}'>${h}<ol>${items}</ol></section>`;
|
|
82
|
+
}
|
|
83
|
+
// ----------------------------------------------------------------------------- data helpers
|
|
84
|
+
function num(v) {
|
|
85
|
+
if (v === null || v === undefined)
|
|
86
|
+
return null;
|
|
87
|
+
if (typeof v === "number")
|
|
88
|
+
return Number.isNaN(v) ? null : v;
|
|
89
|
+
if (typeof v === "boolean")
|
|
90
|
+
return v ? 1 : 0;
|
|
91
|
+
const s = strip(String(v)).replace(/,/g, "");
|
|
92
|
+
if (s === "" || ["nan", "na", "none", "null"].includes(s.toLowerCase()))
|
|
93
|
+
return null;
|
|
94
|
+
return toFloat(s);
|
|
95
|
+
}
|
|
96
|
+
function key(v) {
|
|
97
|
+
const n = num(v);
|
|
98
|
+
return n !== null && Number.isFinite(n) ? String(n) : str(v);
|
|
99
|
+
}
|
|
100
|
+
function loadRows(ex, base) {
|
|
101
|
+
if ("rows" in ex) {
|
|
102
|
+
const rows = ex.rows || [];
|
|
103
|
+
const cols = rows.length ? Object.keys(rows[0]) : [];
|
|
104
|
+
return [rows, cols, writeDict(rows, cols, (v) => (v === null || v === undefined ? "" : str(v)))];
|
|
105
|
+
}
|
|
106
|
+
const text = readText(path.join(base, ex.data));
|
|
107
|
+
const d = readDict(text);
|
|
108
|
+
return [d.rows, d.fields, text.replace(/^/, "")];
|
|
109
|
+
}
|
|
110
|
+
const REC_NOTE = "Shaded areas indicate recessions as dated by the National Bureau of Economic Research.";
|
|
111
|
+
function loadRecessions(base, rel) {
|
|
112
|
+
const p = path.join(base, rel);
|
|
113
|
+
if (!fs.existsSync(p)) {
|
|
114
|
+
fail(`a chart asks for recession shading but ${rel} does not exist: save it with compute (columns start, end)`);
|
|
115
|
+
}
|
|
116
|
+
return readDict(readText(p)).rows.map((r) => [r.start ?? "", r.end ?? ""]);
|
|
117
|
+
}
|
|
118
|
+
function chartSpec(ex, rows0) {
|
|
119
|
+
const c = { ...(ex.chart || {}) };
|
|
120
|
+
if (!Object.keys(c).length || c.type === "table")
|
|
121
|
+
return null;
|
|
122
|
+
let rows = rows0;
|
|
123
|
+
for (const [k, v] of Object.entries(c.filter || {})) {
|
|
124
|
+
const vals = new Set((Array.isArray(v) ? v : [v]).map(key));
|
|
125
|
+
rows = rows.filter((r) => vals.has(key(r[k])));
|
|
126
|
+
}
|
|
127
|
+
const x = c.x;
|
|
128
|
+
let cats = [];
|
|
129
|
+
for (const r of rows)
|
|
130
|
+
if (!cats.includes(key(r[x])))
|
|
131
|
+
cats.push(key(r[x]));
|
|
132
|
+
if (c.x_order && c.x_order.length) {
|
|
133
|
+
const order = c.x_order.map(key);
|
|
134
|
+
cats = [...order.filter((v) => cats.includes(v)), ...cats.filter((v) => !order.includes(v))];
|
|
135
|
+
}
|
|
136
|
+
const scol = Array.isArray(c.y) ? null : c.series;
|
|
137
|
+
const seen = new Map();
|
|
138
|
+
for (const r of rows) { // one value per (series, x), or the chart shows the last row
|
|
139
|
+
const k = JSON.stringify([scol ? key(r[scol]) : "", key(r[x])]);
|
|
140
|
+
seen.set(k, (seen.get(k) || 0) + 1);
|
|
141
|
+
}
|
|
142
|
+
const dup = [...seen.entries()].filter(([, n]) => n > 1).map(([k]) => k);
|
|
143
|
+
if (dup.length) {
|
|
144
|
+
fail(`exhibit '${ex.title ?? ""}': ${dup.length} (series, ${x}) pairs have several rows, e.g. ${dup[0]}; add a chart ` +
|
|
145
|
+
"'filter' (or 'series') so each point is one row");
|
|
146
|
+
}
|
|
147
|
+
const series = new Map();
|
|
148
|
+
if (Array.isArray(c.y))
|
|
149
|
+
for (const col of c.y)
|
|
150
|
+
series.set(col, new Map(rows.map((r) => [key(r[x]), num(r[col])])));
|
|
151
|
+
else {
|
|
152
|
+
for (const r of rows) {
|
|
153
|
+
const name = scol ? key(r[scol]) : c.series_name || ex.subtitle || c.y;
|
|
154
|
+
if (!series.has(name))
|
|
155
|
+
series.set(name, new Map());
|
|
156
|
+
series.get(name).set(key(r[x]), num(r[c.y]));
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
let names = [...series.keys()];
|
|
160
|
+
if (c.series_order && c.series_order.length) {
|
|
161
|
+
const order = c.series_order.map(key);
|
|
162
|
+
names = [...order.filter((n) => series.has(n)), ...names.filter((n) => !order.includes(n))];
|
|
163
|
+
}
|
|
164
|
+
if (names.length > 8)
|
|
165
|
+
MA.note(`exhibit '${ex.title}' has ${names.length} series; more than 8 is unreadable. Fold into 'Other' or split.`);
|
|
166
|
+
let dates = null;
|
|
167
|
+
const dcol = c.date || "date";
|
|
168
|
+
if ((c.type || "line") === "line" && rows.length && dcol in rows[0]) {
|
|
169
|
+
const first = new Map();
|
|
170
|
+
for (const r of rows)
|
|
171
|
+
if (!first.has(key(r[x])))
|
|
172
|
+
first.set(key(r[x]), String(r[dcol] ?? ""));
|
|
173
|
+
dates = cats.map((k) => first.get(k) || "");
|
|
174
|
+
if (!dates.every((d) => /^\d{4}-\d{2}-\d{2}$/.test(d)))
|
|
175
|
+
dates = null;
|
|
176
|
+
}
|
|
177
|
+
return { type: c.type || "line", categories: cats, dates, recessions: c._recessions ?? null,
|
|
178
|
+
series: names.map((n) => ({ name: n, values: cats.map((k) => series.get(n).get(k) ?? null) })),
|
|
179
|
+
format: c.format || "number", decimals: c.decimals ?? null, y_min: c.y_min ?? null, y_max: c.y_max ?? null, x_label: c.x_label || "" };
|
|
180
|
+
}
|
|
181
|
+
// ----------------------------------------------------------------------------- render helpers
|
|
182
|
+
const MONTH_NAMES = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October",
|
|
183
|
+
"November", "December"];
|
|
184
|
+
function isoParts(s) {
|
|
185
|
+
const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(String(s ?? "").slice(0, 10));
|
|
186
|
+
if (!m)
|
|
187
|
+
return null;
|
|
188
|
+
const [y, mo, d] = [Number(m[1]), Number(m[2]), Number(m[3])];
|
|
189
|
+
const dt = new Date(Date.UTC(y, mo - 1, d));
|
|
190
|
+
return dt.getUTCMonth() === mo - 1 && dt.getUTCDate() === d ? [y, mo, d] : null;
|
|
191
|
+
}
|
|
192
|
+
export function longDate(s) {
|
|
193
|
+
if (!s)
|
|
194
|
+
return "";
|
|
195
|
+
const p = isoParts(s);
|
|
196
|
+
return p ? `${MONTH_NAMES[p[1] - 1]} ${p[0]}` : str(s);
|
|
197
|
+
}
|
|
198
|
+
export function dayDate(s) {
|
|
199
|
+
const p = isoParts(s);
|
|
200
|
+
return p ? `${MONTH_NAMES[p[1] - 1]} ${p[2]}, ${p[0]}` : str(s);
|
|
201
|
+
}
|
|
202
|
+
function disclaimerHtml() {
|
|
203
|
+
const paras = split(/\n\s*\n/, fs.readFileSync(DISCLAIMER, "utf8")).map((p) => strip(p)).filter((p) => p);
|
|
204
|
+
return '<aside class="disclaimer" role="note" aria-label="AI-generated content disclaimer">' +
|
|
205
|
+
paras.map((p) => `<p>${esc(p)}</p>`).join("") + "</aside>";
|
|
206
|
+
}
|
|
207
|
+
function exhibitHtml(i, ex, spec, fname, mode = "report", scope) {
|
|
208
|
+
const source = ex.source ? `<p class="source"><span>Data source:</span> ${inlineMd(ex.source, scope)}</p>` : "";
|
|
209
|
+
const notes = ex.notes ? `<p class="notes"><span>Notes:</span> ${inlineMd(ex.notes, scope)}</p>` : "";
|
|
210
|
+
const inner = `
|
|
211
|
+
<p class="ex-num">Exhibit ${i}.</p>
|
|
212
|
+
<h3>${esc(ex.title)}</h3>
|
|
213
|
+
${ex.subtitle ? `<p class="ex-sub">${esc(ex.subtitle)}</p>` : ""}
|
|
214
|
+
<div class="legend" data-legend="${i}"></div>
|
|
215
|
+
<div class="chart" data-chart="${i}" role="img" aria-label="${esc(ex.title)}"></div>
|
|
216
|
+
<details class="table-view"${spec === null ? " open" : ""}>
|
|
217
|
+
<summary>Data table</summary>
|
|
218
|
+
<div class="table-wrap" data-table="${i}"></div>
|
|
219
|
+
<button type="button" class="dl" data-download="${i}">Download ${esc(fname)}</button>
|
|
220
|
+
</details>
|
|
221
|
+
${source}${notes}`;
|
|
222
|
+
if (mode === "paper")
|
|
223
|
+
return `\n<section class="exhibit" id="exhibit-${i}">${inner}\n</section>`;
|
|
224
|
+
const text = ex.text ? `<div class="ex-text prose">${blocksMd(ex.text, scope)}</div>` : "";
|
|
225
|
+
return `\n<section class="exhibit-page${text ? "" : " no-text"}" id="exhibit-${i}">${text}<div class="exhibit">${inner}\n</div></section>`;
|
|
226
|
+
}
|
|
227
|
+
// ----------------------------------------------------------------------------- facts
|
|
228
|
+
export const FACT_RE = new RegExp(String.raw `\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:\|\s*(` + FMT_PATTERN + String.raw `))?\s*\}\}`, "g");
|
|
229
|
+
const ANY_PLACEHOLDER = /\{\{[^{}]*\}\}/g;
|
|
230
|
+
const SENTENCE_START = /(?:^|[.!?][*)"”’]*\s+|\n\s*(?:[-*]\s+)?|\*\*\s*)$/;
|
|
231
|
+
function fillFacts(R, facts, mode) {
|
|
232
|
+
const used = [], missing = [], bad = [];
|
|
233
|
+
const rep = (s) => s.replace(FACT_RE, (all, name, fmt0, offset) => {
|
|
234
|
+
if (!(name in facts)) {
|
|
235
|
+
missing.push(name);
|
|
236
|
+
return all;
|
|
237
|
+
}
|
|
238
|
+
const f = facts[name];
|
|
239
|
+
const fmt = fmt0 || f.fmt || "raw";
|
|
240
|
+
let out;
|
|
241
|
+
try {
|
|
242
|
+
out = formatValue(f.value, fmt, mode);
|
|
243
|
+
}
|
|
244
|
+
catch (e) {
|
|
245
|
+
bad.push(`${name}|${fmt}: ${e.message}`);
|
|
246
|
+
return all;
|
|
247
|
+
}
|
|
248
|
+
used.push([name, fmt]); // the format as cited, which may override the default
|
|
249
|
+
if (out[0] && out[0] !== out[0].toUpperCase() && SENTENCE_START.test(s.slice(0, offset)))
|
|
250
|
+
out = out[0].toUpperCase() + out.slice(1);
|
|
251
|
+
return out;
|
|
252
|
+
});
|
|
253
|
+
const walk = (x) => {
|
|
254
|
+
if (typeof x === "string")
|
|
255
|
+
return rep(x);
|
|
256
|
+
if (Array.isArray(x))
|
|
257
|
+
return x.map(walk);
|
|
258
|
+
if (x && typeof x === "object")
|
|
259
|
+
return Object.fromEntries(Object.entries(x).map(([k, v]) => [k, ["data", "chart", "path"].includes(k) ? v : walk(v)]));
|
|
260
|
+
return x;
|
|
261
|
+
};
|
|
262
|
+
const keepKeys = ["data_files", "facts", "provenance", "request", "sources", "replication"];
|
|
263
|
+
const out = {};
|
|
264
|
+
for (const [k, v] of Object.entries(R))
|
|
265
|
+
out[k] = keepKeys.includes(k) ? v : walk(v);
|
|
266
|
+
if (missing.length)
|
|
267
|
+
fail("report.json cites facts that are not in facts.json: " + [...new Set(missing)].sort().join(", "));
|
|
268
|
+
if (bad.length)
|
|
269
|
+
fail("fact formats that cannot be applied: " + bad.join("; "));
|
|
270
|
+
return [out, used];
|
|
271
|
+
}
|
|
272
|
+
const fileSha = (p) => crypto.createHash("sha256").update(fs.readFileSync(p)).digest("hex");
|
|
273
|
+
// ----------------------------------------------------------------------------- sources and package
|
|
274
|
+
class Ctx {
|
|
275
|
+
R;
|
|
276
|
+
base;
|
|
277
|
+
ids;
|
|
278
|
+
policies = {};
|
|
279
|
+
prov = {};
|
|
280
|
+
withheld = {};
|
|
281
|
+
notices;
|
|
282
|
+
constructor(R, base) {
|
|
283
|
+
this.R = R;
|
|
284
|
+
this.base = base;
|
|
285
|
+
this.ids = PS.usedSources(base, R);
|
|
286
|
+
try {
|
|
287
|
+
this.policies = PS.loadPolicies(this.ids);
|
|
288
|
+
}
|
|
289
|
+
catch (e) {
|
|
290
|
+
fail(e.message);
|
|
291
|
+
}
|
|
292
|
+
for (const sid of this.ids)
|
|
293
|
+
this.prov[sid] = PS.provenance(base, sid, R);
|
|
294
|
+
for (const [sid, P] of Object.entries(this.policies))
|
|
295
|
+
this.withheld[sid] = PS.withheldKeys(base, sid, P, R);
|
|
296
|
+
this.notices = pol.notices(Object.values(this.policies));
|
|
297
|
+
}
|
|
298
|
+
title(sid) {
|
|
299
|
+
return this.policies[sid].title;
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
const isFile = (p) => fs.existsSync(p) && fs.statSync(p).isFile();
|
|
303
|
+
/** Top-level files of the package: the request, facts, decisions, report and every exhibit CSV and recipe. */
|
|
304
|
+
function replicationFiles(R, base) {
|
|
305
|
+
const repl = R.replication || {};
|
|
306
|
+
const want = [...(repl.files || ["spec.json", "plan.json", "analysis.py", "facts.json", "decisions.json", "report.json"])];
|
|
307
|
+
want.push(...(R.exhibits || []).filter((e) => e.data).map((e) => e.data));
|
|
308
|
+
const exdir = path.join(base, "exhibits");
|
|
309
|
+
if (fs.existsSync(exdir))
|
|
310
|
+
want.push(...fs.readdirSync(exdir).filter((f) => f.endsWith(".csv") || f.endsWith(".json")).sort().map((f) => `exhibits/${f}`));
|
|
311
|
+
want.push(...(repl.extra || []));
|
|
312
|
+
return [...new Set(want)].filter((w) => isFile(path.join(base, w))).map((w) => w.replaceAll("\\", "/"));
|
|
313
|
+
}
|
|
314
|
+
function tree(base, top) {
|
|
315
|
+
const out = [];
|
|
316
|
+
const walk = (d) => {
|
|
317
|
+
if (!fs.existsSync(d))
|
|
318
|
+
return;
|
|
319
|
+
for (const e of fs.readdirSync(d, { withFileTypes: true })) {
|
|
320
|
+
const p = path.join(d, e.name);
|
|
321
|
+
if (e.isDirectory())
|
|
322
|
+
walk(p);
|
|
323
|
+
else if (e.isFile())
|
|
324
|
+
out.push(path.relative(base, p).replaceAll("\\", "/"));
|
|
325
|
+
}
|
|
326
|
+
};
|
|
327
|
+
walk(path.join(base, top));
|
|
328
|
+
return out.sort();
|
|
329
|
+
}
|
|
330
|
+
function planPackage(ctx) {
|
|
331
|
+
const names = [...replicationFiles(ctx.R, ctx.base), ...tree(ctx.base, "data"), ...tree(ctx.base, "reproduce")];
|
|
332
|
+
const members = [], excluded = [];
|
|
333
|
+
for (const n of [...new Set(names)]) {
|
|
334
|
+
if (n.endsWith(".pyc") || n.includes("/__pycache__/"))
|
|
335
|
+
continue;
|
|
336
|
+
const d = PS.decide(n, ctx.base, ctx.policies, ctx.R, ctx.withheld);
|
|
337
|
+
if (d.ok)
|
|
338
|
+
members.push({ name: n, path: path.join(ctx.base, n), filter: d.filter });
|
|
339
|
+
else
|
|
340
|
+
excluded.push({ name: n, reason: d.reason, policy: d.policy });
|
|
341
|
+
}
|
|
342
|
+
return [members, excluded];
|
|
343
|
+
}
|
|
344
|
+
/** The request to rebuild one source's data: its provenance "spec", report.json sources[].spec, or spec.json. */
|
|
345
|
+
function sourceSpec(ctx, sid) {
|
|
346
|
+
const P = ctx.prov[sid][1];
|
|
347
|
+
if (P?.spec && typeof P.spec === "object" && !Array.isArray(P.spec) && Object.keys(P.spec).length)
|
|
348
|
+
return P.spec;
|
|
349
|
+
for (const s of ctx.R.sources || []) {
|
|
350
|
+
if (s && typeof s === "object" && (s.source || s.id) === sid && s.spec && typeof s.spec === "object")
|
|
351
|
+
return s.spec;
|
|
352
|
+
}
|
|
353
|
+
const sp = path.join(ctx.base, "spec.json");
|
|
354
|
+
if (ctx.ids.length === 1 && fs.existsSync(sp))
|
|
355
|
+
return JSON.parse(readText(sp));
|
|
356
|
+
return null;
|
|
357
|
+
}
|
|
358
|
+
/** {bash, powershell} rel paths of the download scripts in reproduce/<id>/ (empty when not both). */
|
|
359
|
+
function reproduceScripts(base, sid) {
|
|
360
|
+
const d = path.join(base, "reproduce", sid);
|
|
361
|
+
const out = {};
|
|
362
|
+
for (const [kind, ext] of Object.entries(SCRIPT_KINDS)) {
|
|
363
|
+
const hits = fs.existsSync(d) ? fs.readdirSync(d).filter((f) => f.endsWith(ext)).sort() : [];
|
|
364
|
+
if (hits.length)
|
|
365
|
+
out[kind] = `reproduce/${sid}/${hits[0]}`;
|
|
366
|
+
}
|
|
367
|
+
return Object.keys(out).length === Object.keys(SCRIPT_KINDS).length ? out : {};
|
|
368
|
+
}
|
|
369
|
+
function manualItemsFor(ctx, sid) {
|
|
370
|
+
try {
|
|
371
|
+
const m = JSON.parse(readText(path.join(ctx.base, "manual_download", "manifest.json")));
|
|
372
|
+
return (m.items || []).filter((it) => it.source === sid);
|
|
373
|
+
}
|
|
374
|
+
catch {
|
|
375
|
+
return [];
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
/** reproduce/<id>/ for a definition_only source: download scripts with the request built in, spec.json and
|
|
379
|
+
* REPRODUCE.md. No data files. */
|
|
380
|
+
function writeReproduce(ctx, sid) {
|
|
381
|
+
const P = ctx.policies[sid];
|
|
382
|
+
const d = path.join(ctx.base, "reproduce", sid);
|
|
383
|
+
fs.rmSync(d, { recursive: true, force: true });
|
|
384
|
+
fs.mkdirSync(d, { recursive: true });
|
|
385
|
+
const spec = sourceSpec(ctx, sid);
|
|
386
|
+
if (spec !== null)
|
|
387
|
+
fs.writeFileSync(path.join(d, "spec.json"), JSON.stringify(spec, null, 2) + "\n");
|
|
388
|
+
const rel = `reproduce/${sid}`;
|
|
389
|
+
const items = manualItemsFor(ctx, sid);
|
|
390
|
+
if (items.length)
|
|
391
|
+
manual.writeScripts(d, items, `download_${sid}`);
|
|
392
|
+
const copied = [];
|
|
393
|
+
if (!Object.keys(reproduceScripts(ctx.base, sid)).length) { // no scripts: the analysis's own bundle, as is
|
|
394
|
+
const mdir = path.join(ctx.base, "manual_download", sid);
|
|
395
|
+
for (const f of tree(ctx.base, `manual_download/${sid}`)) {
|
|
396
|
+
const src = path.join(ctx.base, f), relF = path.relative(mdir, src).replaceAll("\\", "/");
|
|
397
|
+
if (PS.isMicrodata(P, relF) || fs.statSync(src).size > PS.LIMITS.MICRO_MAX || path.basename(f).startsWith("extract_number_"))
|
|
398
|
+
continue;
|
|
399
|
+
fs.mkdirSync(path.dirname(path.join(d, "manual", relF)), { recursive: true });
|
|
400
|
+
fs.copyFileSync(src, path.join(d, "manual", relF));
|
|
401
|
+
copied.push(relF);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
const scripts = reproduceScripts(ctx.base, sid);
|
|
405
|
+
const inc = tree(ctx.base, `data/${sid}`).filter((f) => PS.decide(f, ctx.base, ctx.policies, ctx.R, ctx.withheld).ok);
|
|
406
|
+
const specArg = spec !== null ? `${rel}/spec.json` : `<spec.json for ${sid}>`;
|
|
407
|
+
const a = P.auth || {};
|
|
408
|
+
const keys = ["required", "optional"].includes(a.required) && a.env && a.signup_url ? [[a.env, a.signup_url]] : [];
|
|
409
|
+
const lines = [`# Reproduce the ${P.title} data`, "",
|
|
410
|
+
`The terms of this source do not allow its data files to be shared (${P.redistribution.why || ""}). This folder holds ` +
|
|
411
|
+
"what is needed to download the same data under your own account; the analysis then runs unchanged.", ""];
|
|
412
|
+
if (Object.keys(scripts).length) {
|
|
413
|
+
const stem = path.posix.basename(scripts.bash).replace(/\.[^.]*$/, "");
|
|
414
|
+
const scriptDir = path.posix.dirname(scripts.bash);
|
|
415
|
+
lines.push("## Download: one command", "", ...manual.runSection(stem, keys, scriptDir, "spec.json"));
|
|
416
|
+
lines.push("## Then: package the files and rerun the analysis", "", "With the data-analyst-mcp server, in a session whose workspace holds this package (on a remote server, upload the " +
|
|
417
|
+
"downloaded files on its upload page first):", "", "```", `ingest {"source": "${sid}", "spec": <the content of ${specArg}>, "files": ["${scriptDir}"]}`, "rerun {}", "```", "");
|
|
418
|
+
}
|
|
419
|
+
else {
|
|
420
|
+
lines.push("## With the data-analyst-mcp server", "", "```", `fetch {"source": "${sid}", "spec": <the content of ${specArg}>}`, "rerun {}", "```", "", "fetch says which route works for you (a key, a keyless route, or a manual-download bundle) and downloads the data " +
|
|
421
|
+
`into \`data/${sid}/\`. If you download the files by hand, package them with ` +
|
|
422
|
+
`\`ingest {"source": "${sid}", "spec": <the content of ${specArg}>, "files": [<the files>]}\`.`, "");
|
|
423
|
+
}
|
|
424
|
+
lines.push("## By hand", "", P.routes?.manual?.how || "", "");
|
|
425
|
+
if (copied.length)
|
|
426
|
+
lines.push("The manual-download bundle of the original analysis is in `manual/`.", "");
|
|
427
|
+
if (inc.length)
|
|
428
|
+
lines.push("## The original request", "", "Shared in the package (no data):", "", ...inc.map((f) => `- \`${f}\``), "");
|
|
429
|
+
if (keys.length && !Object.keys(scripts).length)
|
|
430
|
+
lines.push(`A free key (${keys[0][0]}) is available at ${keys[0][1]}.`, "");
|
|
431
|
+
fs.writeFileSync(path.join(d, "REPRODUCE.md"), lines.join("\n"));
|
|
432
|
+
}
|
|
433
|
+
const REPRODUCE_ORDER = ["REPRODUCE.md", ".sh", ".ps1", "spec.json", ".json", ".md"];
|
|
434
|
+
function reproduceShown(base, sid) {
|
|
435
|
+
const rank = (f) => {
|
|
436
|
+
const i = REPRODUCE_ORDER.findIndex((k) => f.endsWith(k));
|
|
437
|
+
return i < 0 ? REPRODUCE_ORDER.length : i;
|
|
438
|
+
};
|
|
439
|
+
return tree(base, `reproduce/${sid}`).filter((f) => !f.endsWith(".txt")).sort((a, b) => {
|
|
440
|
+
const [x, y] = [path.posix.basename(a), path.posix.basename(b)];
|
|
441
|
+
return rank(x) - rank(y) || (x < y ? -1 : x > y ? 1 : 0);
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
function replicationReadme(ctx, excluded) {
|
|
445
|
+
const R = ctx.R;
|
|
446
|
+
const lines = [`# Replication materials: ${R.title ?? ""}`, "",
|
|
447
|
+
"This package holds what is needed to reproduce every number, exhibit and chart in the report, within the terms of " +
|
|
448
|
+
"each data source.", "",
|
|
449
|
+
"1. Use the data-analyst-mcp server (an MCP server for U.S. government data) in a session whose workspace holds this " +
|
|
450
|
+
"package's files.",
|
|
451
|
+
"2. Get the data (by source, below).",
|
|
452
|
+
"3. Recompute: `exhibits/lineage.json` holds the recipe of every exhibit (its inputs with their SHA-256 checksums and " +
|
|
453
|
+
"the `compute` steps). The server's `rerun` tool replays every recipe and recomputes every fact in `facts.json` from " +
|
|
454
|
+
"its exhibit cell.",
|
|
455
|
+
"4. Compare the recomputed values with `facts.json`. They must be identical when the data are the ones used (see the " +
|
|
456
|
+
"retrieval dates below).",
|
|
457
|
+
"5. Rebuild and audit the report: `build_report` with `report.json`, then `audit_report` with `rerun: true`.", "",
|
|
458
|
+
"## Data by source", ""];
|
|
459
|
+
for (const sid of ctx.ids) {
|
|
460
|
+
const P = ctx.policies[sid];
|
|
461
|
+
const mode = P.redistribution.mode;
|
|
462
|
+
const [provRel, prov] = ctx.prov[sid];
|
|
463
|
+
const when = prov.retrieved || prov.downloaded || "";
|
|
464
|
+
let head = `- **${P.title}** (\`data/${sid}/\`)` + (when ? `, retrieved ${dayDate(when)}` : "") + ": ";
|
|
465
|
+
if (mode === "allowed")
|
|
466
|
+
head += "the data are included as retrieved (the vintage used).";
|
|
467
|
+
else if (mode === "aggregates_only")
|
|
468
|
+
head += "tables of estimates are included; row-level files stay with the analyst.";
|
|
469
|
+
else {
|
|
470
|
+
head += "no data files are included; the terms forbid sharing them. ";
|
|
471
|
+
const sc = reproduceScripts(ctx.base, sid);
|
|
472
|
+
if (Object.keys(sc).length) {
|
|
473
|
+
const c = manual.runLines(path.posix.basename(sc.bash).replace(/\.[^.]*$/, ""), path.posix.dirname(sc.bash));
|
|
474
|
+
head += `Download the same data under your own account with one command: \`${c.bash}\` (macOS, Linux) or ` +
|
|
475
|
+
`\`${c.powershell}\` (Windows); the request is built into the scripts. Then follow \`reproduce/${sid}/REPRODUCE.md\`.`;
|
|
476
|
+
}
|
|
477
|
+
else
|
|
478
|
+
head += `Download the same data under your own account with \`reproduce/${sid}/REPRODUCE.md\`.`;
|
|
479
|
+
}
|
|
480
|
+
lines.push(head + ` Rule: ${mode}. ${P.redistribution.why || ""}`);
|
|
481
|
+
if (ctx.withheld[sid]?.size) {
|
|
482
|
+
const w = P.redistribution.withhold;
|
|
483
|
+
lines.push(` - Rows of ${[...ctx.withheld[sid]].sort().join(", ")} (${w.key}) are left out of the shared files: their ` +
|
|
484
|
+
"terms do not allow redistribution. Download them again with your own access.");
|
|
485
|
+
}
|
|
486
|
+
if (mode !== "definition_only") {
|
|
487
|
+
lines.push(` - Current data: call \`fetch\` with source \`${sid}\`, the spec in ` + (provRel ? `\`${provRel}\`` : "spec.json") +
|
|
488
|
+
" and `force: true`. Sources revise published data, so new numbers can differ from the report.");
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
const out = excluded.filter((e) => !e.name.startsWith("lib/") && !e.name.startsWith("reproduce/"));
|
|
492
|
+
if (out.length) {
|
|
493
|
+
lines.push("", "## Not included", "", ...out.slice(0, 60).map((e) => `- \`${e.name}\`: ${e.reason}`));
|
|
494
|
+
if (out.length > 60)
|
|
495
|
+
lines.push(`- and ${out.length - 60} more files`);
|
|
496
|
+
}
|
|
497
|
+
lines.push("", "## Contents", "", "- `spec.json` / `plan.json`: the data request; `exhibits/lineage.json`: the recipe of every exhibit; `facts.json`: " +
|
|
498
|
+
"every number in the text, with its source cell or derivation; `decisions.json`: the method record.", "- `data/<id>/`: each source's files as its terms allow, with `request.json` (exact requests without keys, retrieval " +
|
|
499
|
+
"time) and `README.md` (citation, terms).", "- `exhibits/`: the data for each exhibit; `reproduce/`: download instructions for sources whose data may not be shared.", "");
|
|
500
|
+
const cites = R.citation ? [str(R.citation)] : [];
|
|
501
|
+
if (!cites.length) {
|
|
502
|
+
for (const s of ctx.ids) {
|
|
503
|
+
const P = ctx.prov[s][1];
|
|
504
|
+
const v = P.vintage;
|
|
505
|
+
cites.push(pol.citation(ctx.policies[s], str(P.route || ""), { retrieved: String(P.retrieved || P.downloaded || "").slice(0, 10),
|
|
506
|
+
dataset: str(P.spec?.description || ctx.title(s)), vintage: v === null || v === undefined || v === "latest" ? "" : str(v) }));
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
if (cites.length)
|
|
510
|
+
lines.push("## Data citations", "", ...cites, "");
|
|
511
|
+
if (ctx.notices.length)
|
|
512
|
+
lines.push("## Notices", "", ...ctx.notices.map((n) => `- ${n}`), "");
|
|
513
|
+
return lines.join("\n");
|
|
514
|
+
}
|
|
515
|
+
function makePackage(ctx) {
|
|
516
|
+
const [members, excluded] = planPackage(ctx);
|
|
517
|
+
const zp = path.join(ctx.base, "replication_package.zip");
|
|
518
|
+
const files = {};
|
|
519
|
+
files["REPLICATION.md"] = [Buffer.from(replicationReadme(ctx, excluded), "utf8"), { level: 6 }];
|
|
520
|
+
for (const m of members) {
|
|
521
|
+
const data = m.filter ? Buffer.from(PS.filteredCsv(m.path, m.filter[0], m.filter[1])[0], "utf8") : fs.readFileSync(m.path);
|
|
522
|
+
files[m.name] = [data, { level: 6 }];
|
|
523
|
+
}
|
|
524
|
+
fs.writeFileSync(zp + ".part", zipSync(files));
|
|
525
|
+
fs.renameSync(zp + ".part", zp);
|
|
526
|
+
// a last guard: the policy check of core on everything that went in
|
|
527
|
+
const names = members.map((m) => m.name);
|
|
528
|
+
const bad = new Set();
|
|
529
|
+
for (const [sid, P] of Object.entries(ctx.policies)) {
|
|
530
|
+
const scope = names.filter((n) => n.startsWith(`data/${sid}/`) || !["data/", "exhibits/", "lib/"].some((x) => n.startsWith(x)));
|
|
531
|
+
for (const b of PS.checkPackage(P, scope))
|
|
532
|
+
bad.add(b);
|
|
533
|
+
}
|
|
534
|
+
if (bad.size) {
|
|
535
|
+
fs.rmSync(zp, { force: true });
|
|
536
|
+
fail(`the package would hold files the source terms forbid: ${str([...bad].sort().slice(0, 5))}`);
|
|
537
|
+
}
|
|
538
|
+
return ["replication_package.zip", members, excluded];
|
|
539
|
+
}
|
|
540
|
+
// ----------------------------------------------------------------------------- data and methods: vintage, notices
|
|
541
|
+
const SERIES_COLS = ["series_id", "title", "units", "frequency", "seasonal_adjustment", "first_date", "last_date", "vintage",
|
|
542
|
+
"last_updated", "n_future", "n_missing", "converted_to", "transformation", "copyright"];
|
|
543
|
+
const SKIP_SCAN = ["series.csv", "data_dictionary.csv", "codebook.csv", "variables.csv", "notes.csv"];
|
|
544
|
+
function periodText(lp) {
|
|
545
|
+
const m = /^(\d{4}) ?([MQS])(\d\d)$/.exec(str(lp));
|
|
546
|
+
if (!m)
|
|
547
|
+
return str(lp);
|
|
548
|
+
const [y, k, n] = [m[1], m[2], Number(m[3])];
|
|
549
|
+
if (k === "M" && n >= 1 && n <= 12)
|
|
550
|
+
return `${MONTH_NAMES[n - 1]} ${y}`;
|
|
551
|
+
if (k === "Q" && n >= 1 && n <= 4)
|
|
552
|
+
return `the ${["first", "second", "third", "fourth"][n - 1]} quarter of ${y}`;
|
|
553
|
+
if (k === "S" && (n === 1 || n === 2))
|
|
554
|
+
return `the ${n === 1 ? "first" : "second"} half of ${y}`;
|
|
555
|
+
return str(lp);
|
|
556
|
+
}
|
|
557
|
+
function vintText(v0) {
|
|
558
|
+
const v = str(v0);
|
|
559
|
+
const named = { latest: "latest available", first_release: "first release", all: "full revision history" };
|
|
560
|
+
if (v in named)
|
|
561
|
+
return named[v];
|
|
562
|
+
if (v.includes(":") || /^\d{4}-\d\d-\d\d/.test(v)) {
|
|
563
|
+
const tail = v.includes(":") ? v.split(":").slice(1).join(":") : v;
|
|
564
|
+
return "as published on " + tail.split(",").map(dayDate).join(", ");
|
|
565
|
+
}
|
|
566
|
+
return v;
|
|
567
|
+
}
|
|
568
|
+
function flagsAndGaps(base, sid, P) {
|
|
569
|
+
const used = new Set(P.used_series || []);
|
|
570
|
+
const prelim = new Set(), gaps = new Set();
|
|
571
|
+
for (const rel of tree(base, `data/${sid}`)) {
|
|
572
|
+
const full = path.join(base, rel);
|
|
573
|
+
if (!rel.endsWith(".csv") || SKIP_SCAN.includes(path.posix.basename(rel)) || rel.includes("/raw/"))
|
|
574
|
+
continue;
|
|
575
|
+
if (fs.statSync(full).size > PS.LIMITS.MICRO_MAX)
|
|
576
|
+
continue;
|
|
577
|
+
const cols = PS.csvHeader(full);
|
|
578
|
+
if (!cols.includes("preliminary") && !(cols.includes("value") && cols.includes("series_id")))
|
|
579
|
+
continue;
|
|
580
|
+
for (const r of readDict(readText(full)).rows) {
|
|
581
|
+
const sidv = r.series_id ?? "";
|
|
582
|
+
if (used.size && sidv && !used.has(sidv))
|
|
583
|
+
continue;
|
|
584
|
+
const per = r.period_name ? `${r.period_name} ${r.year}` : r.date || r.period || r.time_period || "";
|
|
585
|
+
if (["1", "true", "yes", "p"].includes(strip(String(r.preliminary ?? "")).toLowerCase()))
|
|
586
|
+
prelim.add(strip(`${sidv} ${per}`));
|
|
587
|
+
if (cols.includes("value") && cols.includes("series_id") && strip(String(r.value ?? "")) === "")
|
|
588
|
+
gaps.add(per);
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
return [[...prelim].sort(), [...gaps].sort()];
|
|
592
|
+
}
|
|
593
|
+
function vintageHtml(ctx) {
|
|
594
|
+
const parts = [], tables = [];
|
|
595
|
+
for (const sid of ctx.ids) {
|
|
596
|
+
const [provRel, P] = ctx.prov[sid];
|
|
597
|
+
const T = ctx.title(sid);
|
|
598
|
+
if (!provRel) {
|
|
599
|
+
parts.push(`<p class='vintage'>${esc(T)}: no provenance file was found in <code>data/${esc(sid)}/</code>, so the retrieval date is not recorded.</p>`);
|
|
600
|
+
continue;
|
|
601
|
+
}
|
|
602
|
+
const when = P.retrieved || P.downloaded || "";
|
|
603
|
+
const route = P.route || "", acq = P.acquired || "";
|
|
604
|
+
let txt = `${T}: retrieved on ${when ? dayDate(when) : "an unrecorded date"}`;
|
|
605
|
+
if (route || acq)
|
|
606
|
+
txt += " by the " + [...new Set([str(route), str(acq)])].filter((x) => x).join(" / ") + " route";
|
|
607
|
+
if (P.api_version)
|
|
608
|
+
txt += ` (${str(P.api_version)})`;
|
|
609
|
+
const vf = PS.vintageFields(P);
|
|
610
|
+
const v = vf.vintage;
|
|
611
|
+
if (v && v !== "latest")
|
|
612
|
+
txt += "; vintage " + (Array.isArray(v) ? v.slice(0, 4).map(str).join("; ") : vintText(v));
|
|
613
|
+
if (vf.series_vintages)
|
|
614
|
+
txt += "; series vintages " + vf.series_vintages.map(str).join(", ");
|
|
615
|
+
const lp = P.latest_period || P.latest;
|
|
616
|
+
if (lp && typeof lp === "object" && !Array.isArray(lp))
|
|
617
|
+
txt += "; latest observation " + Object.entries(lp).map(([k, x]) => `${k} ${str(x)}`).join("; ");
|
|
618
|
+
else if (lp)
|
|
619
|
+
txt += `; the latest period in the data is ${periodText(lp)}`;
|
|
620
|
+
if (P.spec?.as_of)
|
|
621
|
+
txt += `; data cut-off ${str(P.spec.as_of)}`;
|
|
622
|
+
parts.push(`<p class='vintage'>${esc(txt)}.</p>`);
|
|
623
|
+
const [prelim, gaps] = flagsAndGaps(ctx.base, sid, P);
|
|
624
|
+
if (prelim.length)
|
|
625
|
+
parts.push("<p class='vintage'>Values marked preliminary at retrieval: " + esc(prelim.slice(0, 12).join("; ")) + (prelim.length > 12 ? "; and others" : "") + ".</p>");
|
|
626
|
+
if (gaps.length) {
|
|
627
|
+
parts.push("<p class='vintage'>Periods with no published value in the series used: " + esc(gaps.slice(0, 12).join(", ")) +
|
|
628
|
+
(gaps.length > 12 ? " and others" : "") + ". These gaps were left empty, not interpolated.</p>");
|
|
629
|
+
}
|
|
630
|
+
const dropped = P.options_dropped || P.v2_options_dropped;
|
|
631
|
+
if (dropped && dropped.length)
|
|
632
|
+
parts.push("<p class='vintage'>Requested but unavailable on the route used: " + esc(dropped.map(str).join(", ")) + ".</p>");
|
|
633
|
+
for (const pr of (P.problems || []).slice(0, 5))
|
|
634
|
+
parts.push(`<p class='vintage'>${esc(T)}: ${esc(pr)}</p>`);
|
|
635
|
+
const sp = path.join(ctx.base, "data", sid, "series.csv");
|
|
636
|
+
if (fs.existsSync(sp)) {
|
|
637
|
+
const meta = readDict(readText(sp)).rows;
|
|
638
|
+
const cols = SERIES_COLS.filter((c) => meta.length && c in meta[0]);
|
|
639
|
+
if (meta.length && cols.includes("series_id")) {
|
|
640
|
+
const head = cols.map((c) => `<th>${esc(c.replaceAll("_", " ").replace(/^./, (x) => x.toUpperCase()))}</th>`).join("");
|
|
641
|
+
const body = meta.map((m) => "<tr>" + cols.map((c) => `<td>${esc(c === "vintage" && m[c] ? vintText(m[c]) : m[c] ?? "")}</td>`).join("") + "</tr>").join("");
|
|
642
|
+
tables.push(`<div class='table-wrap'><table class='audit'><thead><tr>${head}</tr></thead><tbody>${body}</tbody></table></div>`);
|
|
643
|
+
}
|
|
644
|
+
const cites = meta.filter((m) => m.citation).map((m) => m.citation);
|
|
645
|
+
if (cites.length)
|
|
646
|
+
tables.push("<h3>Series Citations</h3><ul class='files'>" + cites.map((c) => `<li>${esc(c)}</li>`).join("") + "</ul>");
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
if (!parts.length)
|
|
650
|
+
return "";
|
|
651
|
+
parts.push("<p class='vintage'>Sources revise their published data (benchmark and seasonal revisions, later releases of " +
|
|
652
|
+
"preliminary values), so a later retrieval can differ from the data used here.</p>");
|
|
653
|
+
return "<h3>Data Vintage</h3>" + parts.join("") + tables.join("");
|
|
654
|
+
}
|
|
655
|
+
const noticesHtml = (ctx) => (ctx.notices.length ? "<h3>Required Notices</h3><ul class='notices'>" +
|
|
656
|
+
ctx.notices.map((n) => `<li>${esc(n)}</li>`).join("") + "</ul>" : "");
|
|
657
|
+
function footerHtml(ctx) {
|
|
658
|
+
const srcs = ctx.ids.map((s) => ctx.title(s)).join("; ");
|
|
659
|
+
return (srcs ? `<p>Data: ${esc(srcs)}. Sources revise published data; later retrievals can differ.</p>` : "") +
|
|
660
|
+
ctx.notices.map((n) => `<p>${esc(n)}</p>`).join("");
|
|
661
|
+
}
|
|
662
|
+
// ----------------------------------------------------------------------------- appendix blocks
|
|
663
|
+
function referencesHtml(R) {
|
|
664
|
+
const refs = R.references || [];
|
|
665
|
+
if (!refs.length)
|
|
666
|
+
return "";
|
|
667
|
+
const items = refs.map((r) => {
|
|
668
|
+
const url = r.url ? ` <a href='${esc(r.url)}'>${esc(r.url)}</a>` : "";
|
|
669
|
+
const acc = r.accessed ? ` (accessed ${esc(r.accessed)})` : "";
|
|
670
|
+
const use = r.used_for ? `<br><span class='muted'>Used for: ${esc(r.used_for)}</span>` : "";
|
|
671
|
+
return `<li id='ref-${esc(r.id)}'>${inlineMd(r.citation ?? "")}.${url}${acc}${use}</li>`;
|
|
672
|
+
});
|
|
673
|
+
return `<section class='prose references' id='references'><h2>References</h2><ol>${items.join("")}</ol></section>`;
|
|
674
|
+
}
|
|
675
|
+
function assumptionsHtml(R) {
|
|
676
|
+
const rows = R.assumptions || [];
|
|
677
|
+
if (!rows.length)
|
|
678
|
+
return "";
|
|
679
|
+
const judged = rows.some((a) => String(a.basis || "").toLowerCase() === "judgment");
|
|
680
|
+
const why = (a) => String(a.basis || "").toLowerCase() !== "judgment" ? "" :
|
|
681
|
+
`<td>${inlineMd(a.why_not_empirical ?? "")} Sensitivity: <code>exhibits/sensitivity.csv</code>, ${esc(a.sensitivity || a.parameter || "")}.</td>`;
|
|
682
|
+
const trs = rows.map((a) => `<tr><td>${inlineMd(a.parameter ?? "")}</td><td>${inlineMd(a.value ?? "")}</td>` +
|
|
683
|
+
`<td>${esc(a.basis ?? "")}</td><td>${inlineMd(a.evidence ?? "")}</td>${judged ? why(a) || "<td></td>" : ""}</tr>`).join("");
|
|
684
|
+
return "<h3>Key Assumptions and Parameters</h3><div class='table-wrap'><table class='audit'>" +
|
|
685
|
+
`<thead><tr><th>Parameter</th><th>Value</th><th>Basis</th><th>Evidence</th>${judged ? "<th>Why Not Empirical</th>" : ""}</tr></thead>` +
|
|
686
|
+
`<tbody>${trs}</tbody></table></div>`;
|
|
687
|
+
}
|
|
688
|
+
function decisionsHtml(R) {
|
|
689
|
+
const out = [];
|
|
690
|
+
const qs = R.questions || [];
|
|
691
|
+
if (qs.length) {
|
|
692
|
+
const how = { dont_know: "Don't know", skipped: "Skipped (treated as Don't know)", not_asked: "Not asked" };
|
|
693
|
+
const trs = qs.map((q) => {
|
|
694
|
+
const no = [null, undefined, "", ...MA.NO_ANSWER].includes(q.answer);
|
|
695
|
+
let ans = no ? how[q.how || q.answer || "not_asked"] ?? "Don't know" : str(q.answer);
|
|
696
|
+
if (!MA.asked(q) && q.reason_code === "settled_by_request" && q.request)
|
|
697
|
+
ans = `Settled by the request: “${q.request}”`;
|
|
698
|
+
else if (!MA.asked(q) && (q.reason_code || q.reason))
|
|
699
|
+
ans += ` (${MA.NOT_ASKED_CODES[q.reason_code] ?? (q.reason || q.reason_code)})`;
|
|
700
|
+
let dec = q.resolution || (no ? "" : "As answered");
|
|
701
|
+
if (q.blocking && q.rationale)
|
|
702
|
+
dec += `; ${q.rationale}`;
|
|
703
|
+
return `<tr><td>${inlineMd(q.question ?? q.id ?? "")}</td><td>${inlineMd(ans)}</td><td>${inlineMd(dec)}</td></tr>`;
|
|
704
|
+
});
|
|
705
|
+
out.push("<h3>Questions and Decisions</h3><div class='table-wrap'><table class='audit'><thead><tr><th>Question</th>" +
|
|
706
|
+
`<th>Answer</th><th>How It Was Decided</th></tr></thead><tbody>${trs.join("")}</tbody></table></div>`);
|
|
707
|
+
}
|
|
708
|
+
const lit = R.literature_search || [];
|
|
709
|
+
if (lit.length) {
|
|
710
|
+
const items = lit.map((s) => {
|
|
711
|
+
if (s.unavailable) {
|
|
712
|
+
const why = MA.UNAVAILABLE_CODES[s.reason_code] ?? (s.reason_code || "");
|
|
713
|
+
return `<li>Literature search was not available: ${inlineMd(why + (s.reason ? ` (${s.reason})` : ""))}</li>`;
|
|
714
|
+
}
|
|
715
|
+
const found = (s.found || []).map((i) => `[@${i}]`).join(", ") || "nothing relevant";
|
|
716
|
+
return `<li>“${esc(s.query ?? "")}” (${esc(s.date ?? "")}${s.tool ? `, ${esc(s.tool)}` : ""}): ${inlineMd(found)}.` +
|
|
717
|
+
`${s.note ? ` ${inlineMd(s.note)}` : ""}</li>`;
|
|
718
|
+
});
|
|
719
|
+
out.push(`<h3>Literature Searched</h3><ul>${items.join("")}</ul>`);
|
|
720
|
+
}
|
|
721
|
+
const disc = MA.escapes(R).map(([k]) => (R.disclosures || {})[k]).filter((t) => t);
|
|
722
|
+
if (disc.length)
|
|
723
|
+
out.push("<h3>Not Verified by the Audit</h3><ul>" + disc.map((t) => `<li>${inlineMd(t)}</li>`).join("") + "</ul>");
|
|
724
|
+
return out.join("");
|
|
725
|
+
}
|
|
726
|
+
function dataNeedsHtml(R) {
|
|
727
|
+
const needs = [...(R.data_needs || [])].sort((a, b) => Number(!a.from_user) - Number(!b.from_user));
|
|
728
|
+
if (!needs.length) {
|
|
729
|
+
return R.data_needs_note ? `<h3 id='data-needs'>Data That Would Improve These Estimates</h3><p>${inlineMd(R.data_needs_note)}</p>` : "";
|
|
730
|
+
}
|
|
731
|
+
const trs = needs.map((d) => `<tr><td>${inlineMd(d.data ?? "")}</td><td>${inlineMd(d.effect ?? "")}</td>` +
|
|
732
|
+
`<td>${inlineMd(d.how_to_get ?? "")}${d.from_user ? " (from you)" : ""}</td></tr>`).join("");
|
|
733
|
+
return "<h3 id='data-needs'>Data That Would Improve These Estimates</h3><div class='table-wrap'><table class='audit'>" +
|
|
734
|
+
`<thead><tr><th>Data</th><th>What It Would Improve</th><th>How to Get It</th></tr></thead><tbody>${trs}</tbody></table></div>`;
|
|
735
|
+
}
|
|
736
|
+
function replicationHtml(ctx, outDir, pkg, members, excluded, mode) {
|
|
737
|
+
const base = ctx.base;
|
|
738
|
+
const rel = (p) => path.relative(outDir, path.join(base, p)).replaceAll("\\", "/");
|
|
739
|
+
const shown = replicationFiles(ctx.R, base);
|
|
740
|
+
for (const sid of ctx.ids)
|
|
741
|
+
shown.push(...[ctx.prov[sid][0], `data/${sid}/README.md`].filter((x) => !!x));
|
|
742
|
+
const defonly = ctx.ids.filter((sid) => ctx.policies[sid].redistribution.mode === "definition_only");
|
|
743
|
+
for (const sid of defonly)
|
|
744
|
+
shown.push(...reproduceShown(base, sid));
|
|
745
|
+
const inPkg = new Map(members.map((m) => [m.name, m]));
|
|
746
|
+
const embeds = [], blocks = [];
|
|
747
|
+
[...new Set(shown.filter((x) => !x.startsWith("exhibits/")))].forEach((f, k) => {
|
|
748
|
+
const fp = path.join(base, f);
|
|
749
|
+
const m = inPkg.get(f);
|
|
750
|
+
if (!m || !isFile(fp))
|
|
751
|
+
return;
|
|
752
|
+
if (m.filter) {
|
|
753
|
+
blocks.push(`<li><code>${esc(f)}</code> (in the package, without the rows of ${esc([...m.filter[1]].sort().join(", "))}: not cleared for redistribution)</li>`);
|
|
754
|
+
return;
|
|
755
|
+
}
|
|
756
|
+
if (fs.statSync(fp).size > EMBED_MAX) {
|
|
757
|
+
blocks.push(`<li><a href='${esc(rel(f))}'>${esc(f)}</a> (large file; it is in the package)</li>`);
|
|
758
|
+
return;
|
|
759
|
+
}
|
|
760
|
+
const text = fs.readFileSync(fp).toString("utf8");
|
|
761
|
+
const name = path.posix.basename(f);
|
|
762
|
+
embeds.push({ id: `att-${k}`, name, text });
|
|
763
|
+
blocks.push(`<li><details><summary><code>${esc(f)}</code></summary><button type='button' class='dl' data-attach='att-${k}'>` +
|
|
764
|
+
`Download ${esc(name)}</button><pre class='code'>${esc(text)}</pre></details></li>`);
|
|
765
|
+
});
|
|
766
|
+
const held = excluded.filter((e) => e.policy && !e.name.startsWith("lib/") && !e.name.startsWith("reproduce/"));
|
|
767
|
+
const heldHtml = held.length ? "<p>Not in the package because of the source terms: " + held.slice(0, 8).map((e) => `<code>${esc(e.name)}</code> (${esc(e.reason)})`).join("; ") + (held.length > 8 ? `; and ${held.length - 8} more` : "") + ".</p>" : "";
|
|
768
|
+
const pkgHtml = pkg ? `<p><a class='pkg' href='${esc(rel(pkg))}'>Download the replication package (ZIP)</a>: the data each ` +
|
|
769
|
+
"source's terms allow to share, download instructions for the rest, the exhibit data and their recipes, the facts file, " +
|
|
770
|
+
"and step-by-step instructions (REPLICATION.md).</p>" : "";
|
|
771
|
+
const title = mode === "paper" ? "Appendix: Replication Materials" : "Replication Materials";
|
|
772
|
+
const get = defonly.map((sid) => {
|
|
773
|
+
const sc = reproduceScripts(base, sid);
|
|
774
|
+
return Object.keys(sc).length ? `${esc(ctx.title(sid))}: run <code>${esc(sc.bash)}</code> (macOS, Linux) or ` +
|
|
775
|
+
`<code>${esc(sc.powershell)}</code> (Windows), attached below. One command downloads the data under your own account ` +
|
|
776
|
+
`(the request is built in; the script asks for your key with hidden input). Then follow <code>reproduce/${esc(sid)}/REPRODUCE.md</code>.`
|
|
777
|
+
: `${esc(ctx.title(sid))}: follow <code>reproduce/${esc(sid)}/REPRODUCE.md</code>.`;
|
|
778
|
+
});
|
|
779
|
+
const getHtml = get.length ? "<ul>" + get.map((g) => `<li>${g}</li>`).join("") + "</ul>" : "";
|
|
780
|
+
const steps = "<ol><li>Get the data: use the included files in <code>data/</code> where the terms allow" +
|
|
781
|
+
(get.length ? ", and for sources whose terms forbid sharing:" + getHtml : ".") +
|
|
782
|
+
"</li><li>Replay the recipes in <code>exhibits/lineage.json</code> with the data-analyst-mcp server's <code>rerun</code> " +
|
|
783
|
+
"tool. It recomputes the exhibit data and <code>facts.json</code>.</li><li>Compare the recomputed facts with the values in " +
|
|
784
|
+
"the audit trail.</li></ol>";
|
|
785
|
+
return [`<section class='prose replication' id='replication'><h2>${title}</h2>${pkgHtml}${heldHtml}${steps}` +
|
|
786
|
+
`<ul class='files code-list'>${blocks.join("")}</ul></section>`, embeds];
|
|
787
|
+
}
|
|
788
|
+
const PROV_ROWS = [["retrieved", "Retrieved"], ["downloaded", "Downloaded"], ["route", "Route"],
|
|
789
|
+
["acquired", "How acquired"], ["api_version", "API version"], ["api_base", "API"], ["collection", "Collection"],
|
|
790
|
+
["extract_number", "Extract number"], ["latest_period", "Latest period"], ["n_requests", "Requests"], ["n_queries", "Queries"],
|
|
791
|
+
["rows", "Rows"], ["fingerprint", "Request fingerprint"], ["csv_sha256", "Data checksum (SHA-256)"]];
|
|
792
|
+
function provenanceHtml(ctx) {
|
|
793
|
+
const parts = [];
|
|
794
|
+
for (const sid of ctx.ids) {
|
|
795
|
+
const [provRel, P] = ctx.prov[sid];
|
|
796
|
+
const p = ctx.policies[sid];
|
|
797
|
+
const rows = [["Source", `${p.title} (${p.agency})`], ["Redistribution rule", p.redistribution.mode]];
|
|
798
|
+
for (const [k, label] of PROV_ROWS) {
|
|
799
|
+
const v = P[k];
|
|
800
|
+
if (v === null || v === undefined || v === "" || (Array.isArray(v) && !v.length) || (typeof v === "object" && !Array.isArray(v) && !Object.keys(v).length))
|
|
801
|
+
continue;
|
|
802
|
+
rows.push([label, typeof v === "number" && Number.isInteger(v) ? commas(v) : typeof v === "object" ? pyJsonDumps(v, false, true) : str(v)]);
|
|
803
|
+
}
|
|
804
|
+
if (Array.isArray(P.requests) && !("n_requests" in P))
|
|
805
|
+
rows.push(["Requests", String(P.requests.length)]);
|
|
806
|
+
if (P.spec?.as_of)
|
|
807
|
+
rows.push(["Data cut-off (as_of)", str(P.spec.as_of)]);
|
|
808
|
+
rows.push(["Provenance file", provRel || "none found"]);
|
|
809
|
+
let tbl = "<div class='table-wrap'><table class='audit'><tbody>" + rows.map(([k, v]) => `<tr><th scope='row'>${esc(k)}</th><td>${esc(v)}</td></tr>`).join("") + "</tbody></table></div>";
|
|
810
|
+
let trs = "";
|
|
811
|
+
if (Array.isArray(P.tables)) {
|
|
812
|
+
trs = P.tables.map((t) => `<tr><td><code>${esc(t.file ?? "")}</code></td><td>${commas(Number(t.rows ?? 0))}</td>` +
|
|
813
|
+
`<td><code>${esc(String(t.sha256 ?? "").slice(0, 16))}</code></td></tr>`).join("");
|
|
814
|
+
}
|
|
815
|
+
else if (P.datasets && typeof P.datasets === "object") {
|
|
816
|
+
trs = Object.entries(P.datasets).map(([n, d]) => `<tr><td><code>${esc(n)}.csv</code> (${esc(d.endpoint ?? "")})</td>` +
|
|
817
|
+
`<td>${commas(Number(d.rows ?? 0))}</td><td>${esc(d.first_date || "")} to ${esc(d.last_date || "")}</td></tr>`).join("");
|
|
818
|
+
}
|
|
819
|
+
else if (Array.isArray(P.files)) {
|
|
820
|
+
trs = P.files.slice(0, 40).map((f) => `<tr><td><code>${esc(f.dataset ?? "")} ${esc(f.year ?? "")}-${esc(f.month ?? "")}</code>` +
|
|
821
|
+
`</td><td>v${esc(f.version ?? "")}</td><td><code>${esc(String(f.sha256 ?? "").slice(0, 16))}</code></td></tr>`).join("");
|
|
822
|
+
}
|
|
823
|
+
if (trs)
|
|
824
|
+
tbl += `<div class='table-wrap'><table class='audit'><tbody>${trs}</tbody></table></div>`;
|
|
825
|
+
const ctxp = path.join(ctx.base, "data", sid, "context_sources.json");
|
|
826
|
+
if (fs.existsSync(ctxp)) {
|
|
827
|
+
const C = JSON.parse(readText(ctxp));
|
|
828
|
+
tbl += "<div class='table-wrap'><table class='audit'><thead><tr><th>Context series</th><th>Source</th><th>Units</th>" +
|
|
829
|
+
"<th>Range</th><th>Retrieved</th></tr></thead><tbody>" + Object.entries(C).map(([k, v]) => `<tr><td>${esc(k)} (<code>${esc(v.series_id ?? "")}</code>)</td><td>${esc(v.source ?? "")}</td><td>${esc(v.units ?? "")}</td>` +
|
|
830
|
+
`<td>${esc(v.first ?? "")} to ${esc(v.last ?? "")}</td><td>${esc(v.retrieved ?? "")}</td></tr>`).join("") + "</tbody></table></div>";
|
|
831
|
+
}
|
|
832
|
+
parts.push(`<h4>${esc(p.title)}</h4>${tbl}`);
|
|
833
|
+
}
|
|
834
|
+
if (!parts.length)
|
|
835
|
+
return "";
|
|
836
|
+
return "<h3>Data provenance</h3>" + parts.join("") + "<p class='audit-note'>The exact requests (without keys) are in each " +
|
|
837
|
+
"source's provenance file; the server's <code>fetch</code> tool downloads the data again.</p>";
|
|
838
|
+
}
|
|
839
|
+
function auditHtml(ctx, outDir, facts, used, mode) {
|
|
840
|
+
const R = ctx.R, base = ctx.base;
|
|
841
|
+
const rel = (p) => path.relative(outDir, path.join(base, p)).replaceAll("\\", "/");
|
|
842
|
+
const parts = [provenanceHtml(ctx)];
|
|
843
|
+
const files = [...(R.data_files || [])];
|
|
844
|
+
for (const extra of ["analysis.py", "facts.json", "report.json", "spec.json", "plan.json", "exhibits/lineage.json"]) {
|
|
845
|
+
if (fs.existsSync(path.join(base, extra)) && !files.some((d) => d.path === extra))
|
|
846
|
+
files.push({ label: extra, path: extra });
|
|
847
|
+
}
|
|
848
|
+
const trs = files.map((d) => {
|
|
849
|
+
const fp = path.join(base, d.path);
|
|
850
|
+
return `<tr><td><a href='${esc(rel(d.path))}'>${esc(d.path)}</a></td><td>${commas(fs.statSync(fp).size)}</td><td><code>${fileSha(fp)}</code></td></tr>`;
|
|
851
|
+
});
|
|
852
|
+
for (const [sid, P] of Object.entries(ctx.policies)) { // definition_only data: checksum only, never linked
|
|
853
|
+
if (P.redistribution.mode !== "definition_only")
|
|
854
|
+
continue;
|
|
855
|
+
for (const f of tree(base, `data/${sid}`)) {
|
|
856
|
+
const fp = path.join(base, f);
|
|
857
|
+
if (PS.isMicrodata(P, f.slice(`data/${sid}/`.length)) && fs.statSync(fp).size <= 4 * PS.LIMITS.MICRO_MAX) {
|
|
858
|
+
trs.push(`<tr><td><code>${esc(f)}</code> (local only, not distributed)</td><td>${commas(fs.statSync(fp).size)}</td><td><code>${fileSha(fp)}</code></td></tr>`);
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
if (trs.length) {
|
|
863
|
+
parts.push("<h3>Files and checksums (SHA-256)</h3><div class='table-wrap'><table class='audit'><thead><tr><th>File</th>" +
|
|
864
|
+
"<th>Bytes</th><th>SHA-256</th></tr></thead><tbody>" + trs.join("") + "</tbody></table></div>");
|
|
865
|
+
}
|
|
866
|
+
if (used.length) {
|
|
867
|
+
const anySe = used.some(([n]) => typeof facts[n].se === "number");
|
|
868
|
+
const seen = new Set();
|
|
869
|
+
const rows = [];
|
|
870
|
+
for (const [name, citedFmt] of used) {
|
|
871
|
+
const k = `${name}\u0000${citedFmt}`;
|
|
872
|
+
if (seen.has(k))
|
|
873
|
+
continue;
|
|
874
|
+
seen.add(k);
|
|
875
|
+
const f = facts[name];
|
|
876
|
+
const pk = f.pack;
|
|
877
|
+
const pkTxt = pk ? (Array.isArray(pk) ? pk.join(", ") : str(pk)) : "";
|
|
878
|
+
const src = f.source ? `<a href='${esc(rel(f.source))}'>${esc(f.source)}</a><br><span class='muted'>row: ` +
|
|
879
|
+
`${esc(Object.entries(f.where || {}).map(([a, b]) => `${a} = ${str(b)}`).join(", "))}; column: ${esc(f.column)}</span>`
|
|
880
|
+
: `<span class='muted'>derived: ${esc(f.derivation ?? "")}</span>`;
|
|
881
|
+
const se = typeof f.se === "number" ? fmtG(f.se, 6) : "";
|
|
882
|
+
rows.push(`<tr><td>${esc(f.label || name)}<br><code>${esc(name)}</code></td><td>${esc(formatValue(f.value, citedFmt, mode))}</td>` +
|
|
883
|
+
`<td><code>${esc(f.value)}</code></td>${anySe ? `<td>${esc(se)}</td>` : ""}<td>${src}</td><td>${esc(pkTxt)}</td></tr>`);
|
|
884
|
+
}
|
|
885
|
+
parts.push("<h3>Numbers cited in the text</h3><div class='table-wrap'><table class='audit'><thead><tr><th>Fact</th>" +
|
|
886
|
+
`<th>As cited</th><th>Unrounded</th>${anySe ? "<th>Standard error</th>" : ""}<th>Source cell</th><th>Source pack</th></tr></thead><tbody>` +
|
|
887
|
+
rows.join("") + "</tbody></table></div>");
|
|
888
|
+
}
|
|
889
|
+
// balanced parentheses stay in a URL ("...RA%20(GW).aspx"), as in audit_report.findUrls
|
|
890
|
+
const urls = [...new Set((JSON.stringify(R).match(/https?:\/\/(?:[^\s"'<>()[\]\\]|\([^\s"'<>()]*\))+/g) || []).map((u) => u.replace(/[.,;:]+$/, "")))].sort();
|
|
891
|
+
if (urls.length)
|
|
892
|
+
parts.push("<h3>External references</h3><ul class='files'>" + urls.map((u) => `<li><a href='${esc(u)}'>${esc(u)}</a></li>`).join("") + "</ul>");
|
|
893
|
+
const now = new Date();
|
|
894
|
+
const pad = (n) => String(n).padStart(2, "0");
|
|
895
|
+
parts.push(`<p class='audit-note'>Generated ${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())} ${pad(now.getHours())}:` +
|
|
896
|
+
`${pad(now.getMinutes())} from <code>report.json</code>. The data for every exhibit are embedded in this file and are also saved ` +
|
|
897
|
+
"in the CSV files linked above.</p>");
|
|
898
|
+
return `<section class='prose audit-trail' id='audit'><h2>${mode === "paper" ? "Appendix: Audit Trail" : "Audit Trail"}</h2>` +
|
|
899
|
+
`<details><summary>Data provenance, file checksums, and the source of every number</summary>${parts.join("")}</details></section>`;
|
|
900
|
+
}
|
|
901
|
+
const findingHtml = (s) => (s && typeof s === "object" && !Array.isArray(s)
|
|
902
|
+
? (s.lead ? `<strong>${inlineMd(s.lead)}</strong> ` : "") + inlineMd(s.text ?? "") : inlineMd(s));
|
|
903
|
+
function tilesHtml(R) {
|
|
904
|
+
const hl = R.highlights || [];
|
|
905
|
+
return hl.length ? "<div class='tiles'>" + hl.map((h) => `<div class='tile'><p class='t-label'>${esc(h.label)}</p>` +
|
|
906
|
+
`<p class='t-value'>${esc(h.value)}</p>${h.note ? `<p class='t-note'>${esc(h.note)}</p>` : ""}</div>`).join("") + "</div>" : "";
|
|
907
|
+
}
|
|
908
|
+
const exRange = (nums) => (!nums.length ? "" : nums.length === 1 ? `Exhibit ${nums[0]}` : `Exhibits ${nums[0]}–${nums[nums.length - 1]}`);
|
|
909
|
+
function validateReport(R) {
|
|
910
|
+
const n = (R.exhibits || []).length;
|
|
911
|
+
const errs = [];
|
|
912
|
+
const placed = (R.sections || []).flatMap((s) => (s.exhibits || []).map(Number));
|
|
913
|
+
if (JSON.stringify(placed) !== JSON.stringify(Array.from({ length: n }, (_, i) => i + 1))) {
|
|
914
|
+
errs.push(`the sections must show every exhibit exactly once, in order 1..${n} (exhibits are numbered in reading order); got ${str(placed)}`);
|
|
915
|
+
}
|
|
916
|
+
(R.sections || []).forEach((s, k) => {
|
|
917
|
+
if (!strip(String(s.body ?? "")))
|
|
918
|
+
errs.push(`section ${k + 1} ('${s.heading ?? "None"}') has no body: it opens with a summary of its findings`);
|
|
919
|
+
});
|
|
920
|
+
const cov = (R.cover || {}).exhibit;
|
|
921
|
+
if (cov && !(Number(cov) >= 1 && Number(cov) <= n))
|
|
922
|
+
errs.push(`cover.exhibit ${cov} is not an exhibit number`);
|
|
923
|
+
if (errs.length)
|
|
924
|
+
fail("report structure:\n - " + errs.join("\n - "));
|
|
925
|
+
}
|
|
926
|
+
function paperBody(R, exBlocks, p) {
|
|
927
|
+
const summary = (R.summary || []).map((s) => `<p>${findingHtml(s)}</p>`).join("");
|
|
928
|
+
const placed = new Set(), secHtml = [], toc = [];
|
|
929
|
+
(R.sections || []).forEach((s, j) => {
|
|
930
|
+
const k = j + 1;
|
|
931
|
+
const nums = (s.exhibits || []).map(Number).filter((n) => exBlocks.has(n) && !placed.has(n));
|
|
932
|
+
nums.forEach((n) => placed.add(n));
|
|
933
|
+
toc.push(`<li><a href='#section-${k}'>${esc(s.heading)}</a></li>`);
|
|
934
|
+
secHtml.push(`<section class='prose' id='section-${k}'><h2>${esc(s.heading)}</h2>${blocksMd(s.body ?? "")}${nums.map((n) => exBlocks.get(n)).join("")}</section>`);
|
|
935
|
+
});
|
|
936
|
+
const rest = [...exBlocks.keys()].filter((n) => !placed.has(n));
|
|
937
|
+
const restHtml = rest.length ? `<section class='exhibits'>${secHtml.length ? "<h2>Exhibits</h2>" : ""}${rest.map((n) => exBlocks.get(n)).join("")}</section>` : "";
|
|
938
|
+
const tocItems = [...toc, "<li><a href='#methods'>Appendix: Data and Methods</a></li>",
|
|
939
|
+
"<li><a href='#replication'>Appendix: Replication Materials</a></li>", "<li><a href='#audit'>Appendix: Audit Trail</a></li>"];
|
|
940
|
+
const nav = toc.length ? `<nav class='toc'><h2>Contents</h2><ol>${tocItems.join("")}</ol></nav>` : "";
|
|
941
|
+
const about = R.about ? `<section class='prose about' id='about'><h2>About This Document</h2>${blocksMd(R.about)}</section>` : "";
|
|
942
|
+
return `
|
|
943
|
+
${disclaimerHtml()}
|
|
944
|
+
<header class="masthead">
|
|
945
|
+
<p class="kicker">${p.meta}${p.badge}</p>
|
|
946
|
+
<h1>${esc(R.title)}</h1>
|
|
947
|
+
${R.subtitle ? `<p class="dek">${inlineMd(R.subtitle)}</p>` : ""}
|
|
948
|
+
</header>
|
|
949
|
+
<section class="summary">
|
|
950
|
+
<h2>Summary</h2>
|
|
951
|
+
${tilesHtml(R)}
|
|
952
|
+
${summary}
|
|
953
|
+
</section>
|
|
954
|
+
${nav}
|
|
955
|
+
${secHtml.join("")}
|
|
956
|
+
${restHtml}
|
|
957
|
+
${p.methods}
|
|
958
|
+
${p.refs}
|
|
959
|
+
${p.repl}
|
|
960
|
+
${p.audit}
|
|
961
|
+
${footnotesHtml(R.footnotes || [])}
|
|
962
|
+
${about}
|
|
963
|
+
`;
|
|
964
|
+
}
|
|
965
|
+
function reportBody(R, exBlocks, p) {
|
|
966
|
+
const sections = R.sections || [], exhibits = R.exhibits || [];
|
|
967
|
+
const secNums = sections.map((s) => (s.exhibits || []).map(Number));
|
|
968
|
+
const cov = R.cover || {};
|
|
969
|
+
let hero = "";
|
|
970
|
+
if (cov.exhibit) {
|
|
971
|
+
const ex = exhibits[Number(cov.exhibit) - 1];
|
|
972
|
+
const t = cov.title || ex.title;
|
|
973
|
+
const subt = ex.subtitle ? `<p class='ex-sub'>${esc(ex.subtitle)}</p>` : "";
|
|
974
|
+
hero = `<figure class='cover-chart'><p class='cover-chart-title'>${esc(t)}</p>${subt}<div class='legend' data-legend='cover'></div>` +
|
|
975
|
+
`<div class='chart' data-chart='cover' role='img' aria-label='${esc(t)}'></div></figure>`;
|
|
976
|
+
}
|
|
977
|
+
const dek = R.subtitle ? `<p class="dek">${inlineMd(R.subtitle)}</p>` : "";
|
|
978
|
+
const cover = `<header class="masthead cover" id="cover"><p class="kicker">${p.meta}${p.badge}</p><h1>${esc(R.title)}</h1>${dek}${hero}</header>`;
|
|
979
|
+
const g = R.at_a_glance || {};
|
|
980
|
+
const findings = (g.findings && g.findings.length ? g.findings : R.summary) || [];
|
|
981
|
+
const glance = `<section class='glance' id='at-a-glance'><h2>At a Glance</h2>${blocksMd(g.text ?? "")}` +
|
|
982
|
+
(findings.length ? "<ul class='glance-findings'>" + findings.map((s) => `<li>${findingHtml(s)}</li>`).join("") + "</ul>" : "") +
|
|
983
|
+
`${tilesHtml(R)}</section>`;
|
|
984
|
+
const intro = R.introduction || {};
|
|
985
|
+
const introTitle = intro.title || "Introduction";
|
|
986
|
+
const subs = intro.subsections || [];
|
|
987
|
+
const items = [];
|
|
988
|
+
if (Object.keys(intro).length) {
|
|
989
|
+
const subList = subs.length ? "<ul>" + subs.map((x, j) => `<li><a href='#intro-${j + 1}'>${esc(x.heading)}</a></li>`).join("") + "</ul>" : "";
|
|
990
|
+
items.push(`<li><a href='#introduction'>${esc(introTitle)}</a>${subList}</li>`);
|
|
991
|
+
}
|
|
992
|
+
sections.forEach((s, j) => {
|
|
993
|
+
const nums = secNums[j];
|
|
994
|
+
items.push(`<li><a href='#section-${j + 1}'>${esc(s.heading)}</a>${nums.length ? `<span class='ex-range'>${exRange(nums)}</span>` : ""}</li>`);
|
|
995
|
+
});
|
|
996
|
+
if (R.footnotes && R.footnotes.length)
|
|
997
|
+
items.push("<li><a href='#footnotes'>Notes</a></li>");
|
|
998
|
+
items.push("<li><a href='#methods'>Appendix A: Data and Methods</a></li>");
|
|
999
|
+
const appx = R.appendixes || [];
|
|
1000
|
+
const letters = appx.map((_, j) => String.fromCharCode(66 + j));
|
|
1001
|
+
appx.forEach((a, j) => items.push(`<li><a href='#appendix-${letters[j]}'>Appendix ${letters[j]}: ${esc(a.title)}</a></li>`));
|
|
1002
|
+
if (p.refs)
|
|
1003
|
+
items.push("<li><a href='#references'>References</a></li>");
|
|
1004
|
+
items.push("<li><a href='#about'>About This Document</a></li>");
|
|
1005
|
+
items.push("<li class='supp'>Supplementary materials: <a href='#replication'>Replication Materials</a> · <a href='#audit'>Audit Trail</a></li>");
|
|
1006
|
+
const contents = `<nav class='contents toc' id='contents'><h2>Contents</h2><ol class='contents-list'>${items.join("")}</ol></nav>`;
|
|
1007
|
+
const groups = sections.map((s, j) => secNums[j].length ? `<h3>${esc(s.heading)}</h3><ol class='ex-list'>` +
|
|
1008
|
+
secNums[j].map((n) => `<li><a href='#exhibit-${n}'><span class='ex-n'>${n}.</span> ${esc(exhibits[n - 1]?.title)}</a></li>`).join("") + "</ol>" : "").join("");
|
|
1009
|
+
const exList = groups ? `<nav class='exhibit-list' id='list-of-exhibits'><h2>List of Exhibits</h2>${groups}</nav>` : "";
|
|
1010
|
+
let notes = R.notes_page;
|
|
1011
|
+
if (notes && notes.length && R.data_needs && R.data_needs.length) {
|
|
1012
|
+
notes = [...(typeof notes === "string" ? [notes] : notes), "Appendix A lists the data that would make these estimates more accurate."];
|
|
1013
|
+
}
|
|
1014
|
+
const notesHtml = notes && notes.length ? `<section class='prose notes-page' id='notes'><h2>Notes About This Report</h2>${blocksMd(notes)}</section>` : "";
|
|
1015
|
+
let introHtml = "";
|
|
1016
|
+
if (Object.keys(intro).length) {
|
|
1017
|
+
const sc = intro.footnotes && intro.footnotes.length ? "intro" : null;
|
|
1018
|
+
const subHtml = subs.map((x, j) => `<h3 id='intro-${j + 1}'>${esc(x.heading)}</h3>${blocksMd(x.body ?? "", sc)}`).join("");
|
|
1019
|
+
introHtml = `<section class='prose intro' id='introduction'><h2>${esc(introTitle)}</h2>${blocksMd(intro.text ?? "", sc)}${subHtml}` +
|
|
1020
|
+
`${footnotesHtml(intro.footnotes || [], sc, null)}</section>`;
|
|
1021
|
+
}
|
|
1022
|
+
const secHtml = sections.map((s, j) => {
|
|
1023
|
+
const sc = s.footnotes && s.footnotes.length ? `s${j + 1}` : null;
|
|
1024
|
+
return `<section class='chapter' id='section-${j + 1}'><div class='section-opener prose'><h2>${esc(s.heading)}</h2>` +
|
|
1025
|
+
`${blocksMd(s.body ?? "", sc)}</div>${secNums[j].map((n) => exBlocks.get(n)).join("")}${footnotesHtml(s.footnotes || [], sc, null)}</section>`;
|
|
1026
|
+
});
|
|
1027
|
+
const globalFn = footnotesHtml(R.footnotes || []).replace("<section class='prose footnotes'>", "<section class='prose footnotes' id='footnotes'>");
|
|
1028
|
+
const extra = appx.map((a, j) => `<section class='prose appendix' id='appendix-${letters[j]}'><h2>Appendix ${letters[j]}: ` +
|
|
1029
|
+
`${esc(a.title)}</h2>${blocksMd(a.body ?? "")}</section>`).join("");
|
|
1030
|
+
const about = `<section class='prose about' id='about'><h2>About This Document</h2>${blocksMd(R.about ?? "")}</section>`;
|
|
1031
|
+
const supp = `<div class='supplementary'><p class='supp-label'>Supplementary materials (not part of the publication)</p>${p.repl}${p.audit}</div>`;
|
|
1032
|
+
return [disclaimerHtml(), cover, glance, contents, exList, notesHtml, introHtml, ...secHtml, globalFn, p.methods, extra, p.refs, about, supp].join("\n");
|
|
1033
|
+
}
|
|
1034
|
+
// ----------------------------------------------------------------------------- build
|
|
1035
|
+
function inside(p, folder) {
|
|
1036
|
+
const rel = path.relative(path.resolve(folder), path.resolve(p));
|
|
1037
|
+
return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
|
|
1038
|
+
}
|
|
1039
|
+
export function build(reportPath, outPath) {
|
|
1040
|
+
const base = path.dirname(path.resolve(reportPath));
|
|
1041
|
+
if (inside(base, ROOT))
|
|
1042
|
+
fail(`${base} is inside the server's package folder; build reports in the session workspace`);
|
|
1043
|
+
MA.NOTES.length = 0;
|
|
1044
|
+
let R = MA.merged(base, JSON.parse(readText(reportPath)));
|
|
1045
|
+
const mode = R.mode || "report";
|
|
1046
|
+
if (mode !== "report" && mode !== "paper")
|
|
1047
|
+
fail(`mode must be 'report' or 'paper', not '${mode}'`);
|
|
1048
|
+
outPath = outPath || path.join(base, "report.html");
|
|
1049
|
+
const outDir = path.dirname(path.resolve(outPath));
|
|
1050
|
+
const ctx = new Ctx(R, base);
|
|
1051
|
+
REF_LABELS = new Map((R.references || []).filter((r) => r.id).map((r) => [r.id, r.label || r.id]));
|
|
1052
|
+
const unknown = [...citedIds(JSON.stringify(R))].filter((i) => !REF_LABELS.has(i));
|
|
1053
|
+
if (unknown.length)
|
|
1054
|
+
fail(`[@citations] without a matching entry in references: ${str(unknown.sort())}`);
|
|
1055
|
+
const factsPath = path.join(base, R.facts || "facts.json");
|
|
1056
|
+
const facts = fs.existsSync(factsPath) ? JSON.parse(readText(factsPath)) : {};
|
|
1057
|
+
let used;
|
|
1058
|
+
[R, used] = fillFacts(R, facts, mode);
|
|
1059
|
+
ctx.R = R;
|
|
1060
|
+
const left = JSON.stringify(Object.fromEntries(Object.entries(R).filter(([k]) => k !== "request"))).match(ANY_PLACEHOLDER) || [];
|
|
1061
|
+
if (left.length)
|
|
1062
|
+
fail(`unresolved or malformed fact placeholders (use {{name}} or {{name|fmt}}): ${str(left)}`);
|
|
1063
|
+
if (!R.date) {
|
|
1064
|
+
R.date = new Date().toISOString().slice(0, 10);
|
|
1065
|
+
MA.note(`report.json has no "date"; using today (${R.date})`);
|
|
1066
|
+
}
|
|
1067
|
+
const bad = [...(R.data_files || []).map((d) => d.path).filter((p) => !fs.existsSync(path.join(base, p))),
|
|
1068
|
+
...(R.exhibits || []).filter((e) => e.data && !fs.existsSync(path.join(base, e.data))).map((e) => e.data)];
|
|
1069
|
+
if (bad.length)
|
|
1070
|
+
fail(`linked files do not exist (every link must resolve): ${str(bad)}`);
|
|
1071
|
+
// files the source terms keep local are never linked from the page (it travels with the package)
|
|
1072
|
+
const keepLocal = (R.data_files || []).map((d) => d.path).filter((p) => {
|
|
1073
|
+
const d = PS.decide(p, base, ctx.policies, R, ctx.withheld);
|
|
1074
|
+
return !d.ok && d.policy;
|
|
1075
|
+
});
|
|
1076
|
+
if (keepLocal.length) {
|
|
1077
|
+
MA.note(`not linked: ${str(keepLocal)} (the source terms keep these files local)`);
|
|
1078
|
+
R.data_files = R.data_files.filter((d) => !keepLocal.includes(d.path));
|
|
1079
|
+
}
|
|
1080
|
+
if ((R.replication || {}).include_microdata) {
|
|
1081
|
+
MA.note("replication.include_microdata is ignored: microdata of a definition_only source never goes into a package");
|
|
1082
|
+
}
|
|
1083
|
+
const sections = R.sections || [];
|
|
1084
|
+
const exScope = new Map();
|
|
1085
|
+
if (mode === "report") {
|
|
1086
|
+
validateReport(R);
|
|
1087
|
+
sections.forEach((s, j) => {
|
|
1088
|
+
for (const n of s.exhibits || [])
|
|
1089
|
+
exScope.set(Number(n), s.footnotes && s.footnotes.length ? `s${j + 1}` : null);
|
|
1090
|
+
});
|
|
1091
|
+
}
|
|
1092
|
+
const coverEx = mode === "report" ? Number((R.cover || {}).exhibit || 0) : 0;
|
|
1093
|
+
const payload = [], exBlocks = new Map();
|
|
1094
|
+
(R.exhibits || []).forEach((ex, j) => {
|
|
1095
|
+
const i = j + 1;
|
|
1096
|
+
const ch = ex.chart || {};
|
|
1097
|
+
const rec = ch.recessions;
|
|
1098
|
+
if (rec) {
|
|
1099
|
+
ch._recessions = loadRecessions(base, typeof rec === "string" ? rec : "exhibits/recessions.csv");
|
|
1100
|
+
const n = ch.recession_note || REC_NOTE;
|
|
1101
|
+
if (!String(ex.notes ?? "").includes(n))
|
|
1102
|
+
ex.notes = (String(ex.notes ?? "").trimEnd() + " " + n).trim();
|
|
1103
|
+
}
|
|
1104
|
+
let [rows, cols, csvText] = loadRows(ex, base);
|
|
1105
|
+
const show = ch.columns; // display/download only these columns (table exhibits)
|
|
1106
|
+
if (show && show.length) {
|
|
1107
|
+
const miss = show.filter((c) => !cols.includes(c));
|
|
1108
|
+
if (miss.length)
|
|
1109
|
+
fail(`exhibit ${i}: chart.columns ${str(miss)} are not in ${ex.data}`);
|
|
1110
|
+
csvText = writeDict(rows, show, (v) => (v === null || v === undefined ? "" : str(v)));
|
|
1111
|
+
rows = rows.map((r) => Object.fromEntries(show.map((k) => [k, r[k]])));
|
|
1112
|
+
cols = [...show];
|
|
1113
|
+
}
|
|
1114
|
+
const spec = chartSpec(ex, rows);
|
|
1115
|
+
const fname = path.posix.basename(String(ex.data ?? `exhibit_${i}.csv`).replaceAll("\\", "/"));
|
|
1116
|
+
const fmt = ch.format || "number";
|
|
1117
|
+
payload.push({ id: i, chart: spec, csv: csvText, csv_name: fname, columns: cols, format: fmt });
|
|
1118
|
+
if (i === coverEx && spec)
|
|
1119
|
+
payload.push({ id: "cover", chart: spec, csv: "", csv_name: "", columns: [], format: fmt });
|
|
1120
|
+
exBlocks.set(i, exhibitHtml(i, ex, spec, fname, mode, exScope.get(i)));
|
|
1121
|
+
});
|
|
1122
|
+
const files = (R.data_files || []).map((d) => {
|
|
1123
|
+
const rel = path.relative(outDir, path.join(base, d.path)).replaceAll("\\", "/");
|
|
1124
|
+
return `<li><a href='${esc(rel)}'>${esc(d.label || d.path)}</a> <code>${esc(d.path)}</code></li>`;
|
|
1125
|
+
});
|
|
1126
|
+
const filesHtml = files.length ? `<h3>Data Underlying the Exhibits</h3><ul class='files'>${files.join("")}</ul>` : "";
|
|
1127
|
+
const requestHtml = R.request ? `<h3>Request</h3><blockquote>${inlineMd(R.request)}</blockquote>` : "";
|
|
1128
|
+
const citeHtml = R.citation ? `<h3>Data Citation</h3><p class='cite'>${inlineMd(R.citation)}</p>` : "";
|
|
1129
|
+
const methodsTitle = mode === "report" ? "Appendix A: Data and Methods" : "Appendix: Data and Methods";
|
|
1130
|
+
const methodsHtml = `<section class='prose methods appendix' id='methods'><h2>${methodsTitle}</h2>${requestHtml}` +
|
|
1131
|
+
`${blocksMd(R.methods ?? "")}${vintageHtml(ctx)}${decisionsHtml(R)}${assumptionsHtml(R)}${dataNeedsHtml(R)}${filesHtml}` +
|
|
1132
|
+
`${citeHtml}${noticesHtml(ctx)}</section>`;
|
|
1133
|
+
for (const [sid, P] of Object.entries(ctx.policies)) {
|
|
1134
|
+
if (P.redistribution.mode === "definition_only")
|
|
1135
|
+
writeReproduce(ctx, sid);
|
|
1136
|
+
else
|
|
1137
|
+
fs.rmSync(path.join(base, "reproduce", sid), { recursive: true, force: true });
|
|
1138
|
+
}
|
|
1139
|
+
const [pkg, members, excluded] = makePackage(ctx);
|
|
1140
|
+
R.data_files = R.data_files || [];
|
|
1141
|
+
if (!R.data_files.some((d) => d.path === pkg))
|
|
1142
|
+
R.data_files.push({ label: "Replication package (ZIP)", path: pkg });
|
|
1143
|
+
const [replHtml, attachments] = replicationHtml(ctx, outDir, pkg, members, excluded, mode);
|
|
1144
|
+
const audit = auditHtml(ctx, outDir, facts, used, mode);
|
|
1145
|
+
const meta = [R.organization, longDate(R.date)].filter((x) => x).map(esc).join(" · ");
|
|
1146
|
+
const badge = R.status ? ` <span class="badge">${esc(R.status)}</span>` : "";
|
|
1147
|
+
const parts = { meta, badge, methods: methodsHtml, refs: referencesHtml(R), repl: replHtml, audit };
|
|
1148
|
+
const body = mode === "report" ? reportBody(R, exBlocks, parts) : paperBody(R, exBlocks, parts);
|
|
1149
|
+
const tpl = fs.readFileSync(TEMPLATE, "utf8");
|
|
1150
|
+
const dataJson = JSON.stringify(payload).replaceAll("</", "<\\/");
|
|
1151
|
+
const attachJson = JSON.stringify(attachments).replaceAll("</", "<\\/");
|
|
1152
|
+
const page = tpl.split("__TITLE__").join(esc(R.title ?? "Report")).split("__MODE__").join(mode)
|
|
1153
|
+
.split("<!--__BODY__-->").join(body).split("<!--__FOOTER__-->").join(footerHtml(ctx))
|
|
1154
|
+
.split("/*__DATA__*/null").join(dataJson).split("/*__ATTACH__*/null").join(attachJson);
|
|
1155
|
+
const leaks = scanText(page, knownKeys(), false, "report.html");
|
|
1156
|
+
if (leaks.length) {
|
|
1157
|
+
fs.rmSync(outPath, { force: true });
|
|
1158
|
+
fail("an API key or credential appeared in the report content; nothing was written (" + leaks.slice(0, 3).join("; ") + ")");
|
|
1159
|
+
}
|
|
1160
|
+
fs.writeFileSync(outPath, page, "utf8");
|
|
1161
|
+
const ext = page.match(/(?:\bsrc\s*=\s*["']|<link[^>]+href\s*=\s*["']|@import\s+|url\(\s*["']?)(?:https?:)?\/\//gi) || [];
|
|
1162
|
+
return { report: path.resolve(outPath), mode, sources: ctx.ids, exhibits: exBlocks.size, facts_cited: new Set(used.map(([n]) => n)).size,
|
|
1163
|
+
bytes: Buffer.byteLength(page), external_refs: ext.length,
|
|
1164
|
+
package: { file: path.join(base, pkg), members: members.length + 1,
|
|
1165
|
+
left_out: excluded.filter((e) => !e.name.startsWith("lib/") && !e.name.startsWith("reproduce/")).map((e) => `${e.name}: ${e.reason}`).slice(0, 20) },
|
|
1166
|
+
notices: ctx.notices.length, notes: [...MA.NOTES] };
|
|
1167
|
+
}
|
|
1168
|
+
//# sourceMappingURL=build_report.js.map
|