data-analyst-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -2
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +716 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +24 -0
- package/dist/report/scan_secrets.js +265 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +247 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +54 -4
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +170 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,716 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Checks shared by audit_report (reports, papers) and audit_answer (quick answers): a port of
|
|
3
|
+
* report/method_audit.py. <workspace>/decisions.json is the method record of every product (record_decisions
|
|
4
|
+
* writes it): the critical questions and how each was answered, the literature searched, the references opened
|
|
5
|
+
* and the assumptions with their basis. report.json may carry "references" and "assumptions" too; merged() joins
|
|
6
|
+
* both (report.json wins on the same id or parameter).
|
|
7
|
+
*
|
|
8
|
+
* Rules (all FAIL-level):
|
|
9
|
+
* questions an asked question offers exactly one "Don't know" option and no "Skip" option; a question not
|
|
10
|
+
* asked has reason_code no_user or settled_by_request (with "request", the user's quoted words); a
|
|
11
|
+
* skipped, unanswered or no_user question counts as "Don't know" and needs a resolution tied to an
|
|
12
|
+
* assumption that is not "user"; a blocking question decided without the user needs a rationale,
|
|
13
|
+
* and a skipped one needs one follow-up first
|
|
14
|
+
* literature at least one search entry with its query, date and what it found, or unavailable with
|
|
15
|
+
* reason_code no_search_tool, network_blocked or tool_unavailable
|
|
16
|
+
* disclosures no_user questions and unavailable searches need a line in "disclosures" the answer carries
|
|
17
|
+
* references citation, URL or DOI, accessed date and opened: true, backed by a snapshot the cite tool saved
|
|
18
|
+
* (references/opened.json: same URL, file unchanged since); a "quote" must be in that snapshot
|
|
19
|
+
* assumptions basis empirical > literature > precedent > user > judgment, each backed by its evidence
|
|
20
|
+
* sensitivity each exhibits/sensitivity.csv row's headlines are registered facts; conclusion_changes agrees
|
|
21
|
+
* data_needs every judgment assumption and every benchmark row "not checked" is named in a data_needs entry
|
|
22
|
+
* benchmark exhibits/benchmark.csv compares the headline numbers with published figures
|
|
23
|
+
*/
|
|
24
|
+
import crypto from "node:crypto";
|
|
25
|
+
import fs from "node:fs";
|
|
26
|
+
import path from "node:path";
|
|
27
|
+
import { rerunWorkspace } from "../analysis/rerun.js";
|
|
28
|
+
import { redact } from "../core/keys.js";
|
|
29
|
+
import { decodeText } from "../core/tabular.js";
|
|
30
|
+
import { pyFloatRepr } from "../core/py.js";
|
|
31
|
+
import { csvGrid, fmtG, match, readDictFile, repr, rx, str, strip, toFloat, unescapeHtml } from "./py.js";
|
|
32
|
+
/** Notes the audits print (not checks): collected per audit call. */
|
|
33
|
+
export const NOTES = [];
|
|
34
|
+
export const note = (m) => void NOTES.push(m);
|
|
35
|
+
export const BASES = ["empirical", "literature", "precedent", "user", "judgment"]; // strongest first
|
|
36
|
+
export const NO_ANSWER = ["dont_know", "skipped", "not_asked"];
|
|
37
|
+
export const NOT_ASKED_CODES = {
|
|
38
|
+
no_user: "nobody could answer (a headless or scheduled run)",
|
|
39
|
+
settled_by_request: "the request already settles it"
|
|
40
|
+
};
|
|
41
|
+
export const UNAVAILABLE_CODES = {
|
|
42
|
+
no_search_tool: "no web search or fetch tool exists in this environment, not even one that could be loaded",
|
|
43
|
+
network_blocked: "the environment's network blocks every search and fetch",
|
|
44
|
+
tool_unavailable: "the search and fetch tools exist but failed for the whole run (a usage limit or an outage): say when they were tried"
|
|
45
|
+
};
|
|
46
|
+
const DONT_KNOW_RE = /^\W*(?:i\s+)?don['’]?t\s+know/i;
|
|
47
|
+
const SKIP_RE = /^\W*skip\b/i;
|
|
48
|
+
const CITE_RE = /\[@[^\]]*\]/g;
|
|
49
|
+
const SENS_COLS = ["parameter", "base", "alternative", "headline_base", "headline_alt", "conclusion_changes"];
|
|
50
|
+
export const BENCH_COLS = ["measure", "period", "computed", "published", "source"];
|
|
51
|
+
const isObj = (x) => !!x && typeof x === "object" && !Array.isArray(x);
|
|
52
|
+
const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
|
|
53
|
+
export function load(base) {
|
|
54
|
+
const p = path.join(base, "decisions.json");
|
|
55
|
+
if (!fs.existsSync(p))
|
|
56
|
+
return {};
|
|
57
|
+
const raw = fs.readFileSync(p);
|
|
58
|
+
let text;
|
|
59
|
+
try {
|
|
60
|
+
text = new TextDecoder("utf-8", { fatal: true }).decode(raw).replace(/^/, "");
|
|
61
|
+
}
|
|
62
|
+
catch { // saved in a Windows code page: read it, and say so
|
|
63
|
+
note("decisions.json is not UTF-8; read as cp1252. Save it as UTF-8.");
|
|
64
|
+
text = decodeText(raw);
|
|
65
|
+
}
|
|
66
|
+
return JSON.parse(text);
|
|
67
|
+
}
|
|
68
|
+
/** R (report.json, or {}) with decisions.json's questions, literature, references and assumptions joined in. */
|
|
69
|
+
export function merged(base, R0) {
|
|
70
|
+
const R = { ...(R0 || {}) };
|
|
71
|
+
const D = load(base);
|
|
72
|
+
const refs = new Map();
|
|
73
|
+
for (const r of D.references || [])
|
|
74
|
+
refs.set(r?.id, r);
|
|
75
|
+
for (const r of R.references || [])
|
|
76
|
+
refs.set(r?.id, r);
|
|
77
|
+
const asm = new Map();
|
|
78
|
+
for (const a of D.assumptions || [])
|
|
79
|
+
asm.set(a?.parameter, a);
|
|
80
|
+
for (const a of R.assumptions || [])
|
|
81
|
+
asm.set(a?.parameter, a);
|
|
82
|
+
if (refs.size)
|
|
83
|
+
R.references = [...refs.values()];
|
|
84
|
+
if (asm.size)
|
|
85
|
+
R.assumptions = [...asm.values()];
|
|
86
|
+
for (const k of ["questions", "literature_search", "data_needs"]) {
|
|
87
|
+
const d = D[k] || [], r = R[k] || [];
|
|
88
|
+
if (d.length || r.length)
|
|
89
|
+
R[k] = [...d, ...r.filter((x) => !d.some((y) => same(x, y)))];
|
|
90
|
+
}
|
|
91
|
+
if ((D.disclosures && Object.keys(D.disclosures).length) || (R.disclosures && Object.keys(R.disclosures).length)) {
|
|
92
|
+
R.disclosures = { ...(D.disclosures || {}), ...(R.disclosures || {}) };
|
|
93
|
+
}
|
|
94
|
+
if ("data_needs" in D || "data_needs" in R)
|
|
95
|
+
if (!("data_needs" in R))
|
|
96
|
+
R.data_needs = D.data_needs ?? [];
|
|
97
|
+
if (D.data_needs_note && !R.data_needs_note)
|
|
98
|
+
R.data_needs_note = D.data_needs_note;
|
|
99
|
+
for (const k of ["audit_allow", "rounding_ties_ok"]) {
|
|
100
|
+
if (D[k] && D[k].length) {
|
|
101
|
+
const have = [...(R[k] || [])];
|
|
102
|
+
R[k] = [...have, ...D[k].filter((x) => !have.some((y) => same(x, y)))];
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return R;
|
|
106
|
+
}
|
|
107
|
+
export function citedRefs(text) {
|
|
108
|
+
const out = new Set();
|
|
109
|
+
for (const g of text.match(CITE_RE) || [])
|
|
110
|
+
for (const m of g.matchAll(/@([A-Za-z0-9_-]+)/g))
|
|
111
|
+
out.add(m[1]);
|
|
112
|
+
return out;
|
|
113
|
+
}
|
|
114
|
+
/** Was the question put to the user? decisions.json's "asked" (default true); record_decisions writes how: "not_asked"
|
|
115
|
+
* for a question it did not ask, which counts as asked: false. */
|
|
116
|
+
export function asked(q) {
|
|
117
|
+
return q.asked ?? q.how !== "not_asked";
|
|
118
|
+
}
|
|
119
|
+
function settled(q) {
|
|
120
|
+
return !asked(q) && q.reason_code === "settled_by_request" && !!strip(String(q.request ?? ""));
|
|
121
|
+
}
|
|
122
|
+
export function noAnswer(q) {
|
|
123
|
+
if (settled(q))
|
|
124
|
+
return false;
|
|
125
|
+
return [null, undefined, "", ...NO_ANSWER].includes(q.answer) || (!asked(q) && !q.answer);
|
|
126
|
+
}
|
|
127
|
+
const qidOf = (q) => q.id || String(q.question ?? "?").slice(0, 40);
|
|
128
|
+
/** [disclosure key, what was claimed] for every claim no script can check. */
|
|
129
|
+
export function escapes(R) {
|
|
130
|
+
const out = [];
|
|
131
|
+
for (const q of R.questions || []) {
|
|
132
|
+
if (!asked(q) && q.reason_code === "no_user")
|
|
133
|
+
out.push([qidOf(q), `question not asked: ${NOT_ASKED_CODES.no_user}`]);
|
|
134
|
+
}
|
|
135
|
+
const lit = R.literature_search || [];
|
|
136
|
+
if (lit.some((s) => s.unavailable)) {
|
|
137
|
+
const codes = [...new Set(lit.filter((s) => s.unavailable).map((s) => s.reason_code ?? ""))].sort();
|
|
138
|
+
out.push(["literature", "literature search not run: " + codes.map((c) => UNAVAILABLE_CODES[c] ??
|
|
139
|
+
(c ? `reason_code ${repr(c)} is not valid` : "no valid reason_code")).join("; ")]);
|
|
140
|
+
}
|
|
141
|
+
return out;
|
|
142
|
+
}
|
|
143
|
+
export function readCsv(p) {
|
|
144
|
+
const d = readDictFile(p);
|
|
145
|
+
return [d.fields, d.rows, d.extra];
|
|
146
|
+
}
|
|
147
|
+
// ---- references opened: the snapshot the cite tool saved
|
|
148
|
+
function openedLog(base) {
|
|
149
|
+
try {
|
|
150
|
+
const v = JSON.parse(fs.readFileSync(path.join(base, "references", "opened.json"), "utf8").replace(/^/, ""));
|
|
151
|
+
return isObj(v) ? v : {};
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
return {};
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
const urlKey = (u) => String(u ?? "").trim().split("#")[0].replace(/^https?:\/\//i, "").replace(/\/+$/, "").toLowerCase();
|
|
158
|
+
/** Text reduced to lowercase words, so a quote from a page's Markdown rendering matches its HTML. */
|
|
159
|
+
export function words(s) {
|
|
160
|
+
return unescapeHtml(s).normalize("NFKC").toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").split(" ").filter((w) => w).join(" ");
|
|
161
|
+
}
|
|
162
|
+
export function sha256(p) {
|
|
163
|
+
return crypto.createHash("sha256").update(fs.readFileSync(p)).digest("hex");
|
|
164
|
+
}
|
|
165
|
+
/** The words of a saved page; null for a PDF (no PDF text reader here: the quote is then not checked). */
|
|
166
|
+
function snapshotText(p) {
|
|
167
|
+
const ext = path.extname(p).toLowerCase();
|
|
168
|
+
if (ext === ".pdf")
|
|
169
|
+
return null;
|
|
170
|
+
let text = fs.readFileSync(p).toString("utf8");
|
|
171
|
+
if (ext === ".html" || ext === ".xml") {
|
|
172
|
+
text = text.replace(/<(script|style)\b[\s\S]*?<\/\1>/gi, " ");
|
|
173
|
+
text = text.replace(/<[^>]+>/g, " ");
|
|
174
|
+
}
|
|
175
|
+
return text;
|
|
176
|
+
}
|
|
177
|
+
function checkOpened(rid, r, opened, base) {
|
|
178
|
+
const url = r.url || r.doi || "<url>";
|
|
179
|
+
const e = opened[rid];
|
|
180
|
+
if (!e) {
|
|
181
|
+
return [false, `reference ${rid}: "opened": true, but references/opened.json has no snapshot of it: open it with the ` +
|
|
182
|
+
`cite tool ({"url": "${url}", "id": "${rid}"}; pass "text" with what your own fetch tool returned when this ` +
|
|
183
|
+
"server cannot reach the page); never cite what you have not opened", null];
|
|
184
|
+
}
|
|
185
|
+
if (r.url && urlKey(e.url) !== urlKey(r.url)) {
|
|
186
|
+
return [false, `reference ${rid}: the snapshot is of ${str(e.url)}, not ${r.url}: cite the page you opened`, null];
|
|
187
|
+
}
|
|
188
|
+
const p = path.join(base, String(e.file ?? ""));
|
|
189
|
+
if (!e.file || !fs.existsSync(p) || !fs.statSync(p).isFile() || sha256(p) !== e.sha256) {
|
|
190
|
+
return [false, `reference ${rid}: ${str(e.file)} is missing or changed since it was opened: call cite again`, null];
|
|
191
|
+
}
|
|
192
|
+
const msg = `reference ${rid}: opened ${String(e.opened ?? "").slice(0, 10)} (${str(e.via)})`;
|
|
193
|
+
const q = strip(String(r.quote ?? ""));
|
|
194
|
+
if (!q)
|
|
195
|
+
return [true, msg, null];
|
|
196
|
+
if (words(q).length < 20) {
|
|
197
|
+
return [false, `reference ${rid}: quote ${repr(q)} is too short to identify a passage: give a full sentence`, null];
|
|
198
|
+
}
|
|
199
|
+
const text = snapshotText(p);
|
|
200
|
+
if (text === null) {
|
|
201
|
+
note(`reference ${rid}: the quote was not checked (${e.file} is a PDF; this server reads no PDF text)`);
|
|
202
|
+
return [true, msg + "; quote not checkable", "uncheckable"];
|
|
203
|
+
}
|
|
204
|
+
if (!words(text).includes(words(q))) {
|
|
205
|
+
return [false, `reference ${rid}: the quote is not in ${e.file}: quote the page's own words (an exact sentence, not a ` +
|
|
206
|
+
"paraphrase)", null];
|
|
207
|
+
}
|
|
208
|
+
return [true, msg + "; quote found", "found"];
|
|
209
|
+
}
|
|
210
|
+
export function key(s) {
|
|
211
|
+
return String(s ?? "").toLowerCase().replace(/[^\p{L}\p{N}]+/gu, "_").replace(/^_+|_+$/g, "");
|
|
212
|
+
}
|
|
213
|
+
/** The CSV's file name, a column or a cell names the parameter (an empirical estimate's own table). */
|
|
214
|
+
function csvNames(p, param, maxRows = 5000) {
|
|
215
|
+
const k = key(param);
|
|
216
|
+
if (!k)
|
|
217
|
+
return false;
|
|
218
|
+
if (key(path.basename(p, path.extname(p))).includes(k))
|
|
219
|
+
return true;
|
|
220
|
+
const grid = csvGrid(decodeText(fs.readFileSync(p)));
|
|
221
|
+
for (let n = 0; n < grid.length; n++) {
|
|
222
|
+
if (n > maxRows)
|
|
223
|
+
break;
|
|
224
|
+
if (grid[n].some((c) => key(c) === k || (n === 0 && key(c).includes(k))))
|
|
225
|
+
return true;
|
|
226
|
+
}
|
|
227
|
+
return false;
|
|
228
|
+
}
|
|
229
|
+
export const NUM = rx(String.raw `(?i)^[^\d\-+.]*([-+]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?)\s*(%|percent|pct)?[^\d]*$`);
|
|
230
|
+
export const SAYS = ["match", "matches", "matched"]; // a benchmark.csv column that states the verdict
|
|
231
|
+
const BOUND = rx(String.raw `(?i)^\s*(>=|<=|>|<|≥|≤|over|more than|above|at least|under|below|less than|at most)\s*(.+)$`);
|
|
232
|
+
/** A sensitivity cell as [value, half a unit of its last digit as written]: "1.10" -> [1.1, 0.005]. */
|
|
233
|
+
function sensNumber(s) {
|
|
234
|
+
const m = match(NUM, strip(String(s ?? "")));
|
|
235
|
+
if (!m)
|
|
236
|
+
return null;
|
|
237
|
+
const txt = m[1].replace(/,/g, "");
|
|
238
|
+
return [Number(txt), txt.includes(".") ? 0.5 * 10 ** -txt.split(".")[1].length : 0.5];
|
|
239
|
+
}
|
|
240
|
+
function checkSensitivity(i, row, facts) {
|
|
241
|
+
const label = `sensitivity.csv line ${i} (${row.parameter ?? ""})`;
|
|
242
|
+
const values = factValues(facts), nums = {};
|
|
243
|
+
for (const side of ["base", "alt"]) {
|
|
244
|
+
const name = strip(String(row[`fact_${side}`] ?? "")), cell = row[`headline_${side}`];
|
|
245
|
+
const x = sensNumber(cell);
|
|
246
|
+
if (name) {
|
|
247
|
+
const v = toFloat((facts?.[name] || {}).value);
|
|
248
|
+
if (v === null) {
|
|
249
|
+
return [false, `${label}: fact_${side} ${repr(name)} is not a numeric fact in facts.json`];
|
|
250
|
+
}
|
|
251
|
+
if (x && !isFact(x[0], x[1] + Math.abs(x[0]) * 1e-9, [v]))
|
|
252
|
+
return [false, `${label}: headline_${side} ${str(cell)} is not fact ${name} (${fmtG(v)})`];
|
|
253
|
+
nums[side] = x ? x[0] : v;
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
if (x === null) {
|
|
257
|
+
return [false, `${label}: headline_${side} ${repr(cell)} is not one number: register the headline under each ` +
|
|
258
|
+
`alternative as a fact (the facts tool) and write it here (or name it in fact_${side})`];
|
|
259
|
+
}
|
|
260
|
+
if (!isFact(x[0], x[1] + Math.abs(x[0]) * 1e-9, values)) {
|
|
261
|
+
return [false, `${label}: headline_${side} ${str(cell)} is not a registered fact: run the ` +
|
|
262
|
+
`${side === "alt" ? "alternative" : "base case"} with compute and register its headline (fact_${side} names ` +
|
|
263
|
+
"it), so the rerun reproduces it"];
|
|
264
|
+
}
|
|
265
|
+
nums[side] = x[0];
|
|
266
|
+
}
|
|
267
|
+
const cc = strip(String(row.conclusion_changes ?? "")).toLowerCase();
|
|
268
|
+
const says = /^(yes|true)\b/.test(cc) ? "yes" : /^(no|false)\b/.test(cc) ? "no" : null;
|
|
269
|
+
if (says === null)
|
|
270
|
+
return [false, `${label}: conclusion_changes ${repr(cc)}: start it with yes or no`];
|
|
271
|
+
const b = nums.base, a = nums.alt;
|
|
272
|
+
const thr = sensNumber(row.threshold);
|
|
273
|
+
if (thr) {
|
|
274
|
+
// a sign flip is a change too (a fall that becomes a rise), whichever side of the threshold both are on
|
|
275
|
+
const crosses = (b - thr[0]) * (a - thr[0]) < 0, flips = b * a < 0;
|
|
276
|
+
if ((says === "yes") !== (crosses || flips)) {
|
|
277
|
+
const why = crosses ? `are on opposite sides of the threshold ${fmtG(thr[0])}` : flips ? "differ in sign"
|
|
278
|
+
: `are on the same side of the threshold ${fmtG(thr[0])} and of zero`;
|
|
279
|
+
return [false, `${label}: base ${fmtG(b)} and alternative ${fmtG(a)} ${why}, so conclusion_changes must be ` +
|
|
280
|
+
`${crosses || flips ? "yes" : "no"}`];
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
else if (says === "no" && b * a < 0) {
|
|
284
|
+
return [false, `${label}: the headline changes sign (${fmtG(b)} to ${fmtG(a)}), so the conclusion changes: write yes, ` +
|
|
285
|
+
"or give the threshold the conclusion turns on in a threshold column"];
|
|
286
|
+
}
|
|
287
|
+
return [true, `${label}: ${fmtG(b)} vs ${fmtG(a)}, conclusion changes: ${says}`];
|
|
288
|
+
}
|
|
289
|
+
/** [check, ok, message] for the method record in R (already merged with decisions.json). */
|
|
290
|
+
export function checkMethod(R, base, facts) {
|
|
291
|
+
const out = [];
|
|
292
|
+
const res = (check, ok, msg) => void out.push([check, !!ok, msg]);
|
|
293
|
+
const refs = new Map((R.references || []).map((r) => [r?.id, r]));
|
|
294
|
+
const asm = new Map((R.assumptions || []).map((a) => [a?.parameter, a]));
|
|
295
|
+
const questions = R.questions || [];
|
|
296
|
+
// ---- questions
|
|
297
|
+
for (const q of questions) {
|
|
298
|
+
const qid = qidOf(q);
|
|
299
|
+
if (asked(q)) {
|
|
300
|
+
const opts = (q.options || []).map((o) => str(o));
|
|
301
|
+
const dk = opts.filter((o) => DONT_KNOW_RE.test(o)), skip = opts.filter((o) => SKIP_RE.test(o));
|
|
302
|
+
const good = dk.length === 1 && !skip.length;
|
|
303
|
+
res("questions", good, good ? `question ${qid}: options ok` : `question ${qid}: offer exactly one "Don't know — decide ` +
|
|
304
|
+
`for me" option and no "Skip" (found ${dk.length} Don't know, ${skip.length} Skip)`);
|
|
305
|
+
}
|
|
306
|
+
else {
|
|
307
|
+
const code = q.reason_code;
|
|
308
|
+
if (!(code in NOT_ASKED_CODES)) {
|
|
309
|
+
res("questions", false, `question ${qid}: not asked; set "reason_code": "no_user" only when nobody can answer (a ` +
|
|
310
|
+
'headless or scheduled run; with a question tool, ask), or "settled_by_request" with "request": the user\'s own words');
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
if (code === "settled_by_request") {
|
|
314
|
+
res("questions", settled(q), settled(q) ? `question ${qid}: settled by the request (${repr(q.request)})`
|
|
315
|
+
: `question ${qid}: settled_by_request needs "request": the user's own words`);
|
|
316
|
+
continue;
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
if (!noAnswer(q))
|
|
320
|
+
continue;
|
|
321
|
+
const how = q.how || q.answer || "not_asked";
|
|
322
|
+
const a = asm.get(q.assumption);
|
|
323
|
+
const ok = !!q.resolution && a !== undefined && String(a?.basis || "").toLowerCase() !== "user";
|
|
324
|
+
res("questions", ok, ok ? `question ${qid}: ${how}, decided as ${repr(q.resolution)} (${str(a.basis)})`
|
|
325
|
+
: `question ${qid}: ${how} counts as "Don't know"; record the "resolution" and the "assumption" (a parameter in ` +
|
|
326
|
+
"the assumptions, basis other than user) that decides it");
|
|
327
|
+
if (q.blocking) {
|
|
328
|
+
const ok2 = !!q.rationale && (how !== "skipped" || !asked(q) || !!q.followed_up);
|
|
329
|
+
res("questions", ok2, ok2 ? `question ${qid}: blocking, decided without the user by ${repr(q.rationale)}; state it in ` +
|
|
330
|
+
"the opening paragraph of the answer, its disclosures line word for word"
|
|
331
|
+
: `question ${qid}: blocking (what the user means): ask one follow-up after a skip ("followed_up": true), then ` +
|
|
332
|
+
'record the literal reading as "rationale"');
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
// ---- literature search
|
|
336
|
+
const lit = R.literature_search || [];
|
|
337
|
+
if (!lit.length) {
|
|
338
|
+
res("literature", false, "no literature_search: search the agency methodology and research behind the definitions " +
|
|
339
|
+
"and parameters, and record each query, date and what it found");
|
|
340
|
+
}
|
|
341
|
+
lit.forEach((s, j) => {
|
|
342
|
+
const i = j + 1;
|
|
343
|
+
if (s.unavailable) {
|
|
344
|
+
const ok = s.reason_code in UNAVAILABLE_CODES;
|
|
345
|
+
res("literature", ok, ok ? `search ${i}: unavailable (${s.reason_code})` : `search ${i}: unavailable needs ` +
|
|
346
|
+
'"reason_code": "no_search_tool" (no search or fetch tool, not even a deferred one that could be loaded), ' +
|
|
347
|
+
'"network_blocked" or "tool_unavailable" (the tools failed for the whole run); otherwise run the search');
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
const ok = !!s.query && !!s.date && ("found" in s || !!s.note);
|
|
351
|
+
res("literature", ok, ok ? `search ${i}: ${repr(s.query)}` : `search ${i}: needs query, date and found (ids of the ` +
|
|
352
|
+
"references opened, or [] with a note)");
|
|
353
|
+
for (const rid of s.found || [])
|
|
354
|
+
if (!refs.has(rid))
|
|
355
|
+
res("literature", false, `search ${i}: found ${repr(rid)}, which is not in references`);
|
|
356
|
+
});
|
|
357
|
+
// ---- disclosures: what the gate cannot check, the reader must be told
|
|
358
|
+
const disc = R.disclosures || {};
|
|
359
|
+
for (const [k, what] of escapes(R)) {
|
|
360
|
+
const text = strip(String(disc[k] ?? ""));
|
|
361
|
+
res("disclosures", !!text, text ? `${k}: disclose in the answer: ${text}` : `${k}: ${what}. No script can check that ` +
|
|
362
|
+
`claim: add "disclosures": {"${k}": "<one line>"} to decisions.json (record_decisions) and carry that line in the ` +
|
|
363
|
+
"answer or report");
|
|
364
|
+
}
|
|
365
|
+
// ---- references
|
|
366
|
+
const opened = openedLog(base), verified = new Map();
|
|
367
|
+
for (const [rid, r] of refs) {
|
|
368
|
+
const miss = [["citation", r.citation], ["url or doi", r.url || r.doi], ["accessed", r.accessed],
|
|
369
|
+
["opened: true", r.opened === true]].filter(([, v]) => !v).map(([k]) => k);
|
|
370
|
+
if (miss.length) {
|
|
371
|
+
res("references", false, `reference ${str(rid)}: needs ${miss.join(", ")} (never cite what you have not opened)`);
|
|
372
|
+
continue;
|
|
373
|
+
}
|
|
374
|
+
const [ok, msg, quote] = checkOpened(String(rid), r, opened, base);
|
|
375
|
+
res("references", ok, msg);
|
|
376
|
+
if (ok)
|
|
377
|
+
verified.set(String(rid), quote);
|
|
378
|
+
}
|
|
379
|
+
for (const rid of [...citedRefs(JSON.stringify(R))].filter((x) => !refs.has(x)).sort()) {
|
|
380
|
+
res("references", false, `[@${rid}] is cited but not in references`);
|
|
381
|
+
}
|
|
382
|
+
if (!refs.size) {
|
|
383
|
+
res("references", false, "no references: cite the publications and research that the definitions, parameters and " +
|
|
384
|
+
"methods rely on");
|
|
385
|
+
}
|
|
386
|
+
// ---- assumptions
|
|
387
|
+
const sensPath = path.join(base, "exhibits", "sensitivity.csv");
|
|
388
|
+
let sensRows = null;
|
|
389
|
+
let sensList = [];
|
|
390
|
+
if (fs.existsSync(sensPath)) {
|
|
391
|
+
const [cols, rows] = readCsv(sensPath);
|
|
392
|
+
sensList = rows;
|
|
393
|
+
const miss = SENS_COLS.filter((c) => !cols.includes(c));
|
|
394
|
+
res("assumptions", !miss.length, !miss.length ? "exhibits/sensitivity.csv: columns ok"
|
|
395
|
+
: `exhibits/sensitivity.csv lacks columns ${repr(miss)}`);
|
|
396
|
+
sensRows = new Map(rows.map((r) => [r.parameter, r]));
|
|
397
|
+
}
|
|
398
|
+
const bases = {};
|
|
399
|
+
for (const [p, a] of asm) {
|
|
400
|
+
const b = String(a?.basis || "").toLowerCase();
|
|
401
|
+
bases[b] = (bases[b] || 0) + 1;
|
|
402
|
+
const ev = String(a?.evidence ?? "");
|
|
403
|
+
if (!BASES.includes(b)) {
|
|
404
|
+
res("assumptions", false, `assumption ${repr(p)}: basis ${repr(b || "missing")} is not one of ${BASES.join(" > ")}`);
|
|
405
|
+
continue;
|
|
406
|
+
}
|
|
407
|
+
if (["empirical", "literature", "precedent"].includes(b) && !strip(ev)) {
|
|
408
|
+
res("assumptions", false, `assumption ${repr(p)} (${b}): needs "evidence"`);
|
|
409
|
+
continue;
|
|
410
|
+
}
|
|
411
|
+
let ok, why;
|
|
412
|
+
if (b === "empirical") {
|
|
413
|
+
const paths = (ev.match(/(?:exhibits|data|published)\/[\w./-]+\.csv/g) || []).filter((x) => fs.existsSync(path.join(base, x)));
|
|
414
|
+
const fs_ = [...ev.matchAll(/\{\{\s*([A-Za-z_]\w*)/g)].map((m) => m[1]).filter((x) => facts && x in facts);
|
|
415
|
+
ok = fs_.length > 0 || paths.some((x) => csvNames(path.join(base, x), String(p)));
|
|
416
|
+
why = "name the {{fact}} that holds the estimate, or a data, exhibit or published CSV whose file name, a column or " +
|
|
417
|
+
`a row names the parameter ${repr(p)}: an estimate the evidence cannot point to was not made`;
|
|
418
|
+
}
|
|
419
|
+
else if (b === "literature" || b === "precedent") {
|
|
420
|
+
const idsCited = citedRefs(ev);
|
|
421
|
+
ok = [...idsCited].some((i) => verified.has(i) && ["found", "uncheckable"].includes(verified.get(i)));
|
|
422
|
+
why = 'cite the opened reference it comes from as [@id], with a "quote" in that reference: the sentence of the page ' +
|
|
423
|
+
"that supports it (the audit finds it in the saved snapshot)";
|
|
424
|
+
if (ok && match(NUM, strip(String(a?.value ?? ""))) && !strip(String(a?.why_not_empirical ?? ""))) {
|
|
425
|
+
ok = false;
|
|
426
|
+
why = "a number taken from the literature or an agency is a parameter the data may be able to estimate: fit the " +
|
|
427
|
+
"model the literature uses with compute (basis empirical, the published value as a sensitivity row), or state " +
|
|
428
|
+
"why_not_empirical (the data cannot identify it, or it is an official definition such as a threshold)";
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
else if (b === "user") {
|
|
432
|
+
const q = questions.find((x) => x.assumption === p || (x.id !== undefined && x.id === a?.question));
|
|
433
|
+
const quoted = !!strip(String(a?.request ?? ""));
|
|
434
|
+
ok = quoted || (q !== undefined && !noAnswer(q));
|
|
435
|
+
why = 'point to the question the user answered ("question": id), or quote the user\'s own words ("request") when ' +
|
|
436
|
+
"the request settles it";
|
|
437
|
+
}
|
|
438
|
+
else {
|
|
439
|
+
const key = a?.sensitivity || p;
|
|
440
|
+
const row = (sensRows || new Map()).get(key);
|
|
441
|
+
ok = !!strip(String(a?.why_not_empirical ?? "")) && row !== undefined && !!strip(String(row?.conclusion_changes ?? ""));
|
|
442
|
+
why = "judgment is the last resort: estimate it from the data or adopt a published estimate; otherwise state " +
|
|
443
|
+
"why_not_empirical and add its row to exhibits/sensitivity.csv (" + SENS_COLS.join(", ") + ")";
|
|
444
|
+
// "sensitivity" is the parameter of that row, not a description: say so when it names no row.
|
|
445
|
+
if (row === undefined && a?.sensitivity && sensRows?.size) {
|
|
446
|
+
why += `; "sensitivity" ${repr(key)} names no row: it is the row's parameter (${repr([...sensRows.keys()])}), ` +
|
|
447
|
+
"or leave it out when the row's parameter is the assumption's own";
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
res("assumptions", ok, ok ? `assumption ${repr(p)}: ${b} ok` : `assumption ${repr(p)} (${b}): ${why}`);
|
|
451
|
+
}
|
|
452
|
+
sensList.forEach((row, j) => res("sensitivity", ...checkSensitivity(j + 2, row, facts)));
|
|
453
|
+
if (!asm.size)
|
|
454
|
+
res("assumptions", false, "no assumptions: list each key definition and parameter with its basis and evidence");
|
|
455
|
+
// ---- data that would improve the estimates
|
|
456
|
+
const gaps = [...asm.entries()].filter(([, a]) => String(a?.basis || "").toLowerCase() === "judgment").map(([p]) => str(p));
|
|
457
|
+
const bench = path.join(base, "exhibits", "benchmark.csv");
|
|
458
|
+
if (fs.existsSync(bench)) {
|
|
459
|
+
for (const r of readCsv(bench)[1]) {
|
|
460
|
+
if (String(r.published ?? "").toLowerCase().includes("not checked") && r.measure)
|
|
461
|
+
gaps.push(r.measure);
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
const needs = R.data_needs;
|
|
465
|
+
const covered = new Set();
|
|
466
|
+
const norm = (x) => str(x).replace(/\s+/g, " ").trim().toLowerCase();
|
|
467
|
+
(needs || []).forEach((d, j) => {
|
|
468
|
+
const miss = ["data", "improves", "effect", "how_to_get"].filter((k) => !d?.[k] || (Array.isArray(d[k]) && !d[k].length));
|
|
469
|
+
const id = d?.id ?? j + 1;
|
|
470
|
+
res("data_needs", !miss.length, !miss.length ? `data need ${str(id)}: ok` : `data need ${str(id)}: needs ${miss.join(", ")}`);
|
|
471
|
+
const imp = d?.improves || [];
|
|
472
|
+
for (const x of typeof imp === "string" ? [imp] : imp)
|
|
473
|
+
covered.add(norm(x));
|
|
474
|
+
});
|
|
475
|
+
for (const g of [...new Set(gaps)]) {
|
|
476
|
+
const ok = covered.has(norm(g));
|
|
477
|
+
res("data_needs", ok, ok ? `${g}: data that would improve it are listed` : `${g} rests on judgment or an unchecked ` +
|
|
478
|
+
`benchmark: add a data_needs entry whose "improves" names it (the assumption's parameter or the benchmark.csv ` +
|
|
479
|
+
`measure, "${g}"), with the data, its effect, how to get it and from_user`);
|
|
480
|
+
}
|
|
481
|
+
if (needs === undefined || needs === null) {
|
|
482
|
+
res("data_needs", false, "no data_needs: list the data that would make the estimates more accurate, or [] with a " +
|
|
483
|
+
"data_needs_note saying why none would");
|
|
484
|
+
}
|
|
485
|
+
else if (!needs.length && !R.data_needs_note) {
|
|
486
|
+
res("data_needs", false, "data_needs is empty: add a data_needs_note saying why no data would help");
|
|
487
|
+
}
|
|
488
|
+
const n = Object.values(bases).reduce((x, y) => x + y, 0);
|
|
489
|
+
if (n) {
|
|
490
|
+
note("assumption bases: " + BASES.filter((k) => k in bases).map((k) => `${k} ${bases[k]}`).join(", ") +
|
|
491
|
+
` (judgment ${Math.round((100 * (bases.judgment || 0)) / n)}%)`);
|
|
492
|
+
}
|
|
493
|
+
return out;
|
|
494
|
+
}
|
|
495
|
+
/** A benchmark cell as [value, rounding tolerance]: "$72,380" -> [72380, 0.5], "0.3" -> [0.3, 0.05]. null when the
|
|
496
|
+
* cell is not one number ("219 D / 24 R", "not checked"). Zero decimals are dropped ("8.0" reads as 8). */
|
|
497
|
+
export function benchNumber(s) {
|
|
498
|
+
const m = match(NUM, strip(String(s ?? "")));
|
|
499
|
+
if (!m)
|
|
500
|
+
return null;
|
|
501
|
+
let txt = m[1].replace(/,/g, "");
|
|
502
|
+
if (txt.includes(".") && /^0*$/.test(txt.split(".")[1]))
|
|
503
|
+
txt = txt.replace(/0+$/, "").replace(/\.$/, "");
|
|
504
|
+
let tol;
|
|
505
|
+
if (txt.includes("."))
|
|
506
|
+
tol = 0.5 * 10 ** -txt.split(".")[1].length;
|
|
507
|
+
else {
|
|
508
|
+
const digits = txt.replace(/^[+-]+/, "");
|
|
509
|
+
const zeros = digits.replace(/^0+|0+$/g, "") ? digits.length - digits.replace(/0+$/, "").length : 0;
|
|
510
|
+
// one or two trailing zeros are usually exact (570 laws, 931,960 million); three or more (140,000) read as
|
|
511
|
+
// rounding, at most 0.1 percent
|
|
512
|
+
tol = zeros < 3 ? 0.5 : Math.max(0.5, Math.min(0.5 * 10 ** zeros, Math.abs(Number(txt)) * 0.001));
|
|
513
|
+
}
|
|
514
|
+
return [Number(txt), tol];
|
|
515
|
+
}
|
|
516
|
+
export function factValues(facts) {
|
|
517
|
+
const out = [];
|
|
518
|
+
for (const f of Object.values(facts || {})) {
|
|
519
|
+
const v = toFloat((f || {}).value);
|
|
520
|
+
if (v !== null)
|
|
521
|
+
out.push(v);
|
|
522
|
+
}
|
|
523
|
+
return out;
|
|
524
|
+
}
|
|
525
|
+
/** x (as written in benchmark.csv, maybe rounded and in percent or another scale) equals a registered fact. */
|
|
526
|
+
export function isFact(x, tol, values) {
|
|
527
|
+
for (const v of values) {
|
|
528
|
+
for (const k of [1, 100, 0.01, 1e3, 1e-3, 1e6, 1e-6, 1e9, 1e-9, 1e12, 1e-12]) {
|
|
529
|
+
if (Math.abs(Math.abs(v) * k - Math.abs(x)) <= tol * 1.0001 + 1e-12)
|
|
530
|
+
return true; // a fall may lose its sign
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
return false;
|
|
534
|
+
}
|
|
535
|
+
/** Computed and published agree within the published rounding, or the computed value's own (5.66 vs 5.65891). */
|
|
536
|
+
export function benchAgree(c, pb) {
|
|
537
|
+
return Math.abs(c[0] - pb[0]) <= Math.max(pb[1] * 1.0001, Math.abs(pb[0]) * 1e-6, Math.min(c[1] * 1.0001, Math.abs(pb[0]) * 0.005)) + 1e-12;
|
|
538
|
+
}
|
|
539
|
+
/** A note says the publisher rounds, and the gap is one its figure can show. */
|
|
540
|
+
export function roundingGap(c, pb, pub, noteText) {
|
|
541
|
+
const ps = String(pub ?? "");
|
|
542
|
+
const digits = ps.split(".")[0].replace(/\D/g, "");
|
|
543
|
+
const zeros = digits.replace(/^0+|0+$/g, "") && !ps.includes(".") ? digits.length - digits.replace(/0+$/, "").length : 0;
|
|
544
|
+
return /\bround/i.test(String(noteText ?? "")) && Math.abs(c[0] - pb[0]) <= Math.max(10 * pb[1], zeros ? 0.5 * 10 ** zeros : 0);
|
|
545
|
+
}
|
|
546
|
+
function checkHeadlines(facts, rows) {
|
|
547
|
+
if (!facts || !Object.keys(facts).length)
|
|
548
|
+
return [];
|
|
549
|
+
const heads = Object.entries(facts).filter(([, f]) => (f || {}).headline);
|
|
550
|
+
if (!heads.length) {
|
|
551
|
+
return [["benchmark", false, "no headline facts: register the numbers the answer leads with using the facts tool " +
|
|
552
|
+
"(headline: true); each needs a benchmark.csv row"]];
|
|
553
|
+
}
|
|
554
|
+
const out = [];
|
|
555
|
+
for (const [n, f] of heads) {
|
|
556
|
+
const keys = new Set([key(n), key(f.label)].filter((k) => k));
|
|
557
|
+
const fv = toFloat(f.value);
|
|
558
|
+
const v = fv === null ? [] : [fv];
|
|
559
|
+
let hit = false;
|
|
560
|
+
for (const r of rows) {
|
|
561
|
+
const c = benchNumber(r.computed);
|
|
562
|
+
if (keys.has(key(r.measure)) || (c && v.length && isFact(c[0], Math.max(c[1], Math.abs(c[0]) * 1e-9), v))) {
|
|
563
|
+
hit = true;
|
|
564
|
+
break;
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
out.push(["benchmark", hit, hit ? `headline ${n}: benchmarked` : `headline ${n}: no benchmark.csv row: compare it with ` +
|
|
568
|
+
`the agency's published figure (measure = ${n} or its label, computed = its value), or add a row with published ` +
|
|
569
|
+
'"not checked" and a data_needs entry for it']);
|
|
570
|
+
}
|
|
571
|
+
return out;
|
|
572
|
+
}
|
|
573
|
+
/** benchmark.csv exists with rows; each numeric computed/published pair agrees within the published figure's
|
|
574
|
+
* rounding unless a note explains the difference; a row never says it matches when it does not. */
|
|
575
|
+
export function checkBenchmark(base, facts) {
|
|
576
|
+
const p = path.join(base, "exhibits", "benchmark.csv");
|
|
577
|
+
if (!fs.existsSync(p)) {
|
|
578
|
+
return [["benchmark", false, "exhibits/benchmark.csv missing: compare each headline number with the agency's published " +
|
|
579
|
+
"figure (" + BENCH_COLS.join(", ") + ', url, accessed); "not checked" when none exists']];
|
|
580
|
+
}
|
|
581
|
+
const [cols, rows, extra] = readCsv(p);
|
|
582
|
+
const miss = BENCH_COLS.filter((c) => !cols.includes(c));
|
|
583
|
+
if (miss.length || !rows.length) {
|
|
584
|
+
return [["benchmark", false, `exhibits/benchmark.csv: ${miss.length ? "lacks columns " + repr(miss) : "no rows"}`]];
|
|
585
|
+
}
|
|
586
|
+
const out = [], values = factValues(facts);
|
|
587
|
+
let unchecked = 0, compared = 0, uncited = 0;
|
|
588
|
+
rows.forEach((r, j) => {
|
|
589
|
+
const i = j + 2;
|
|
590
|
+
const label = `benchmark.csv line ${i} (${r.measure ?? ""}, ${r.period ?? ""})`;
|
|
591
|
+
if (extra[j] || Object.values(r).includes(null)) { // an unquoted comma shifts every column after it
|
|
592
|
+
out.push(["benchmark", false, `${label}: ${extra[j] ? cols.length + extra[j].length : "fewer"} fields against the ` +
|
|
593
|
+
`header's ${cols.length}: quote fields that contain commas (write the file with compute or a CSV writer)`]);
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
const pub = String(r.published ?? "");
|
|
597
|
+
if (pub.toLowerCase().includes("not checked")) {
|
|
598
|
+
unchecked++;
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
if (!strip(pub)) {
|
|
602
|
+
out.push(["benchmark", false, `${label}: the published column is empty: give the published figure, or write "not ` +
|
|
603
|
+
'checked" in it (and say why in the note)']);
|
|
604
|
+
return;
|
|
605
|
+
}
|
|
606
|
+
const bound = match(BOUND, pub); // "over 45", ">= 45", "at most 2": a published bound, not a figure
|
|
607
|
+
if (bound && benchNumber(r.computed) && benchNumber(bound[2])) {
|
|
608
|
+
let c0 = benchNumber(r.computed)[0], b0 = benchNumber(bound[2])[0];
|
|
609
|
+
if (c0 < 0 && b0 < 0)
|
|
610
|
+
[c0, b0] = [-c0, -b0]; // a fall "over 45 percent" written -45: compare sizes
|
|
611
|
+
const up = [">", ">=", "≥", "over", "more than", "above", "at least"].includes(bound[1].toLowerCase());
|
|
612
|
+
if (up ? c0 >= b0 : c0 <= b0)
|
|
613
|
+
compared++;
|
|
614
|
+
else
|
|
615
|
+
out.push(["benchmark", false, `${label}: computed ${str(r.computed)} is outside the published bound ${pub}: find ` +
|
|
616
|
+
"the cause or explain it in the note"]);
|
|
617
|
+
return;
|
|
618
|
+
}
|
|
619
|
+
const c = benchNumber(r.computed), pb = benchNumber(pub);
|
|
620
|
+
const noteText = strip(String(r.note || r.notes || ""));
|
|
621
|
+
const sk = SAYS.find((k) => k in r && r[k]);
|
|
622
|
+
const says = sk ? strip(String(r[sk])).toLowerCase() : "";
|
|
623
|
+
if (c && values.length && !isFact(c[0], Math.max(c[1], Math.abs(c[0]) * 0.005), values))
|
|
624
|
+
uncited++;
|
|
625
|
+
if (c === null || pb === null) {
|
|
626
|
+
const flat = (x) => String(x ?? "").replace(/\s+/g, " ").trim().toLowerCase();
|
|
627
|
+
if (flat(r.computed) !== flat(pub) && !noteText && !says) {
|
|
628
|
+
out.push(["benchmark", false, `${label}: computed and published are not single numbers and differ; add a match ` +
|
|
629
|
+
"column (yes/no) and a note saying how they compare"]);
|
|
630
|
+
}
|
|
631
|
+
return;
|
|
632
|
+
}
|
|
633
|
+
compared++;
|
|
634
|
+
const agree = benchAgree(c, pb);
|
|
635
|
+
const rounding = roundingGap(c, pb, pub, noteText);
|
|
636
|
+
if (!agree && says.startsWith("yes") && !rounding) { // only a gap the size of a rounding is a match
|
|
637
|
+
out.push(["benchmark", false, `${label}: says it matches, but computed ${str(r.computed)} differs from published ` +
|
|
638
|
+
`${pub} by more than the published rounding (±${fmtG(pb[1])}); when the publisher rounds (to 4 significant ` +
|
|
639
|
+
'digits, to $10), say so in the note; when the gap is sampling error, write match "within MOE" (not yes) and ' +
|
|
640
|
+
"say so in the note; in the note"]);
|
|
641
|
+
}
|
|
642
|
+
else if (!agree && !noteText) {
|
|
643
|
+
out.push(["benchmark", false, `${label}: computed ${str(r.computed)} vs published ${pub} differ by more than rounding ` +
|
|
644
|
+
`(±${fmtG(pb[1])}); find the cause and fix it, or add a note column explaining the difference (definition, ` +
|
|
645
|
+
"vintage, universe) and say so in the answer"]);
|
|
646
|
+
}
|
|
647
|
+
});
|
|
648
|
+
out.push(...checkHeadlines(facts, rows));
|
|
649
|
+
if (uncited && !out.some(([, ok]) => !ok)) {
|
|
650
|
+
note(`benchmark.csv has ${uncited} computed value(s) not in facts.json: fine for cross-checks; the headline numbers the ` +
|
|
651
|
+
"answer cites should be registered facts");
|
|
652
|
+
}
|
|
653
|
+
if (!out.some(([, ok]) => !ok)) {
|
|
654
|
+
out.push(["benchmark", true, `${rows.length} benchmark rows: ${compared} compared within rounding or explained, ` +
|
|
655
|
+
`${unchecked} not checked` + (uncited ? `, ${uncited} computed values not in facts.json (cross-checks)` : "")]);
|
|
656
|
+
}
|
|
657
|
+
return out;
|
|
658
|
+
}
|
|
659
|
+
/** A fact read from a CSV cell must match that cell; one without a source needs a derivation. */
|
|
660
|
+
export function checkFact(name, f, base) {
|
|
661
|
+
if (f.source) {
|
|
662
|
+
const sp = path.join(base, f.source);
|
|
663
|
+
if (!fs.existsSync(sp))
|
|
664
|
+
return [false, `${name}: source ${f.source} does not exist`];
|
|
665
|
+
const where = isObj(f.where) ? f.where : {};
|
|
666
|
+
const rows = readCsv(sp)[1].filter((r) => Object.entries(where).every(([k, v]) => str(r[k]) === str(v)));
|
|
667
|
+
if (rows.length !== 1)
|
|
668
|
+
return [false, `${name}: ${rows.length} rows in ${f.source} match ${str(f.where ?? null)}`];
|
|
669
|
+
const raw = f.column !== undefined && f.column in rows[0] ? rows[0][f.column] : undefined;
|
|
670
|
+
const cell = raw === undefined ? null : toFloat(raw);
|
|
671
|
+
if (cell === null)
|
|
672
|
+
return [false, `${name}: column ${str(f.column ?? null)} missing or not numeric`];
|
|
673
|
+
const v = toFloat(f.value);
|
|
674
|
+
const ok = v !== null && Math.abs(cell - v) <= 1e-9 * Math.max(1.0, Math.abs(cell));
|
|
675
|
+
return [ok, ok ? `${name}: matches ${f.source}` : `${name}: facts.json ${str(f.value)} vs CSV ${pyFloatRepr(cell)}`];
|
|
676
|
+
}
|
|
677
|
+
if (!f.derivation)
|
|
678
|
+
return [false, `${name}: no source cell and no derivation`];
|
|
679
|
+
return [true, `${name}: derived (${String(f.derivation).slice(0, 60)})`];
|
|
680
|
+
}
|
|
681
|
+
/** --rerun: replay every exhibit's lineage and recompute every table fact (rerunWorkspace); facts.json and the
|
|
682
|
+
* tracked exhibit CSVs must come out the same. Nothing in the workspace is changed. */
|
|
683
|
+
export async function rerun(base, fpath, tracked) {
|
|
684
|
+
if (!fs.existsSync(fpath))
|
|
685
|
+
return [["replication", false, "--rerun needs facts.json; register the facts first (the facts tool)"]];
|
|
686
|
+
let r;
|
|
687
|
+
try {
|
|
688
|
+
r = await rerunWorkspace(base);
|
|
689
|
+
}
|
|
690
|
+
catch (e) {
|
|
691
|
+
return [["replication", false, "the rerun failed: " + redact(e instanceof Error ? e.message : String(e)).slice(-400)]];
|
|
692
|
+
}
|
|
693
|
+
const out = [];
|
|
694
|
+
const orig = JSON.parse(fs.readFileSync(fpath, "utf8"));
|
|
695
|
+
const diff = Object.entries(r.facts || {}).filter(([, v]) => !v.same).map(([k]) => k).sort();
|
|
696
|
+
const detail = (k) => r.facts[k].detail;
|
|
697
|
+
const tag = (k) => k + (!(k in orig) ? " (new: facts.json predates the recipe)" : detail(k) ? ` (${detail(k)})` : "");
|
|
698
|
+
out.push(["replication", !diff.length, !diff.length ? "rerun reproduces every fact exactly"
|
|
699
|
+
: `rerun changed facts: ${repr(diff.slice(0, 8).map(tag))}`]);
|
|
700
|
+
const rel = (p) => (path.isAbsolute(p) ? path.relative(base, p) : p).replaceAll("\\", "/");
|
|
701
|
+
const ex = new Map(Object.entries(r.exhibits || {}).map(([k, v]) => [rel(k), v]));
|
|
702
|
+
for (const t of [...new Set(tracked)]) {
|
|
703
|
+
const k = rel(t);
|
|
704
|
+
const e = ex.get(k);
|
|
705
|
+
if (!fs.existsSync(path.join(base, k))) {
|
|
706
|
+
if (e)
|
|
707
|
+
out.push(["replication", false, `rerun removed ${k}`]);
|
|
708
|
+
continue;
|
|
709
|
+
}
|
|
710
|
+
const why = e?.detail || "the saved file differs from what its recipe (exhibits/lineage.json) computes now";
|
|
711
|
+
if (e && !e.same)
|
|
712
|
+
out.push(["replication", false, `rerun changed ${k}: ${why} (the file was left as it was)`]);
|
|
713
|
+
}
|
|
714
|
+
return out;
|
|
715
|
+
}
|
|
716
|
+
//# sourceMappingURL=method_audit.js.map
|