data-analyst-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -2
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +716 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +24 -0
- package/dist/report/scan_secrets.js +265 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +247 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +54 -4
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +170 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,1349 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Release gate for a quick answer (the default product): a port of report/audit_answer.py. Run it on the reply as
|
|
3
|
+
* it will be sent. The same method rules as a report (method_audit.ts) on decisions.json, plus the parts of the
|
|
4
|
+
* report audit that apply without a page. FAIL-level checks:
|
|
5
|
+
* facts facts.json exists; every fact traced to a CSV cell matches that cell; others state a derivation
|
|
6
|
+
* answer answer.md (the reply as it will be sent): every number in it is a registered fact as rounded in the
|
|
7
|
+
* text, or listed in decisions.json "audit_allow" with a reason; dollar amounts of different years
|
|
8
|
+
* compared only with their basis stated (nominal or real), or "dollar_basis_ok" with a reason
|
|
9
|
+
* questions one "Don't know" option per question, no "Skip"; skipped or unanswered = "Don't know", resolved;
|
|
10
|
+
* not asked only with reason_code no_user or settled_by_request (quoting the request)
|
|
11
|
+
* literature the searches run (query, date, what was found), or reason_code no_search_tool / network_blocked
|
|
12
|
+
* disclosures each no_user question and unavailable search has a line in "disclosures" the answer carries
|
|
13
|
+
* references citation, URL or DOI, accessed date, opened (a page the cite tool saved, unchanged; its quote is
|
|
14
|
+
* on it); every [@id] resolves
|
|
15
|
+
* assumptions basis empirical > literature > precedent > user > judgment with its evidence
|
|
16
|
+
* sensitivity each sensitivity.csv row's headlines are registered facts and conclusion_changes agrees with them
|
|
17
|
+
* benchmark exhibits/benchmark.csv compares the headline numbers with published figures
|
|
18
|
+
* vintage every source in data/<id>/ has its provenance file; every table read by hand from an agency page,
|
|
19
|
+
* in published/<name>.csv, gives source_url and accessed on each row
|
|
20
|
+
* links with online, every external link in decisions.json answers HTTP < 400
|
|
21
|
+
* secrets no API key or key-shaped value in any file of the workspace
|
|
22
|
+
* replication with rerun, every exhibit recipe and table fact is replayed and reproduces facts.json and the
|
|
23
|
+
* exhibit CSVs exactly
|
|
24
|
+
*/
|
|
25
|
+
import fs from "node:fs";
|
|
26
|
+
import path from "node:path";
|
|
27
|
+
import { redact } from "../core/keys.js";
|
|
28
|
+
import { pyFloatRepr } from "../core/py.js";
|
|
29
|
+
import * as AR from "./audit_report.js";
|
|
30
|
+
import { roundingTie } from "../analysis/facts.js";
|
|
31
|
+
import * as MA from "./method_audit.js";
|
|
32
|
+
import * as PS from "./policyset.js";
|
|
33
|
+
import { echoSources, knownKeys, relHits, scanTree } from "./scan_secrets.js";
|
|
34
|
+
import { escapeRe, end, finditer, fullmatch, lstrip, match, repr, rx, search, sorted, split, splitlines, str, strip, sub, toFloat, pyRound } from "./py.js";
|
|
35
|
+
export const CHECKS = ["facts", "answer", "policy", "questions", "literature", "disclosures", "references", "assumptions",
|
|
36
|
+
"sensitivity", "data_needs", "benchmark", "vintage", "links", "secrets", "replication"];
|
|
37
|
+
const isObj = (x) => !!x && typeof x === "object" && !Array.isArray(x);
|
|
38
|
+
const isUpper = (c) => c !== c.toLowerCase() && c === c.toUpperCase();
|
|
39
|
+
const isDigit = (c) => /^\d$/.test(c);
|
|
40
|
+
/** str.rfind(ch, 0, endPos) */
|
|
41
|
+
const rfind = (s, ch, endPos) => (endPos <= 0 ? -1 : s.lastIndexOf(ch, endPos - ch.length));
|
|
42
|
+
const count = (s, ch, a = 0, b = s.length) => s.slice(a, b).split(ch).length - 1;
|
|
43
|
+
const cellsOf = (line) => strip(strip(line), "|").split("|");
|
|
44
|
+
// ---------------------------------------------------------------------- the answer text
|
|
45
|
+
const SCALE = { thousand: 1e3, million: 1e6, billion: 1e9, trillion: 1e12, k: 1e3, m: 1e6, mn: 1e6,
|
|
46
|
+
bn: 1e9, b: 1e9, t: 1e12, tn: 1e12 };
|
|
47
|
+
const ANSWER_NUM = rx(String.raw `(?i)(?<![\w.,\-–/#@:])(?P<neg>[-−])?\$?(?P<num>\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?)` +
|
|
48
|
+
String.raw `(?P<unit>[\s-]*(?:%|percent(?:age[\s-]points?)?|percentage[\s-]points?|pp\b|p\.p\.|basis[\s-]points?|bps?\b` +
|
|
49
|
+
String.raw `|(?:thousand|million|billion|trillion)\b|(?:k|m|mn|bn|b|t|tn)\b))?` +
|
|
50
|
+
String.raw `(?![\w/]|[-–](?!points?\b|percent|(?:days?|weeks?|months?|hours?|pages?|volumes?|records?|rows?|` +
|
|
51
|
+
String.raw `people|persons|jobs|units|members|employees)\b)|\.\d|,\d)`); // "9.9-point drop" and "43-day shutdown" are checked
|
|
52
|
+
const TALLY = rx(String.raw `(?<![\w.,/$#-])(\d{1,3})\s*[-–]\s*(\d{1,3})(?![\w/%]|[.,]\d|\s*(?:percent|years?|months?))`);
|
|
53
|
+
const VOTE_WORDS = rx(String.raw `(?i)\b(?:vote[sd]?|voting|roll|tally|yeas?|nays?|passed|failed|sustain\w*|overr\w*|` +
|
|
54
|
+
String.raw `adopted|confirmed|rejected|agreed|cloture|motion|margin)\b`);
|
|
55
|
+
const RANK_WORDS = rx(String.raw `(?i)\b(?:top|bottom|first|last|next|past|other)\s+\d{1,2}\b` +
|
|
56
|
+
// "all 50 states" names a universe; "all 79 succeeded" is a result to check
|
|
57
|
+
String.raw `|\ball\s+\d{1,2}(?=\s+(?:[a-z-]+\s+)?(?:states?|areas?|jurisdictions?|departments?|` +
|
|
58
|
+
String.raw `agencies|counties|districts|metros?|cities|countries|sectors|industries|categories|groups|` +
|
|
59
|
+
String.raw `members|senators|committees|months|quarters|years|weeks|days|volumes|series|rows|columns|` +
|
|
60
|
+
String.raw `components|occupations|figures|values|estimates)\b)`);
|
|
61
|
+
const FLAG_WORDS = {
|
|
62
|
+
preliminary: rx(String.raw `preliminar|subject to revision|advance estimate|first estimate|early estimate|(?:will|to) be revised|provisional`, "i"),
|
|
63
|
+
projection: rx(String.raw `project|forecast|estimate|expect|outlook`, "i"),
|
|
64
|
+
vintage: rx(String.raw `as (?:first )?(?:reported|published|estimated)|first (?:release|estimate|reported)|` +
|
|
65
|
+
String.raw `initial(?:ly)?|vintage|real[- ]time|at the time|originally|first published|revis|` +
|
|
66
|
+
String.raw `(?:advance|second|third|earlier|annual|comprehensive|benchmark)\s+(?:estimates?|updates?|releases?|revisions?)`, "i")
|
|
67
|
+
};
|
|
68
|
+
// a year qualifies these ("2017 dollars", "2025 estimates"); another plural after 1789-2100 makes it a count
|
|
69
|
+
const YEAR_NOUNS = new Set(("dollars prices levels estimates values data figures rates terms numbers quarters months years " +
|
|
70
|
+
"editions vintages releases budgets elections sessions laws acts amendments appropriations reports surveys tables files " +
|
|
71
|
+
"samples weights results rules standards projections baselines totals peaks highs lows averages volumes issues q1 q2 q3 q4").split(" "));
|
|
72
|
+
const VERBS_S = new Set(("does goes shows appears remains marks sees seems ends begins starts covers includes contains lists " +
|
|
73
|
+
"looks gives brings rises falls stands follows means makes takes holds runs gets comes becomes needs uses matches leads adds " +
|
|
74
|
+
"plus thus less across towards versus always also").split(" "));
|
|
75
|
+
// a bare 1789-2100 integer in prose is a count only after such a word and before a plural noun ("had 2017 pages")
|
|
76
|
+
const COUNT_BEFORE = rx(String.raw `(?i)\b(?:had|has|have|against|totaled|totalled|totaling|totalling|were|are|was|is|` +
|
|
77
|
+
String.raw `about|nearly|almost|roughly|approximately|exactly)\s+$`);
|
|
78
|
+
const SEP_ROW = rx(String.raw `\s*\|?[\s:|-]+\|?\s*`);
|
|
79
|
+
const YEAR_COL = rx(String.raw `\**\s*(?:years?|period|fy|fiscal|calendar|date|vintage|edition|session|` +
|
|
80
|
+
String.raw `congress|month|quarter|as of|latest|release)\b`);
|
|
81
|
+
/** Is this bare 1789-2100 integer a year? It is, unless it reads as a count. */
|
|
82
|
+
function yearLike(t, m, text) {
|
|
83
|
+
const a = rfind(t, "\n", m.index) + 1;
|
|
84
|
+
const b = t.indexOf("\n", end(m));
|
|
85
|
+
const line = b < 0 ? t.slice(a) : t.slice(a, b);
|
|
86
|
+
if (lstrip(line).startsWith("|")) {
|
|
87
|
+
const cells = cellsOf(line);
|
|
88
|
+
const cell = count(line.slice(0, m.index - a), "|") - 1;
|
|
89
|
+
const nxt = b >= 0 ? t.slice(b + 1).split("\n")[0] : "";
|
|
90
|
+
if (cell <= 0 || (fullmatch(SEP_ROW, nxt) && nxt.includes("-")))
|
|
91
|
+
return true; // first column or header row
|
|
92
|
+
const orig = text ? cellsOf(sourceLine(text, t, m.index)) : cells;
|
|
93
|
+
if (cell >= orig.length || strip(strip(strip(orig[cell]), "*")) !== strip(m[0]))
|
|
94
|
+
return true; // a label or a date
|
|
95
|
+
const rows = splitlines(t.slice(0, a));
|
|
96
|
+
const sep = rows.map((l, i) => (fullmatch(SEP_ROW, l) && l.includes("-") ? i : -1)).filter((i) => i >= 0);
|
|
97
|
+
if (sep.length && sep[sep.length - 1] >= 1) {
|
|
98
|
+
const head = cellsOf(rows[sep[sep.length - 1] - 1]).map((c) => strip(c).toLowerCase());
|
|
99
|
+
const col = cell < head.length ? head[cell] : "";
|
|
100
|
+
return !!match(YEAR_COL, col);
|
|
101
|
+
}
|
|
102
|
+
return true;
|
|
103
|
+
}
|
|
104
|
+
if (!search(COUNT_BEFORE, t.slice(Math.max(0, m.index - 20), m.index)))
|
|
105
|
+
return true;
|
|
106
|
+
const after = match(/\s+([a-z]{4,}s)\b/, t.slice(end(m), end(m) + 40));
|
|
107
|
+
return !(after && !YEAR_NOUNS.has(after[1]) && !VERBS_S.has(after[1]));
|
|
108
|
+
}
|
|
109
|
+
const APPROX = rx(String.raw `(?i)\b(?:about|nearly|almost|roughly|approximately|around|some|close to|more than|less than|over|under)\s*\$?$`);
|
|
110
|
+
// a unit shared with the next number: "3.4 and 3.6 percent", "between 2.1 and 2.5 percent"
|
|
111
|
+
const UNIT_AFTER = rx(String.raw `(?i)(?:\s*(?:,|and|or|to|–|-)\s*(?:[A-Za-z]+\s+){0,3}[-−+]?\d[\d,.]*)*\s*(?:%|percent|percentage points?|points?|pp\b|basis points?)`);
|
|
112
|
+
const COUNT_NOUN = rx(String.raw `(?i)\s+(?:of\s+(?:the\s+)?\d|(?!percent|points?|pp\b|basis|in\b|and\b|or\b|to\b|vs\b|from\b|at\b|for\b|on\b|by\b)[a-z]{3,})`);
|
|
113
|
+
const tokRe = (tok) => new RegExp(String.raw `(?<![\w.,$-])` + escapeRe(tok) + String.raw `(?![\d])`);
|
|
114
|
+
/** Where the number tok stands in x as a whole number (not "0.6 percent" inside "30.6 percent"); -1 if not. */
|
|
115
|
+
export function at(x, tok) {
|
|
116
|
+
const m = tokRe(tok).exec(x);
|
|
117
|
+
return m ? m.index : -1;
|
|
118
|
+
}
|
|
119
|
+
const has = (x, tok) => at(x, tok) >= 0;
|
|
120
|
+
/** The line of the original text that holds position pos of its exempted copy t. */
|
|
121
|
+
function sourceLine(text, t, pos) {
|
|
122
|
+
const i = count(t, "\n", 0, pos);
|
|
123
|
+
const lines = text.split("\n");
|
|
124
|
+
if (count(text, "\n") === count(t, "\n") && i < lines.length)
|
|
125
|
+
return lines[i];
|
|
126
|
+
const j = t.indexOf("\n", pos);
|
|
127
|
+
return t.slice(rfind(t, "\n", pos) + 1, j >= 0 ? j : t.length);
|
|
128
|
+
}
|
|
129
|
+
const PERIOD_COUNT = rx(String.raw `(?i)\b\d{1,2}\s+of\s+(?:the\s+)?\d{1,2}\s+(?:days|weeks|months|quarters|years)\b|\(\d\)`);
|
|
130
|
+
/** The statistics in an answer: {tok, num, unit ("pct", "pp", "bp", "scaled" or ""), scale, tol}. Years, dates, table
|
|
131
|
+
* and line references, bill and law numbers, ordinals, list markers and "top 5" are skipped. */
|
|
132
|
+
export function answerNumbers(text) {
|
|
133
|
+
const t = sub(PERIOD_COUNT, " ", sub(RANK_WORDS, " ", AR.exempt(text)));
|
|
134
|
+
const out = [];
|
|
135
|
+
for (const m of finditer(ANSWER_NUM, t)) {
|
|
136
|
+
const g = m.groups;
|
|
137
|
+
const num = g.num.replace(/,/g, ""), unit = strip(g.unit || "", " -").toLowerCase();
|
|
138
|
+
const v = Number(num);
|
|
139
|
+
if (!unit && !num.includes(".") && !g.num.includes(",") && v >= 1789 && v <= 2100 && yearLike(t, m, text))
|
|
140
|
+
continue;
|
|
141
|
+
const kind = ["percentage", "pp", "p.p"].some((p) => unit.startsWith(p)) ? "pp" : ["%", "percent"].some((p) => unit.startsWith(p)) ? "pct"
|
|
142
|
+
: ["basis", "bp"].some((p) => unit.startsWith(p)) ? "bp" : unit ? "scaled" : "";
|
|
143
|
+
const dec = num.includes(".") ? num.split(".")[1].length : 0;
|
|
144
|
+
const line = sourceLine(text, t, m.index);
|
|
145
|
+
out.push({ tok: strip(m[0]), num: v, unit: kind, scale: SCALE[unit] ?? 1.0, tol: 0.5 * 10 ** -dec, dec,
|
|
146
|
+
neg: !!g.neg, dollar: m[0].includes("$"), plus: !g.neg && t.slice(Math.max(0, m.index - 1), m.index) === "+", line,
|
|
147
|
+
approx: !!search(APPROX, t.slice(Math.max(0, m.index - 25), m.index)),
|
|
148
|
+
unit_after: !!match(UNIT_AFTER, t.slice(end(m), end(m) + 40)),
|
|
149
|
+
counts: !num.includes(".") && !!match(COUNT_NOUN, t.slice(end(m), end(m) + 25)),
|
|
150
|
+
table: lstrip(line).startsWith("|") });
|
|
151
|
+
}
|
|
152
|
+
for (const m of finditer(TALLY, t)) { // "passed 236-188", "| 248-177 |": yeas and nays, each a fact
|
|
153
|
+
const line = sourceLine(text, t, m.index);
|
|
154
|
+
if (!search(VOTE_WORDS, t.slice(Math.max(0, m.index - 45), end(m) + 25)) && !lstrip(line).startsWith("|"))
|
|
155
|
+
continue;
|
|
156
|
+
if (search(rx(String.raw `(?i)\b(?:votes?|rolls?|roll\s+calls?|numbers?|nos?\.?|#)\s*$`), t.slice(Math.max(0, m.index - 15), m.index)))
|
|
157
|
+
continue;
|
|
158
|
+
for (const gi of [1, 2]) {
|
|
159
|
+
out.push({ tok: m[gi], num: Number(m[gi]), unit: "", scale: 1.0, tol: 0.5, dec: 0, neg: false, dollar: false, plus: false,
|
|
160
|
+
line, approx: false, unit_after: false, counts: true, table: lstrip(line).startsWith("|") });
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return out;
|
|
164
|
+
}
|
|
165
|
+
/** A fact's value (with its sign) in the unit its format shows, that unit's kind and the format's digits. */
|
|
166
|
+
export function factDisplay(f) {
|
|
167
|
+
let v = toFloat(f?.value);
|
|
168
|
+
if (v === null || f?.value === null || f?.value === undefined)
|
|
169
|
+
return null;
|
|
170
|
+
const fmt = String(f.fmt || "");
|
|
171
|
+
const nums = fmt.split(":").filter((p) => /^\d+$/.test(lstrip(p, "-")));
|
|
172
|
+
const digits = nums.length ? Number(nums[0]) : null;
|
|
173
|
+
const parts = fmt.split(":").filter((p) => p && !/^\d+$/.test(lstrip(p, "-")) && p !== "abs");
|
|
174
|
+
let mult = 1.0;
|
|
175
|
+
while (parts.length && ["signed", "abs", "from_thousands", "from_millions", "from_billions"].includes(parts[0])) {
|
|
176
|
+
mult *= { from_thousands: 1e3, from_millions: 1e6, from_billions: 1e9 }[parts.shift()] ?? 1.0;
|
|
177
|
+
}
|
|
178
|
+
const kind = parts.length ? parts[0] : "";
|
|
179
|
+
v *= mult;
|
|
180
|
+
if (kind === "pct" || kind === "pp")
|
|
181
|
+
return [v * 100, kind, digits];
|
|
182
|
+
if (kind === "pct100" || kind === "pp100")
|
|
183
|
+
return [v, kind.slice(0, -3), digits];
|
|
184
|
+
if (kind === "bp" || kind === "bp100")
|
|
185
|
+
return [v * (kind === "bp" ? 1e4 : 100), "bp", digits];
|
|
186
|
+
if (kind === "thousands" || kind === "kmillions")
|
|
187
|
+
return [v * 1e3, "base", digits];
|
|
188
|
+
if (kind.endsWith("_scaled")) {
|
|
189
|
+
const k = { m: 1e6, b: 1e9, t: 1e12 }[kind[4]];
|
|
190
|
+
if (k === undefined)
|
|
191
|
+
return [v, "base", digits];
|
|
192
|
+
return [v * k, "money", digits];
|
|
193
|
+
}
|
|
194
|
+
if (kind.startsWith("dollars") || kind.startsWith("usd"))
|
|
195
|
+
return [v, "money", digits];
|
|
196
|
+
if (kind)
|
|
197
|
+
return [v, "base", digits];
|
|
198
|
+
return [v, "", digits];
|
|
199
|
+
}
|
|
200
|
+
/** Does answer number n show this fact? */
|
|
201
|
+
export function matches(n, shown, kind, digits = null) {
|
|
202
|
+
const dec = n.dec ?? 0;
|
|
203
|
+
if ((n.neg && shown >= 0) || (n.dollar && !["money", "", "pub", "pubnum"].includes(kind))
|
|
204
|
+
|| (n.plus && shown < -n.tol) || (n.table && shown < -n.tol && !n.neg))
|
|
205
|
+
return false; // a table cell states its sign
|
|
206
|
+
shown = Math.abs(shown);
|
|
207
|
+
kind = kind === "money" ? "base" : kind;
|
|
208
|
+
let x = n.num, tol = n.tol * 1.0001 + 1e-12;
|
|
209
|
+
let cands;
|
|
210
|
+
if (["pubpct", "pubpp", "pubnum", "pub"].includes(kind)) { // a published benchmark figure, as benchmark.csv writes it
|
|
211
|
+
if (digits !== null && dec < digits)
|
|
212
|
+
return false; // quoted at its published precision
|
|
213
|
+
const units = { pubpct: ["pct", ""], pubpp: ["pp", ""], pubnum: ["", "scaled"] }[kind];
|
|
214
|
+
const okUnit = (units === undefined || units.includes(n.unit)) && !(n.dollar && (kind === "pubpct" || kind === "pubpp"));
|
|
215
|
+
return okUnit && Math.abs(x - shown) <= tol;
|
|
216
|
+
}
|
|
217
|
+
if (n.unit === "pct" || n.unit === "pp") { // a percent is not a percentage-point difference
|
|
218
|
+
if ((kind === "pct" || kind === "pp") && digits !== null && dec < Math.max(digits, 0) && !n.approx)
|
|
219
|
+
return false;
|
|
220
|
+
cands = kind === n.unit ? [shown] : kind === "" ? [shown, shown * 100] : [];
|
|
221
|
+
}
|
|
222
|
+
else if (n.unit === "bp") {
|
|
223
|
+
cands = kind === "bp" ? [shown] : kind === "" ? [shown * 1e4, shown * 100] : [];
|
|
224
|
+
}
|
|
225
|
+
else if (n.unit === "scaled") {
|
|
226
|
+
if (kind === "" && Math.abs(x - shown) <= tol)
|
|
227
|
+
return true; // a published figure kept in its table's unit
|
|
228
|
+
x *= n.scale;
|
|
229
|
+
tol *= n.scale;
|
|
230
|
+
cands = kind === "base" || kind === "" ? [shown] : [];
|
|
231
|
+
}
|
|
232
|
+
else if (["pct", "pp", "bp"].includes(kind) && !n.table && n.counts && !n.unit_after) {
|
|
233
|
+
return false; // a bare count ("13 of the 15") is not the fact "-13 percent"
|
|
234
|
+
}
|
|
235
|
+
else if (["pct", "pp", "bp"].includes(kind) && digits !== null && dec !== Math.max(digits, 0)) {
|
|
236
|
+
return false; // a bare "70" is not the fact shown as "70.4 percent"
|
|
237
|
+
}
|
|
238
|
+
else {
|
|
239
|
+
if (kind === "base" && digits !== null && digits > 0 && dec < digits && !n.approx && Math.abs(x - shown) <= tol)
|
|
240
|
+
return false;
|
|
241
|
+
const down = (n.tok.includes(".") || n.tok.includes(",")) && n.table ? [1e-3, 1e-6, 1e-9, 1e-12] : [];
|
|
242
|
+
cands = ["pct", "pp", "bp"].includes(kind) ? [shown] : kind === "base" ? [1, ...down].map((k) => shown * k)
|
|
243
|
+
: [1, 100, ...down].map((k) => shown * k);
|
|
244
|
+
}
|
|
245
|
+
return cands.some((c) => Math.abs(x - c) <= tol);
|
|
246
|
+
}
|
|
247
|
+
function norm(s) {
|
|
248
|
+
return strip(strip(String(s).replace(/[*_`>]/g, "").replace(/\s+/g, " ")), ".").toLowerCase();
|
|
249
|
+
}
|
|
250
|
+
const NUMBER_WORDS = Object.fromEntries("zero one two three four five six seven eight nine ten eleven twelve"
|
|
251
|
+
.split(" ").map((w, i) => [w, i]));
|
|
252
|
+
export const DOWN_WORDS = rx(String.raw `(?i)\b(?:fell|fall(?:s|ing|en)?|declin\w*|decreas\w*|drop\w*|down|lower\w*|less|loss\w*|` +
|
|
253
|
+
String.raw `lost|negative|shr[ai]nk\w*|shrunk|contract\w*|slow\w*|below|cut|reduc\w*|minus|deficit\w*|` +
|
|
254
|
+
String.raw `behind|short(?:fall)?|trail\w*|lagg\w*|` +
|
|
255
|
+
String.raw `weak\w*|slid\w*|sank|sunk|tumbl\w*|plung\w*|gap|narrow\w*|smaller|fewer|worse|` +
|
|
256
|
+
String.raw `subtract\w*|drag\w*|detract\w*|held back|hold\w* back|offset\w*|outflow\w*|` +
|
|
257
|
+
String.raw `withdr[ae]w\w*|erod\w*|deduct\w*|refund\w*|deficit|shortfall|net of)\b`);
|
|
258
|
+
export const UP_WORDS = rx(String.raw `(?i)\b(?:rose|rise[sn]?|rising|gain\w*|increas\w*|grew|grow\w*|added|adding|up|higher|improv\w*|` +
|
|
259
|
+
String.raw `climb\w*|jump\w*|surg\w*|expand\w*)\b`);
|
|
260
|
+
/** An audit_allow entry and an answer number compare as written, unit included. */
|
|
261
|
+
export function allowKey(s0) {
|
|
262
|
+
let s = str(s0);
|
|
263
|
+
const m = /^\s*(\$?[\d,.]+)\s+([a-z]+)\s*$/i.exec(s);
|
|
264
|
+
if (m && !/^(?:percent\w*|thousand|million|billion|trillion|basis|points?|pp|bps?)$/i.test(m[2]))
|
|
265
|
+
s = m[1]; // "60 days"
|
|
266
|
+
const k = s.replace(/[\s$,\-]/g, "").toLowerCase();
|
|
267
|
+
return k.replace(/percent$/, "%");
|
|
268
|
+
}
|
|
269
|
+
const below0 = (f) => {
|
|
270
|
+
const v = toFloat(f?.value);
|
|
271
|
+
return v !== null && f?.value !== null && v < 0;
|
|
272
|
+
};
|
|
273
|
+
const negative = (f) => below0(f) && !String(f?.fmt || "").includes("abs");
|
|
274
|
+
const isChange = (f) => /(?:^|:)(?:pp|pp100|bp|bp100|signed)(?::|$)/.test(String(f?.fmt || ""));
|
|
275
|
+
const cellKey = (x) => str(x).replace(/[\s*_`]+/g, " ").trim().toLowerCase();
|
|
276
|
+
const NA_CELL = rx(String.raw `(?i)\s*\**\s*(?:n/?a|none|nil|not\s+(?:published|available|reported|applicable|` +
|
|
277
|
+
String.raw `checked|on\s+\w+)|missing|withheld|suppressed|[(]?na[)]?|x|—|–|-|\.\.\.)\s*\**\s*`);
|
|
278
|
+
const RANK_CELL = rx(String.raw `(?i)\s*\**\s*(?:#?\d+|(?:rank|no\.?|oldest|youngest|top|bottom|highest|lowest|first|` +
|
|
279
|
+
String.raw `last)\s*\d+)\s*(?:\(tie\)|=)?\s*\**\s*`);
|
|
280
|
+
/** The label of a table cell's row; null outside tables, for a period or an unreadable row. */
|
|
281
|
+
export function rowLabel(n) {
|
|
282
|
+
const line = n.line || "";
|
|
283
|
+
if (!lstrip(line).startsWith("|"))
|
|
284
|
+
return null;
|
|
285
|
+
let cells = cellsOf(line);
|
|
286
|
+
let k = cells.findIndex((c) => has(c, n.tok));
|
|
287
|
+
if (k < 0)
|
|
288
|
+
k = cells.length;
|
|
289
|
+
const names = cells.slice(0, k).filter((c) => /[A-Za-z]/.test(c) && !/\d/.test(c) && !fullmatch(NA_CELL, c));
|
|
290
|
+
const rank = fullmatch(RANK_CELL, cells[0]);
|
|
291
|
+
if (names.length && (!/[A-Za-z]/.test(cells[0]) || rank))
|
|
292
|
+
cells = [names[names.length - 1]];
|
|
293
|
+
else if (names.length > 1)
|
|
294
|
+
cells = [names[names.length - 1]];
|
|
295
|
+
const row = cellKey(cells[0]);
|
|
296
|
+
if (!row || row.length > 60 || periodKey(row.replace(/\s*\(.*\)\s*$/, "")) !== null
|
|
297
|
+
|| !/[a-z]{3}|\b\d+(?:st|nd|rd|th)\b|\b[a-z][a-z.]{0,10}\s*\d{1,6}\b/.test(row)
|
|
298
|
+
|| /^(?:(?:fy\s?)?\d{4}\s*(?:[–-]|to)\s*\d{2,4}|\d{4}\s*q[1-4]|q[1-4]\s*\d{4})$/.test(row))
|
|
299
|
+
return null;
|
|
300
|
+
return row;
|
|
301
|
+
}
|
|
302
|
+
/** A name as written and its short form: "Natural Resources Committee" -> natural resources. */
|
|
303
|
+
export function forms(v0) {
|
|
304
|
+
const v = cellKey(v0);
|
|
305
|
+
let short = v.replace(/^(?:the\s+|u\.?s\.?\s+|(?:house|senate)\s+)?(?:committee\s+on\s+(?:the\s+)?|department\s+of\s+(?:the\s+)?)/, "");
|
|
306
|
+
short = short.replace(/\s+(?:committee|department|agency|administration|bureau|office|county|state)$/, "");
|
|
307
|
+
const alias = new Set(["united states", "u.s. total", "united states total"].includes(v) ? ["u.s."] : []);
|
|
308
|
+
const party = { democratic: ["democrat", "democrats"], democrat: ["democrats", "democratic"],
|
|
309
|
+
republican: ["republicans"], independent: ["independents"] };
|
|
310
|
+
for (const p of party[v] || [])
|
|
311
|
+
alias.add(p);
|
|
312
|
+
const out = new Set([v, ...alias]);
|
|
313
|
+
if (short !== v && short.length >= 4 && /[a-z]{3}/.test(short))
|
|
314
|
+
out.add(short);
|
|
315
|
+
return out;
|
|
316
|
+
}
|
|
317
|
+
/** Names the facts are keyed by, each form to [keys, the name]; periods and codes are not names. */
|
|
318
|
+
export function entities(facts) {
|
|
319
|
+
const out = new Map();
|
|
320
|
+
for (const f of Object.values(facts || {})) {
|
|
321
|
+
if (!isObj(f))
|
|
322
|
+
continue;
|
|
323
|
+
for (const [k, v] of Object.entries(isObj(f.where) ? f.where : {})) {
|
|
324
|
+
const name = cellKey(v);
|
|
325
|
+
if (name.length >= 3 && name.length <= 60 && /[a-z]{3}/.test(name) && !/\d/.test(name)) {
|
|
326
|
+
for (const form of forms(name)) {
|
|
327
|
+
if (!out.has(form))
|
|
328
|
+
out.set(form, [new Set(), name]);
|
|
329
|
+
out.get(form)[0].add(String(k));
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
return out;
|
|
335
|
+
}
|
|
336
|
+
const ABBR = rx(String.raw `(?:\b(?:U\.S|U\.K|St|Mt|Ft|Dr|Mr|Ms|Mrs|No|vs|etc|Inc|Co|Corp|Jr|Sr|Gov|Sen|Rep|e\.g|i\.e|approx)\.|\b[A-Z]\.)$`);
|
|
337
|
+
/** The block's sentences, not cut after an abbreviation or initial (U.S., St., e.g.). */
|
|
338
|
+
export function sentences(block) {
|
|
339
|
+
const out = [];
|
|
340
|
+
for (const piece of split(/(?<=[.;!?])\s+|(?<=[.;!?]\*\*)\s+|(?<=[.;!?]\*)\s+/, block)) {
|
|
341
|
+
if (out.length && search(ABBR, out[out.length - 1]))
|
|
342
|
+
out[out.length - 1] += " " + piece;
|
|
343
|
+
else
|
|
344
|
+
out.push(piece);
|
|
345
|
+
}
|
|
346
|
+
return out;
|
|
347
|
+
}
|
|
348
|
+
const namePat = (e, extra = "") => rx(String.raw `(?i)(?<![\w-])` + escapeRe(e) + String.raw `(?![\w-])` + extra);
|
|
349
|
+
/** Prose that names an entity right before a number gives that entity's figure; returns the other entity's name. */
|
|
350
|
+
export function misnamed(sents, n, fhits, ents) {
|
|
351
|
+
if (!fhits.length || !fhits.every((f) => isObj(f.where) && Object.keys(f.where).length))
|
|
352
|
+
return null;
|
|
353
|
+
const own = new Set();
|
|
354
|
+
for (const f of fhits)
|
|
355
|
+
for (const v of Object.values(f.where))
|
|
356
|
+
for (const form of forms(v))
|
|
357
|
+
own.add(form);
|
|
358
|
+
const spots = [];
|
|
359
|
+
for (const x of sents)
|
|
360
|
+
for (const m of finditer(tokRe(n.tok), x))
|
|
361
|
+
spots.push([x, m.index]);
|
|
362
|
+
for (const [x, i] of spots) {
|
|
363
|
+
let win = x.slice(Math.max(0, i - 40), i);
|
|
364
|
+
win = sub(/\(\s*\d{3,5}\s*,/, (m) => "(" + " ".repeat(m[0].length - 1), win); // "(1863, "
|
|
365
|
+
const cut = Math.max(0, ...finditer(/\d/, win).map(end));
|
|
366
|
+
win = win.slice(cut); // since the previous number
|
|
367
|
+
const seen = new Map();
|
|
368
|
+
for (const e of ents.keys()) {
|
|
369
|
+
const m = search(namePat(e), win);
|
|
370
|
+
if (m)
|
|
371
|
+
seen.set(e, m);
|
|
372
|
+
}
|
|
373
|
+
if (cut && seen.size && !strip(win.slice(0, Math.min(...[...seen.values()].map((m) => m.index)))))
|
|
374
|
+
continue;
|
|
375
|
+
const named = [...seen.entries()].filter(([, m]) => isUpper(m[0][0]) || /^[\s(:+-]*$/.test(win.slice(end(m))))
|
|
376
|
+
.map(([e]) => [e, ents.get(e)[0]]);
|
|
377
|
+
if (!named.length || new Set([...seen.keys()].map((e) => ents.get(e)[1])).size > 1)
|
|
378
|
+
continue;
|
|
379
|
+
const gapAt = Math.max(...named.flatMap(([e]) => finditer(namePat(e), win).map(end)));
|
|
380
|
+
const gap = win.slice(gapAt);
|
|
381
|
+
if (!fullmatch(rx(String.raw `(?i)[\s(:,$–+-]*(?:(?:at|was|is|of|with|had|reached|down|up|fell|rose)\s+)?[\s($+]*`), gap))
|
|
382
|
+
continue;
|
|
383
|
+
const after = x.slice(i + n.tok.length, i + n.tok.length + 60);
|
|
384
|
+
let follows = false;
|
|
385
|
+
for (const e of own) {
|
|
386
|
+
if (!ents.has(e))
|
|
387
|
+
continue;
|
|
388
|
+
const m = search(namePat(e, String.raw `(?!\s*[(:]\s*\$?\d)`), after);
|
|
389
|
+
if (m && m.index < 25 && fullmatch(/\s*(?:[a-z%$]+\s+){0,3}/i, after.slice(0, m.index)))
|
|
390
|
+
follows = true;
|
|
391
|
+
}
|
|
392
|
+
if (follows)
|
|
393
|
+
continue; // the number's name follows it ("$1,072.6 billion on Medicare")
|
|
394
|
+
if (named.some(([e]) => own.has(e)))
|
|
395
|
+
continue;
|
|
396
|
+
if (named.some(([, keys]) => [...keys].some((k) => fhits.some((f) => k in f.where))))
|
|
397
|
+
return ents.get(named[named.length - 1][0])[1];
|
|
398
|
+
}
|
|
399
|
+
return null;
|
|
400
|
+
}
|
|
401
|
+
const OTHER = rx(String.raw `(?i)\b(?:than|compar\w*|versus|vs\.?|against|above|below|average|whereas|while|relative|` +
|
|
402
|
+
String.raw `ratio|share|of\s+(?:the|its|their)\s+\w+|percent\s+of)\b`);
|
|
403
|
+
/** A where value that names a row (Colorado, 117th, Energy), not a period, code or number. */
|
|
404
|
+
function rowName(v0) {
|
|
405
|
+
const v = cellKey(v0);
|
|
406
|
+
if (v.length >= 2 && v.length <= 60 && ((/[a-z]{3}/.test(v) && !/\d/.test(v)) || /^\d{1,3}(?:st|nd|rd|th)(?:\s+[a-z]+)?$/.test(v)))
|
|
407
|
+
return v;
|
|
408
|
+
return null;
|
|
409
|
+
}
|
|
410
|
+
function value(f) {
|
|
411
|
+
const v = toFloat(f?.value);
|
|
412
|
+
return v === null || f?.value === null || f?.value === undefined ? 0.0 : v;
|
|
413
|
+
}
|
|
414
|
+
const dictEq = (a, b) => {
|
|
415
|
+
const ka = Object.keys(a), kb = Object.keys(b);
|
|
416
|
+
return ka.length === kb.length && ka.every((k) => k in b && JSON.stringify(a[k]) === JSON.stringify(b[k]));
|
|
417
|
+
};
|
|
418
|
+
const without = (o, k) => Object.fromEntries(Object.entries(o).filter(([a]) => a !== k));
|
|
419
|
+
/** A clause about one row must cite that row's figure; returns the named row when it cites a sibling's. */
|
|
420
|
+
export function clauseOwner(sents, n, fhits, facts) {
|
|
421
|
+
if (!fhits.length || !fhits.every((f) => isObj(f.where) && Object.keys(f.where).length))
|
|
422
|
+
return null;
|
|
423
|
+
const keySets = fhits.map((f) => new Set(Object.entries(f.where).filter(([, v]) => rowName(v)).map(([k]) => k)));
|
|
424
|
+
const keys = [...keySets[0]].filter((k) => keySets.every((s) => s.has(k)));
|
|
425
|
+
const all = Object.values(facts || {}).filter((g) => isObj(g) && isObj(g.where));
|
|
426
|
+
for (const k of keys) {
|
|
427
|
+
const rows = new Set(all.map((f2) => rowName(f2.where[k])).filter((x) => x !== null));
|
|
428
|
+
const own = new Set(fhits.map((f) => rowName(f.where[k])));
|
|
429
|
+
for (const x of sents) {
|
|
430
|
+
const i = at(x, n.tok);
|
|
431
|
+
if (i < 0)
|
|
432
|
+
continue;
|
|
433
|
+
const named = [...rows].filter((r) => [...forms(r)].some((form) => search(namePat(form), x)));
|
|
434
|
+
if (named.length !== 1 || named.some((r) => own.has(r)))
|
|
435
|
+
continue;
|
|
436
|
+
const row = named[0];
|
|
437
|
+
const ms = [...forms(row)].flatMap((form) => finditer(namePat(form), x.slice(0, i)))
|
|
438
|
+
.filter((m) => isUpper(m[0][0]) || isDigit(m[0][0])); // a name, not a phrase ("all civilian workers")
|
|
439
|
+
if (!ms.length || search(OTHER, x.slice(Math.max(...ms.map(end)), i)))
|
|
440
|
+
continue;
|
|
441
|
+
const stem = (f, name) => strip(sub(namePat(name ?? "None"), "", cellKey(f.label ?? "")));
|
|
442
|
+
const sib = all.filter((g) => rowName(g.where[k]) === row && fhits.some((f) => dictEq(without(g.where, k), without(f.where, k)) && stem(g, row) === stem(f, rowName(f.where[k]))));
|
|
443
|
+
if (sib.length && !sib.some((g) => fhits.some((f) => value(g) === value(f))))
|
|
444
|
+
return row;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
return null;
|
|
448
|
+
}
|
|
449
|
+
const GENERIC_ACR = new Set("GDP USA BLS BEA OPM FRED IPUMS CPS ACS OMB CBO GAO MTS API NSA SA SAAR FY CY NAICS MOE SE CI USD DC".split(" "));
|
|
450
|
+
const ACR_BEFORE = rx(String.raw `\b([A-Z][A-Z0-9]{1,}(?:-[A-Z0-9]+)?)\s*[,:(]?\s*(?:(?:was|is|at|of|up|down|rose|fell|grew|` +
|
|
451
|
+
String.raw `increased|decreased|declined|by|=)\s+){0,2}[+\-−]?\$?$`);
|
|
452
|
+
const NOT_THIS = rx(String.raw `(?i)\b(?:instead\s+of|rather\s+than|except|excluding|without|not|than|vs\.?|versus)\s*$`);
|
|
453
|
+
/** A measure named by its acronym right before the number must be a fact of that measure; returns the acronym. */
|
|
454
|
+
export function measureNamed(sents, n, fhits, facts) {
|
|
455
|
+
const labels = Object.values(facts || {}).filter(isObj).map((f) => str(f.label || ""));
|
|
456
|
+
if (!fhits.length || !labels.length || !fhits.every((f) => f.label))
|
|
457
|
+
return null;
|
|
458
|
+
for (const x of sents) {
|
|
459
|
+
const i = at(x, n.tok);
|
|
460
|
+
if (i < 0)
|
|
461
|
+
continue;
|
|
462
|
+
const m = search(ACR_BEFORE, x.slice(Math.max(0, i - 40), i));
|
|
463
|
+
if (!m)
|
|
464
|
+
continue;
|
|
465
|
+
const before = x.slice(Math.max(0, i - 40 - 30), Math.max(0, i - 40) + m.index);
|
|
466
|
+
if (search(NOT_THIS, before))
|
|
467
|
+
continue; // "CPI-W instead of CPI-U, +$21": the acronym is not the measure
|
|
468
|
+
const a = m[1];
|
|
469
|
+
if (GENERIC_ACR.has(a) || a.length < 3)
|
|
470
|
+
continue;
|
|
471
|
+
const pat = new RegExp(String.raw `(?<![\w-])` + escapeRe(a) + String.raw `(?![\w-])`);
|
|
472
|
+
const carry = labels.filter((lab) => pat.test(lab));
|
|
473
|
+
if (!carry.length || carry.length > labels.length / 2)
|
|
474
|
+
continue;
|
|
475
|
+
if (!fhits.some((f) => pat.test(str(f.label))))
|
|
476
|
+
return a;
|
|
477
|
+
}
|
|
478
|
+
return null;
|
|
479
|
+
}
|
|
480
|
+
const tableLines = (block) => splitlines(strip(block)).filter((l) => lstrip(l).startsWith("|"));
|
|
481
|
+
/** A table's columns each hold one measure: a cell whose facts give another column's measure was moved across. */
|
|
482
|
+
export function columnMisfits(block, facts) {
|
|
483
|
+
const lines = tableLines(block);
|
|
484
|
+
if (lines.length < 5)
|
|
485
|
+
return [];
|
|
486
|
+
const named = Object.entries(facts || {}).filter(([, f]) => isObj(f) && f.label);
|
|
487
|
+
if (!named.length)
|
|
488
|
+
return [];
|
|
489
|
+
const shown = named.map(([k, f]) => [k, factDisplay(f), f]).filter(([, d]) => d);
|
|
490
|
+
const stem = (f) => {
|
|
491
|
+
let s = cellKey(f.label);
|
|
492
|
+
for (const v of Object.values(isObj(f.where) ? f.where : {}))
|
|
493
|
+
s = s.split(cellKey(v)).join(" ");
|
|
494
|
+
return s.replace(/[^a-z0-9 ]/g, " ").replace(/\s+/g, " ").trim();
|
|
495
|
+
};
|
|
496
|
+
const cols = new Map();
|
|
497
|
+
for (const l of lines.slice(2)) {
|
|
498
|
+
const cells = cellsOf(l).map((c) => strip(c));
|
|
499
|
+
cells.slice(1).forEach((c, j) => {
|
|
500
|
+
const k = j + 1;
|
|
501
|
+
for (const n of answerNumbers(c)) {
|
|
502
|
+
n.table = true;
|
|
503
|
+
const stems = new Set(shown.filter(([, d]) => matches(n, ...d)).map(([, , f]) => stem(f)));
|
|
504
|
+
if (stems.size) {
|
|
505
|
+
if (!cols.has(k))
|
|
506
|
+
cols.set(k, []);
|
|
507
|
+
cols.get(k).push([n.tok, stems]);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
});
|
|
511
|
+
}
|
|
512
|
+
const mode = new Map();
|
|
513
|
+
for (const [k, cells] of cols) {
|
|
514
|
+
if (cells.length < 3)
|
|
515
|
+
continue;
|
|
516
|
+
const counts = new Map();
|
|
517
|
+
for (const [, st] of cells)
|
|
518
|
+
for (const s of st)
|
|
519
|
+
counts.set(s, (counts.get(s) || 0) + 1);
|
|
520
|
+
let top = "", nTop = -1;
|
|
521
|
+
for (const [s, c] of counts)
|
|
522
|
+
if (c > nTop)
|
|
523
|
+
[top, nTop] = [s, c];
|
|
524
|
+
if (nTop >= Math.max(3, cells.length - 1) && [...counts.values()].filter((c) => c === nTop).length === 1)
|
|
525
|
+
mode.set(k, top);
|
|
526
|
+
}
|
|
527
|
+
const out = [];
|
|
528
|
+
for (const [k, cells] of cols) {
|
|
529
|
+
if (!mode.has(k))
|
|
530
|
+
continue;
|
|
531
|
+
const others = new Set([...mode.entries()].filter(([j, m]) => j !== k && m !== mode.get(k)).map(([, m]) => m));
|
|
532
|
+
out.push(...cells.filter(([, st]) => !st.has(mode.get(k)) && [...st].some((s) => others.has(s))).map(([tok]) => tok));
|
|
533
|
+
}
|
|
534
|
+
return out;
|
|
535
|
+
}
|
|
536
|
+
/** A table whose rows hold facts but none of whose row labels keys a fact (`where=`). */
|
|
537
|
+
function unboundTable(block, shown, facts) {
|
|
538
|
+
const rows = [];
|
|
539
|
+
for (const l of tableLines(block).slice(2)) {
|
|
540
|
+
const nums = answerNumbers(l);
|
|
541
|
+
for (const n of nums)
|
|
542
|
+
n.table = true;
|
|
543
|
+
if (!nums.some((n) => shown.some(([name, d]) => name in (facts || {}) && matches(n, ...d))))
|
|
544
|
+
continue;
|
|
545
|
+
const row = nums.length ? rowLabel(nums[0]) : null;
|
|
546
|
+
if (row !== null)
|
|
547
|
+
rows.push(row);
|
|
548
|
+
}
|
|
549
|
+
if (rows.length < 2)
|
|
550
|
+
return null;
|
|
551
|
+
const vals = Object.values(facts || {}).filter(isObj).flatMap((f) => Object.values(isObj(f.where) ? f.where : {}).map(cellKey));
|
|
552
|
+
return rows.some((r) => vals.some((v) => new RegExp("^" + escapeRe(r) + String.raw `(?:$|\s*[,;:(–-])`).test(v))) ? null : rows[0];
|
|
553
|
+
}
|
|
554
|
+
const FROM_TO = rx(String.raw `(?i)\b(rose|risen|increased|grew|grown|climbed|jumped|fell|fallen|declined|dropped|decreased|slipped|shrank|` +
|
|
555
|
+
String.raw `shrunk)\s+(?:[a-z]+\s+){0,3}?from\s+(?:about\s+|roughly\s+)?(-?\$?\d[\d,]*(?:\.\d+)?)\s*` +
|
|
556
|
+
String.raw `(%|percent|thousand|million|billion|trillion)?[^.;\n]{0,40}?\bto\s+(?:about\s+|roughly\s+)?` +
|
|
557
|
+
String.raw `(-?\$?\d[\d,]*(?:\.\d+)?)\s*(%|percent|thousand|million|billion|trillion)?`);
|
|
558
|
+
const VS_FROM = rx(String.raw `(?i)(?<![\w.,])(-?\$?\d[\d,]*(?:\.\d+)?)\s*(%|percent|thousand|million|billion|trillion)?` +
|
|
559
|
+
String.raw `(?:\s+(?:in|for)\s+(?:fy\s?)?(?:19|20)\d\d)?\s*,?\s+(up|down|higher|lower)\s+from\s+(?:about\s+)?` +
|
|
560
|
+
String.raw `(-?\$?\d[\d,]*(?:\.\d+)?)\s*(%|percent|thousand|million|billion|trillion)?`);
|
|
561
|
+
/** "fell from $890.8 billion to $984.0 billion": a direction word whose own from-to numbers move the other way. */
|
|
562
|
+
export function fromToMisreads(text) {
|
|
563
|
+
const out = [];
|
|
564
|
+
const num = (t) => Number(t.replace(/\$/g, "").replace(/,/g, ""));
|
|
565
|
+
for (const m of finditer(FROM_TO, text)) {
|
|
566
|
+
const a = m[2], ua = (m[3] || "").toLowerCase(), b = m[4], ub = (m[5] || "").toLowerCase();
|
|
567
|
+
if (/^(?:1[89]|20)\d\d$/.test(a) && /^(?:1[89]|20)\d\d$/.test(b) && !(ua || ub))
|
|
568
|
+
continue;
|
|
569
|
+
const x = num(a) * (SCALE[ua || ub] ?? 1.0), y = num(b) * (SCALE[ub || ua] ?? 1.0);
|
|
570
|
+
const up = !/^(?:fell|fallen|declin|drop|decreas|slip|shr)/i.test(m[1]);
|
|
571
|
+
if ((up && y < x) || (!up && y > x))
|
|
572
|
+
out.push(m[0].slice(0, 60));
|
|
573
|
+
}
|
|
574
|
+
for (const m of finditer(VS_FROM, text)) { // "18.3 percent in 2025, down from 16.8 percent": now vs then
|
|
575
|
+
const a = m[1].replace(/,+$/, ""), ua = (m[2] || "").toLowerCase(), word = m[3].toLowerCase();
|
|
576
|
+
const b = m[4].replace(/,+$/, ""), ub = (m[5] || "").toLowerCase();
|
|
577
|
+
if (!ua && /^(?:19|20)\d\d$/.test(a))
|
|
578
|
+
continue; // "in 2025, down from 17.6": the year is not the level
|
|
579
|
+
if ((ua === "%" || ua === "percent") !== (ub === "%" || ub === "percent") && ua && ub)
|
|
580
|
+
continue;
|
|
581
|
+
const x = num(a) * (SCALE[ua || ub] ?? 1.0), y = num(b) * (SCALE[ub || ua] ?? 1.0);
|
|
582
|
+
const up = ["up", "higher", "rising", "rose"].includes(word);
|
|
583
|
+
if ((up && x < y) || (!up && x > y))
|
|
584
|
+
out.push(m[0].slice(0, 60));
|
|
585
|
+
}
|
|
586
|
+
return out;
|
|
587
|
+
}
|
|
588
|
+
/** A table cell matches a fact of its own row, when facts are registered by that row. */
|
|
589
|
+
export function rowOk(n, fhits, facts) {
|
|
590
|
+
const row = rowLabel(n);
|
|
591
|
+
if (row === null)
|
|
592
|
+
return true;
|
|
593
|
+
const base = row.replace(/\s*\(.*\)\s*$/, "") || row; // "retirement (voluntary, early out)" -> retirement
|
|
594
|
+
const pat = new RegExp("^" + escapeRe(row) + String.raw `(?:$|\s*[,;:(–-])`);
|
|
595
|
+
const keyed = (f) => Object.values(isObj(f.where) ? f.where : {}).some((v) => pat.test(cellKey(v)) || cellKey(v) === base);
|
|
596
|
+
if (!Object.values(facts || {}).some((f) => isObj(f) && keyed(f)))
|
|
597
|
+
return true;
|
|
598
|
+
return fhits.some(keyed);
|
|
599
|
+
}
|
|
600
|
+
const MONTHS = Object.fromEntries(["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct",
|
|
601
|
+
"nov", "dec"].map((m, i) => [m, i + 1]));
|
|
602
|
+
const MONTH_RE = new RegExp(String.raw `\b(` + Object.keys(MONTHS).join("|") + String.raw `)[a-z]*\b`);
|
|
603
|
+
/** A period as year*100 + month (month 0 when only a year is given; a quarter is its first month). */
|
|
604
|
+
export function periodKey(x0) {
|
|
605
|
+
const x = strip(strip(strip(str(x0).toLowerCase()), "*"));
|
|
606
|
+
const ym = /^((?:19|20)\d\d)(0[1-9]|1[0-2])$/.exec(x); // OPM's yyyymm (202508)
|
|
607
|
+
if (ym)
|
|
608
|
+
return Number(ym[1]) * 100 + Number(ym[2]);
|
|
609
|
+
if (!/^(?:(?:fy\s?)?\d{4}(?:-\d{2}(?:-\d{2})?)?(?:\s*:?\s*q[1-4])?|q[1-4]\s+\d{4}|[a-z]{3,9}\.?\s+\d{4})$/.test(x))
|
|
610
|
+
return null;
|
|
611
|
+
const y = /(?<!\d)(17|18|19|20)\d\d(?!\d)/.exec(x);
|
|
612
|
+
if (!y)
|
|
613
|
+
return null;
|
|
614
|
+
const q = /q([1-4])/.exec(x);
|
|
615
|
+
const m = /(?<!\d)(?:17|18|19|20)\d\d-(\d\d)/.exec(x) || MONTH_RE.exec(x);
|
|
616
|
+
const month = q ? 3 * Number(q[1]) - 2 : m ? (/^\d+$/.test(m[1]) ? Number(m[1]) : MONTHS[m[1].slice(0, 3)]) : 0;
|
|
617
|
+
return Number(y[0]) * 100 + month;
|
|
618
|
+
}
|
|
619
|
+
/** Months from period fk to period ref; whole years when either gives only a year. */
|
|
620
|
+
function lag(ref, fk) {
|
|
621
|
+
if (ref % 100 && fk % 100)
|
|
622
|
+
return (Math.floor(ref / 100) * 12 + (ref % 100)) - (Math.floor(fk / 100) * 12 + (fk % 100));
|
|
623
|
+
return (Math.floor(ref / 100) - Math.floor(fk / 100)) * 12;
|
|
624
|
+
}
|
|
625
|
+
/** The periods a fact is about, from its `where`; a vintage, release or retrieval date is not one. */
|
|
626
|
+
export function factPeriods(f) {
|
|
627
|
+
const w = Object.entries(isObj(f?.where) ? f.where : {}).filter(([name]) => !/vintage|release|as_?of|retriev|published|accessed/i.test(name));
|
|
628
|
+
const per = (v) => periodKey(str(v).replace(/\s*\(.*\)\s*$/, "")); // "2026 (to Oct 15)"
|
|
629
|
+
let keys = w.map(([, v]) => per(v)).filter((k) => k !== null);
|
|
630
|
+
const monE = w.find(([name]) => /month/i.test(name));
|
|
631
|
+
const mon = monE ? strip(str(monE[1])).toLowerCase() : "";
|
|
632
|
+
const m = /^(?:0?[1-9]|1[0-2])$/.test(mon) ? Number(mon) : /^[a-z]{3,9}\.?$/.test(mon) ? MONTHS[mon.slice(0, 3)] ?? null : null;
|
|
633
|
+
const years = w.map(([name, v]) => [per(v), name]).filter(([k]) => k !== null && k % 100 === 0);
|
|
634
|
+
if (m && years.length === 1) {
|
|
635
|
+
const y = Math.floor(years[0][0] / 100);
|
|
636
|
+
const fv = w.find(([name]) => name === years[0][1])[1];
|
|
637
|
+
const fiscal = /fiscal|\bfy/i.test(years[0][1]) || str(fv).toLowerCase().startsWith("fy");
|
|
638
|
+
keys = [...keys.filter((k) => k !== years[0][0]), (fiscal && m >= 10 ? y - 1 : y) * 100 + m];
|
|
639
|
+
}
|
|
640
|
+
if (keys.some((k) => k % 100))
|
|
641
|
+
keys = keys.filter((k) => k % 100); // the month is the fact's period
|
|
642
|
+
if (!keys.length) { // a wide table's column ("share_2019", "2025"): its one year
|
|
643
|
+
const col = str(f?.column || "");
|
|
644
|
+
const ys = /fy|fiscal/i.test(col) ? [] : [...col.matchAll(/(?<!\d)((?:19|20)\d\d)(?!\d)/g)].map((x) => x[1]);
|
|
645
|
+
if (ys.length === 1)
|
|
646
|
+
keys = [Number(ys[0]) * 100];
|
|
647
|
+
}
|
|
648
|
+
return keys;
|
|
649
|
+
}
|
|
650
|
+
const NO_BASE = String.raw `(?!\s*(?:dollars|prices|\$|=\s*100))`; // "2024 dollars" is a price base, not a period
|
|
651
|
+
const HEAD_PERIOD = rx(String.raw `(?i)\s*\**\s*((?:fy\s?)?\d{4}(?:-\d{2})?|[a-z]{3,9}\.?\s+\d{4})\b` + NO_BASE);
|
|
652
|
+
const HEAD_CMP = rx(String.raw `(?i)\b(?:since|from|vs\.?|versus|change|minus|relative|compared|than|over)\b`);
|
|
653
|
+
const HEAD_YEAR = rx(String.raw `(?i)(?<![\w-])(?<!constant\s)(?<!chained\s)((?:fy\s?)?(?:19|20)\d\d(?:-\d{2})?)(?![\w-]|\s*[–-]\s*\d)` + NO_BASE);
|
|
654
|
+
/** In a table keyed by periods, a cell whose matching facts all break its column's lag is a value moved. */
|
|
655
|
+
export function periodMisfits(block, shown, facts) {
|
|
656
|
+
const lines = tableLines(block);
|
|
657
|
+
if (lines.length < 4)
|
|
658
|
+
return [];
|
|
659
|
+
const cols = new Map();
|
|
660
|
+
const head = cellsOf(lines[0]).map((c) => strip(c));
|
|
661
|
+
for (const l of lines.slice(2)) {
|
|
662
|
+
const cells = cellsOf(l).map((c) => strip(c));
|
|
663
|
+
const rk = cells.length ? periodKey(cells[0].replace(/\s*\(.*\)\s*$/, "")) : null;
|
|
664
|
+
cells.slice(1).forEach((c, j) => {
|
|
665
|
+
const k = j + 1;
|
|
666
|
+
let hm = null;
|
|
667
|
+
if (k < head.length) {
|
|
668
|
+
const m0 = match(HEAD_PERIOD, head[k]);
|
|
669
|
+
if (m0)
|
|
670
|
+
hm = m0[1];
|
|
671
|
+
else if (!search(HEAD_CMP, head[k])) {
|
|
672
|
+
// one period anywhere in the header: "Share, 2026 (%)", but not "Change since September 2024"
|
|
673
|
+
const found = finditer(HEAD_YEAR, head[k]).map((x) => x[1]);
|
|
674
|
+
if (found.length === 1 && head[k].includes(found[0]))
|
|
675
|
+
hm = found[0];
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
const hk = hm !== null ? periodKey(hm) : null;
|
|
679
|
+
for (const [key, ref] of [["r", rk], ["h", hk]]) {
|
|
680
|
+
if (ref === null)
|
|
681
|
+
continue;
|
|
682
|
+
for (const n of answerNumbers(c)) {
|
|
683
|
+
n.table = true;
|
|
684
|
+
const lags = new Set();
|
|
685
|
+
let matched = false;
|
|
686
|
+
for (const [name, d, f] of shown) {
|
|
687
|
+
if (name in (facts || {}) && matches(n, ...d)) {
|
|
688
|
+
matched = true;
|
|
689
|
+
const fy = match(/\s*\**\s*(?:fy|fiscal)/i, key === "r" ? cells[0] : head[k]);
|
|
690
|
+
for (let fk of factPeriods(f)) {
|
|
691
|
+
if (fy && fk % 100 >= 10)
|
|
692
|
+
fk = (Math.floor(fk / 100) + 1) * 100; // Oct-Dec belong to the next FY
|
|
693
|
+
lags.add(lag(ref, fk));
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
if (lags.size || matched) { // judged by its dated facts; a cell matching only undated ones fits no row
|
|
698
|
+
const ck = `${key}|${k}`;
|
|
699
|
+
if (!cols.has(ck))
|
|
700
|
+
cols.set(ck, []);
|
|
701
|
+
cols.get(ck).push([n.tok, lags]);
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
});
|
|
706
|
+
}
|
|
707
|
+
const out = [];
|
|
708
|
+
for (const cells of cols.values()) {
|
|
709
|
+
if (cells.length === 2 && cells.some(([, lags]) => lags.has(0))) {
|
|
710
|
+
out.push(...cells.filter(([, lags]) => !lags.has(0)).map(([tok]) => tok)); // "FY2025 | FY2026": each row its own
|
|
711
|
+
continue;
|
|
712
|
+
}
|
|
713
|
+
if (cells.length === 2 && cells.every(([, lags]) => lags.size) && ![...cells[0][1]].some((g) => cells[1][1].has(g))) {
|
|
714
|
+
out.push(...cells.map(([tok]) => tok)); // two rows with no lag in common: their values were swapped
|
|
715
|
+
continue;
|
|
716
|
+
}
|
|
717
|
+
if (cells.length < 3)
|
|
718
|
+
continue;
|
|
719
|
+
if (cells.filter(([, lags]) => lags.size).length < 3)
|
|
720
|
+
continue; // too few dated cells to read the column's lag
|
|
721
|
+
const counts = new Map();
|
|
722
|
+
for (const [, lags] of cells)
|
|
723
|
+
for (const g of lags)
|
|
724
|
+
counts.set(g, (counts.get(g) || 0) + 1);
|
|
725
|
+
let mode = 0, top = -1;
|
|
726
|
+
for (const [g, c] of counts)
|
|
727
|
+
if (c > top)
|
|
728
|
+
[mode, top] = [g, c];
|
|
729
|
+
if (top < cells.length - 1)
|
|
730
|
+
continue; // no lag shared by the column: nothing to compare
|
|
731
|
+
out.push(...cells.filter(([, lags]) => !lags.has(mode)).map(([tok]) => tok));
|
|
732
|
+
}
|
|
733
|
+
return out;
|
|
734
|
+
}
|
|
735
|
+
const YEAR_AFTER = rx(String.raw `(?i)^\s*(?:percent(?:age points?)?|points?|pp)?(?:\s+of\s+(?:the\s+)?(?:[a-z.]{2,15}\s+){0,3}[a-z]{2,15})?\s*` +
|
|
736
|
+
String.raw `(?:in|for|during)\s+(?:the\s+)?` + // "percent of GDP in 2019"
|
|
737
|
+
String.raw `(?:(?P<mon>jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?\s+)?` + // in September 2024
|
|
738
|
+
String.raw `(?:fy\s?)?((?:17|18|19|20)\d\d)` +
|
|
739
|
+
String.raw `\b(?!\s*(?:-|–|to|through)\s*(?:\d|[a-z]{3,9}\.?\s+\d)|\s*(?:dollars|prices|terms|chained|constant|real))|^\s*(?:percent(?:age points?)?|points?)?\s*\((?:fy\s?)?((?:17|18|19|20)\d\d)\)`);
|
|
740
|
+
const YB1 = rx(String.raw `(?i)\b(?:in|for|during)\s+((?:fy\s?)?(?:17|18|19|20)\d\d)\b([^\d.;,:]{0,25})$`);
|
|
741
|
+
const YB2 = rx(String.raw `(?i)(?<![\w-])((?:fy\s?)?(?:17|18|19|20)\d\d)()\s*\(\s*(?:about\s+)?[$+\-−]?$`);
|
|
742
|
+
const YB_CMP = rx(String.raw `(?i)\b(?:to|from|than|vs|versus|compared|since|over|until|through|and)\b`);
|
|
743
|
+
/** "for 2025 (men 20.2": a year that dates the number from just before it, as " in 2025" for wrongYear. */
|
|
744
|
+
export function yearBefore(before, after = "") {
|
|
745
|
+
const m = search(YB1, before) || search(YB2, before);
|
|
746
|
+
const own = split(/[,;]/, after.slice(0, 50), 1)[0]; // the number's own clause
|
|
747
|
+
if (!m || search(YB_CMP, m[2]) || /(?<!\d)(?:17|18|19|20)\d\d(?!\d)/.test(own))
|
|
748
|
+
return "";
|
|
749
|
+
return ` in ${m[1]}`;
|
|
750
|
+
}
|
|
751
|
+
const LABEL_YEAR = rx(String.raw `(?i)(?<!since )(?<!from )(?<!to )(?<!through )(?<!until )(?<!before )(?<!after )` +
|
|
752
|
+
String.raw `(?<![\d-])(?:fy\s?)?((?:19|20)\d\d)(?![\d-])`);
|
|
753
|
+
/** "6.3 percent in 2019": when every fact the number matches is keyed to periods and none falls in that year. */
|
|
754
|
+
export function wrongYear(after, fhits) {
|
|
755
|
+
const m = match(YEAR_AFTER, after);
|
|
756
|
+
if (!m)
|
|
757
|
+
return false;
|
|
758
|
+
const year = Number(m[2] || m[3]);
|
|
759
|
+
const mon = MONTHS[(m.groups?.mon || "").toLowerCase().slice(0, 3)];
|
|
760
|
+
const keys = [];
|
|
761
|
+
for (const f of fhits) {
|
|
762
|
+
let ks = factPeriods(f);
|
|
763
|
+
if (!ks.length) { // a sensitivity fact keyed by its alternative: the one year its label names
|
|
764
|
+
const ys = new Set(finditer(LABEL_YEAR, str(f.label || "")).map((x) => x[1]));
|
|
765
|
+
ks = ys.size === 1 ? [Number([...ys][0]) * 100] : [];
|
|
766
|
+
}
|
|
767
|
+
if (!ks.length)
|
|
768
|
+
return false; // a fact without a period could be any year
|
|
769
|
+
keys.push(...ks);
|
|
770
|
+
}
|
|
771
|
+
if (mon && keys.every((k) => k % 100))
|
|
772
|
+
return !keys.includes(year * 100 + mon);
|
|
773
|
+
return keys.length > 0 && !keys.map((k) => Math.floor(k / 100)).includes(year);
|
|
774
|
+
}
|
|
775
|
+
/** The facts of the year the sentence gives for the number, when some match it; otherwise all of fhits. */
|
|
776
|
+
export function dated(sents, n, fhits) {
|
|
777
|
+
for (const x of sents) {
|
|
778
|
+
const i = at(x, n.tok);
|
|
779
|
+
if (i < 0)
|
|
780
|
+
continue;
|
|
781
|
+
const after = x.slice(i + n.tok.length);
|
|
782
|
+
const m = match(YEAR_AFTER, after.slice(0, 40)) || match(YEAR_AFTER, yearBefore(x.slice(0, i), after));
|
|
783
|
+
if (m) {
|
|
784
|
+
const year = Number(m[2] || m[3]);
|
|
785
|
+
const own = fhits.filter((f) => factPeriods(f).map((k) => Math.floor(k / 100)).includes(year));
|
|
786
|
+
if (own.length)
|
|
787
|
+
return own;
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
return fhits;
|
|
791
|
+
}
|
|
792
|
+
function pubYearFits(period, spots, tok) {
|
|
793
|
+
const have = new Set([...period.matchAll(/(?<!\d)((?:17|18|19|20)\d\d)(?!\d)/g)].map((x) => x[1]));
|
|
794
|
+
if (!have.size)
|
|
795
|
+
return true;
|
|
796
|
+
for (const [x, i] of spots) {
|
|
797
|
+
const after = x.slice(i + tok.length);
|
|
798
|
+
const m = match(YEAR_AFTER, after.slice(0, 40)) || match(YEAR_AFTER, yearBefore(x.slice(0, i), after));
|
|
799
|
+
if (m && !have.has(m[2] || m[3]))
|
|
800
|
+
return false;
|
|
801
|
+
}
|
|
802
|
+
return true;
|
|
803
|
+
}
|
|
804
|
+
// a citing word: the number is someone else's ("BLS publishes 770", "matches the published 23.3")
|
|
805
|
+
const CITING = rx(String.raw `(?i)benchmark|publish|\breports?\b|\breported\b|\brelease[sd]?\b|according to|official|` +
|
|
806
|
+
String.raw `\bmatch(?:es|ed)?\b|\bagrees?\b|compared? with|vintage|first (?:estimate|release)|superseded|` +
|
|
807
|
+
String.raw `as (?:first )?(?:reported|published)`);
|
|
808
|
+
const COMMON = new Set(("the and for from table tables data release report page series news annual monthly latest estimates " +
|
|
809
|
+
"estimate summary file files statistics united states national official published labor force current population survey " +
|
|
810
|
+
"department bureau office averages average income employment economic analysis interactive database dashboard workforce " +
|
|
811
|
+
"federal public monthly weekly quarterly detailed historical with https http www gov htm html pdf xlsx csv").split(" "));
|
|
812
|
+
/** The publisher's names in benchmark.csv's source column: acronyms, codes and proper names. */
|
|
813
|
+
function sourceWords(sources) {
|
|
814
|
+
const out = new Set();
|
|
815
|
+
for (const src0 of sources) {
|
|
816
|
+
const src = str(src0).replace(/https?:\/\/\S+/g, " ");
|
|
817
|
+
for (const w of src.match(/[A-Za-z][A-Za-z0-9&'-]+/g) || []) {
|
|
818
|
+
const low = w.toLowerCase();
|
|
819
|
+
if (COMMON.has(low) || w.length < 2)
|
|
820
|
+
continue;
|
|
821
|
+
if (/^[A-Z]{2,}[A-Za-z0-9]*$/.test(w) || /\d/.test(w) || (isUpper(w[0]) && w.length >= 4))
|
|
822
|
+
out.add(low);
|
|
823
|
+
}
|
|
824
|
+
// the initials of a capitalized name: "Office of the Federal Register" -> ofr
|
|
825
|
+
for (const run of src.match(/[A-Z][a-z]+(?:\s+(?:(?:of|the|and|for|on)\s+)*[A-Z][a-z]+)+/g) || []) {
|
|
826
|
+
const ini = (run.match(/[A-Z][a-z]+/g) || []).map((x) => x[0]).join("");
|
|
827
|
+
if (ini.length >= 2)
|
|
828
|
+
out.add(ini.toLowerCase());
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
return out;
|
|
832
|
+
}
|
|
833
|
+
function cites(text, ws) {
|
|
834
|
+
const low = text.toLowerCase();
|
|
835
|
+
return !!search(CITING, text) || [...ws].some((w) => new RegExp(String.raw `(?<![\w-])` + escapeRe(w) + String.raw `(?![\w-])`).test(low));
|
|
836
|
+
}
|
|
837
|
+
/** Is a number that is only a published benchmark figure credited to its publisher? */
|
|
838
|
+
function attributed(block, n, isTable, sources = []) {
|
|
839
|
+
const ws = sourceWords(sources);
|
|
840
|
+
if (!isTable) {
|
|
841
|
+
const lead = match(/\s*(?:[-*]\s+)?\*\*(.+?)\*\*/, block); // a bold lead-in labels its paragraph
|
|
842
|
+
if (lead && search(CITING, lead[1]))
|
|
843
|
+
return true;
|
|
844
|
+
const sent = split(/(?<=[.!?])\s+(?=[A-Z*(\[])/, block).filter((x) => has(x, n.tok));
|
|
845
|
+
return (sent.length ? sent : [block]).some((x) => cites(x, ws));
|
|
846
|
+
}
|
|
847
|
+
const lines = tableLines(block);
|
|
848
|
+
if (!lines.length)
|
|
849
|
+
return true;
|
|
850
|
+
const head = cellsOf(lines[0]).map((c) => strip(c));
|
|
851
|
+
let seen = false; // every cell holding the number sits under a credited header
|
|
852
|
+
const body = lines.length > 2 ? lines.slice(2) : lines;
|
|
853
|
+
const own = body.filter((l) => strip(n.line || "") === strip(l));
|
|
854
|
+
for (const l of own.length ? own : body) {
|
|
855
|
+
const cells = cellsOf(l).map((c) => strip(c));
|
|
856
|
+
for (let k = 0; k < cells.length; k++) {
|
|
857
|
+
if (has(cells[k], n.tok)) {
|
|
858
|
+
seen = true;
|
|
859
|
+
if (!(k < head.length && cites(head[k], ws)))
|
|
860
|
+
return false;
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
return seen;
|
|
865
|
+
}
|
|
866
|
+
const BENCH_WORDS = rx(String.raw `(?i)benchmark|published|\bmatch(?:es|ed)?\b|\bagrees?\b|compared? with|same as the`);
|
|
867
|
+
// ---------------------------------------------------------------------- dollar basis
|
|
868
|
+
const CURRENCY = rx(String.raw `(?i)(?:US)?\$\s?\d|\bUSD\s?\d|\d(?:[\d,.]*\d)?\s*(?:thousand|million|billion|trillion)?\s+dollars\b`);
|
|
869
|
+
const YEAR = rx(String.raw `(?<![\d$.,])(?:FY\s?)?((?:19|20)\d{2})(?!\d|,\d{3}|\.\d)`);
|
|
870
|
+
const CHANGE = rx(String.raw `(?i)\b(?:up|down)\s+(?:\S+\s+){0,3}?from\b|\b(?:rose|risen|rising|rises?|grew|grown|grows?|growth|` +
|
|
871
|
+
String.raw `increas\w*|decreas\w*|fell|fallen|falls?|declin\w*|jump\w*|doubl\w*|tripl\w*|halv\w*|climb\w*|` +
|
|
872
|
+
String.raw `dropp?\w*|compared\s+(?:with|to)|versus|vs\.?|(?:higher|lower|more|less)\s+than)\b` +
|
|
873
|
+
String.raw `|\bfrom\s+\$[\d.,]+[^.;]{0,40}?\bto\s+\$`);
|
|
874
|
+
const BASIS = rx(String.raw `(?i)\bnominal\b|\bcurrent[- ](?:dollars?|prices|\$)|\b(?:not|un|without|before)[- ]?` +
|
|
875
|
+
String.raw `(?:being\s+)?(?:adjust\w*|correct\w*)\s+for\s+inflation|\b(?:inflation|price|cpi|pce)[- ]adjusted\b|` +
|
|
876
|
+
String.raw `\badjust\w*\s+(?:\w+\s+)?for\s+(?:inflation|prices)|\breal\s+(?:terms|dollars?|values?|spending|` +
|
|
877
|
+
String.raw `income|incomes|wages?|earnings|growth|prices?|gdp|basis|change|increase|decline|rise|fall)\b|` +
|
|
878
|
+
String.raw `\bin\s+real\b|\bconstant[- ](?:(?:19|20)\d{2}\s+)?(?:dollars?|prices|\$)|\bchained\b|` +
|
|
879
|
+
String.raw `\b(?:19|20)\d{2}\s+dollars\b|\(\s*(?:19|20)\d{2}\s*\$\s*\)|\btoday'?s\s+dollars\b|\bdeflat\w*`);
|
|
880
|
+
const years = (s) => new Set(finditer(YEAR, s).map((m) => m[1]));
|
|
881
|
+
const isTableBlock = (b) => splitlines(strip(b)).every((l) => lstrip(l).startsWith("|"));
|
|
882
|
+
/** Sentences (and tables) that put dollar amounts of two or more different years side by side. */
|
|
883
|
+
export function dollarComparisons(text) {
|
|
884
|
+
const found = [];
|
|
885
|
+
for (const [block] of blocks(text)) {
|
|
886
|
+
const lines = splitlines(strip(block));
|
|
887
|
+
if (lines.every((l) => lstrip(l).startsWith("|"))) {
|
|
888
|
+
const rows = lines.filter((l) => !/^\|?[\s:|-]+\|?$/.test(strip(l))).map((l) => cellsOf(l).map((c) => strip(c)));
|
|
889
|
+
if (rows.length < 2)
|
|
890
|
+
continue;
|
|
891
|
+
const inHead = new Set(rows[0].flatMap((c) => [...years(c)]));
|
|
892
|
+
const ys = inHead.size > 1 ? inHead : new Set(rows.slice(1).filter((r) => r.length).flatMap((r) => [...years(r[0])]));
|
|
893
|
+
const money = /\$|dollar|\bUSD\b/i.test(rows[0].join(" ")) ||
|
|
894
|
+
rows.slice(1).flatMap((r) => r).filter((c) => search(CURRENCY, c)).length > 1;
|
|
895
|
+
if (ys.size > 1 && money)
|
|
896
|
+
found.push(strip(lines[0]).slice(0, 80));
|
|
897
|
+
continue;
|
|
898
|
+
}
|
|
899
|
+
const sents = sentences(block);
|
|
900
|
+
sents.forEach((s, i) => {
|
|
901
|
+
const amounts = finditer(CURRENCY, s).length;
|
|
902
|
+
if (amounts && years(s).size > 1 && (amounts > 1 || search(CHANGE, s))) {
|
|
903
|
+
found.push(s.split(/\s+/).filter((x) => x).join(" ").slice(0, 80));
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
const two = sents.slice(i, i + 2).join(" "); // "$X in 2019. By 2024 it reached $Y, an increase ..."
|
|
907
|
+
if (i + 1 < sents.length && amounts && finditer(CURRENCY, two).length > 1 && years(two).size > 1 && search(CHANGE, sents[i + 1])) {
|
|
908
|
+
found.push(two.split(/\s+/).filter((x) => x).join(" ").slice(0, 80));
|
|
909
|
+
}
|
|
910
|
+
});
|
|
911
|
+
}
|
|
912
|
+
return found;
|
|
913
|
+
}
|
|
914
|
+
/** FAIL when answer.md compares dollar amounts across years and nowhere says nominal or real. */
|
|
915
|
+
export function checkDollarBasis(text, D) {
|
|
916
|
+
const entry = D.dollar_basis_ok;
|
|
917
|
+
const reason = isObj(entry) ? entry.reason : entry;
|
|
918
|
+
const found = dollarComparisons(text);
|
|
919
|
+
if (!found.length)
|
|
920
|
+
return ["answer", true, "no dollar amounts compared across years"];
|
|
921
|
+
if (search(BASIS, text))
|
|
922
|
+
return ["answer", true, "dollar amounts compared across years carry their basis (nominal or real)"];
|
|
923
|
+
if (typeof reason === "string" && strip(reason)) {
|
|
924
|
+
return ["answer", true, `dollar amounts compared across years without a basis: ${strip(reason).slice(0, 80)}`];
|
|
925
|
+
}
|
|
926
|
+
return ["answer", false, `answer.md compares dollar amounts of different years (${found.slice(0, 2).map(repr).join("; ")}) ` +
|
|
927
|
+
"and never says their basis: say whether they are nominal (current dollars, not adjusted for inflation) or real " +
|
|
928
|
+
"(which index, in which year's dollars) next to the first comparison, and record the choice in decisions.json. When " +
|
|
929
|
+
'the basis truly does not matter, give decisions.json "dollar_basis_ok": {"reason": "..."} (record_decisions)'];
|
|
930
|
+
}
|
|
931
|
+
/** [text to scan, text a label may sit in]: each paragraph; a table with its caption and notes. */
|
|
932
|
+
export function blocks(text) {
|
|
933
|
+
const paras = split(/\n\s*\n/, text).filter((p) => strip(p));
|
|
934
|
+
return paras.map((p, i) => {
|
|
935
|
+
const t = isTableBlock(p);
|
|
936
|
+
const ctx = [...(t && i ? [paras[i - 1]] : []), p, ...(t && i + 1 < paras.length ? [paras[i + 1]] : [])].join("\n");
|
|
937
|
+
return [p, ctx];
|
|
938
|
+
});
|
|
939
|
+
}
|
|
940
|
+
/** Counts spelled out as a whole table cell ("| four |"): the facts tool writes 0-9 in words. */
|
|
941
|
+
export function tableWords(block) {
|
|
942
|
+
const out = [];
|
|
943
|
+
for (const line of splitlines(block)) {
|
|
944
|
+
if (!lstrip(line).startsWith("|"))
|
|
945
|
+
continue;
|
|
946
|
+
for (const cell of cellsOf(line)) {
|
|
947
|
+
const w = strip(cell).toLowerCase();
|
|
948
|
+
if (w in NUMBER_WORDS)
|
|
949
|
+
out.push({ tok: w, num: NUMBER_WORDS[w], unit: "", scale: 1.0, tol: 0.5, neg: false, dollar: false });
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
return out;
|
|
953
|
+
}
|
|
954
|
+
const WORD_ALT = Object.keys(NUMBER_WORDS).sort((a, b) => b.length - a.length).join("|");
|
|
955
|
+
const SUBSET_WORDS = rx(String.raw `(?i)\b(` + WORD_ALT + String.raw `)\s+of\s+(?:the\s+|these\s+|those\s+)?(` + WORD_ALT + String.raw `|\d{1,3})\b`);
|
|
956
|
+
/** Data rows of each markdown table in the answer (header and |---| rule not counted). */
|
|
957
|
+
export function tableSizes(text) {
|
|
958
|
+
const sizes = new Set();
|
|
959
|
+
for (const [block] of blocks(text)) {
|
|
960
|
+
const rows = tableLines(block);
|
|
961
|
+
if (rows.length && rows.length === splitlines(strip(block)).length) {
|
|
962
|
+
sizes.add(rows.slice(1).filter((l) => !/^\s*\|[\s:|-]+\|?\s*$/.test(l)).length);
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
return sizes;
|
|
966
|
+
}
|
|
967
|
+
/** "four of the five" in prose over a five-row table summarizes that table: a count to register. */
|
|
968
|
+
export function subsetWords(block, sizes) {
|
|
969
|
+
const out = [];
|
|
970
|
+
const t = AR.exempt(block);
|
|
971
|
+
for (const m of finditer(SUBSET_WORDS, t)) {
|
|
972
|
+
const line = sourceLine(block, t, m.index);
|
|
973
|
+
if (lstrip(line).startsWith("|"))
|
|
974
|
+
continue;
|
|
975
|
+
const w = m[1].toLowerCase(), tw = m[2].toLowerCase();
|
|
976
|
+
const total = tw in NUMBER_WORDS ? NUMBER_WORDS[tw] : Number(tw);
|
|
977
|
+
if (!sizes.has(total) || !(NUMBER_WORDS[w] > 0 && NUMBER_WORDS[w] < total))
|
|
978
|
+
continue;
|
|
979
|
+
out.push({ tok: m[1], num: NUMBER_WORDS[w], unit: "", scale: 1.0, tol: 0.5, dec: 0, neg: false, dollar: false, plus: false,
|
|
980
|
+
line, approx: false, unit_after: false, counts: true, table: false });
|
|
981
|
+
}
|
|
982
|
+
return out;
|
|
983
|
+
}
|
|
984
|
+
const PCT_DESC = /%|percent|\bpct\b|\bpp\b|\brates?\b|\bshares?\b|\byields?\b|\bratios?\b/i;
|
|
985
|
+
const AMT_DESC = /\$|dollar|\bthousands?\b|\bmillions?\b|\bbillions?\b|\btrillions?\b|\bcount|number/i;
|
|
986
|
+
const FELL_BEFORE = rx(String.raw `(?i)\b(?:down|fell|fallen|declined|decreased|dropped|(?:an?\s+)?(?:decline|` +
|
|
987
|
+
String.raw `drop|fall|decrease|reduction|loss)\s+of)\s+(?:by\s+)?(?:about\s+)?$`);
|
|
988
|
+
const LOWER_AFTER = rx(String.raw `(?i)\s*(?:percentage\s+points?|points?|pp|basis\s+points?)?\s*(?:below|lower|less|down|fewer|smaller|under)\b`);
|
|
989
|
+
const SIZE_LABEL = rx(String.raw `(?i)declin|decreas|drop|fall|fell|loss|fewer|reduc|lower|cut|down|` +
|
|
990
|
+
String.raw `\bgap\b|differen|shortfall|spread|margin|premium|\bminus\b|\bvs\.?\b|\bbelow\b|\bbehind\b|\bshort\b`);
|
|
991
|
+
const CMP_AFTER = rx(String.raw `(?i)\s*(?:percent(?:age)?\s+points?|points?|pp|percent|basis\s+points?)?\s*(?:less|lower|below|fewer|smaller|slower|under)\b`);
|
|
992
|
+
const UP_AFTER = rx(String.raw `(?i)\s*(?:percent(?:age)?\s+points?|points?|pp|percent|basis\s+points?)?\s*(?:above|higher|more|greater|up|faster)\b`);
|
|
993
|
+
const SIZE_BEFORE = rx(String.raw `(?i)(?:\bwithin|\bdiffer\w*\s+by|\bdifference\s+of|\bgap\s+of|±)\s*(?:about\s+)?$`);
|
|
994
|
+
const FELL_MINUS = rx(String.raw `(?i)\b(?:fell|falls?|fallen|falling|declined|declines?|decreased|decreases?|dropped|` +
|
|
995
|
+
String.raw `drops?|down|lost|shr[ai]nk|shrunk)\s+(?:by\s+)?(?:about\s+|nearly\s+|almost\s+)?$`);
|
|
996
|
+
/** answer.md, the reply as it will be sent: every number in it is a registered fact as its format shows it (or a
|
|
997
|
+
* published benchmark figure, or in audit_allow); it carries every disclosure line; flagged facts are labelled; it
|
|
998
|
+
* reports the benchmark check. Returns [results, answer text]. */
|
|
999
|
+
export function checkAnswer(base, facts, D) {
|
|
1000
|
+
const p = path.join(base, "answer.md");
|
|
1001
|
+
if (!fs.existsSync(p)) {
|
|
1002
|
+
return [[["answer", false, "answer.md missing: pass the reply exactly as you will send it to audit_answer (answer)"]], ""];
|
|
1003
|
+
}
|
|
1004
|
+
const text = AR.readTextFile(p);
|
|
1005
|
+
facts = facts || {};
|
|
1006
|
+
const out = [];
|
|
1007
|
+
let shown = Object.entries(facts).map(([name, f]) => [name, factDisplay(f || {}), f || {}])
|
|
1008
|
+
.filter(([, d]) => d);
|
|
1009
|
+
// a fact printed without its sign (fmt ...:abs) may stand in a table cell as printed or with its sign
|
|
1010
|
+
shown = shown.concat(shown.filter(([, d, f]) => d[0] < 0 && String(f.fmt || "").split(":").includes("abs"))
|
|
1011
|
+
.map(([name, d, f]) => [name, [Math.abs(d[0]), d[1], d[2]], f]));
|
|
1012
|
+
const bp = path.join(base, "exhibits", "benchmark.csv"); // a published figure the answer compares with
|
|
1013
|
+
const benchRows = fs.existsSync(bp) ? MA.readCsv(bp)[1] : [];
|
|
1014
|
+
for (const r of benchRows) {
|
|
1015
|
+
const b = MA.benchNumber(r.published);
|
|
1016
|
+
if (!b)
|
|
1017
|
+
continue;
|
|
1018
|
+
// a published figure keeps its kind when the row says it (percent or an amount), never rescaled
|
|
1019
|
+
const desc = `${r.published ?? ""} ${r.measure ?? ""} ${r.units ?? ""}`;
|
|
1020
|
+
let pct = PCT_DESC.test(desc);
|
|
1021
|
+
const amt = AMT_DESC.test(`${desc} ${r.source ?? ""}`);
|
|
1022
|
+
if (!pct && !amt && /\bgrowth\b|\bchanges?\b/i.test(desc))
|
|
1023
|
+
pct = true; // a change with no unit: read it as a percent
|
|
1024
|
+
let kind = pct && !amt ? "pubpct" : amt && !pct ? "pubnum" : "pub";
|
|
1025
|
+
if (kind === "pubpct" && /\bpoints?\b|\bpp\b/i.test(desc))
|
|
1026
|
+
kind = "pubpp";
|
|
1027
|
+
const dec = b[1] < 0.5 ? pyRound(-Math.log10(2 * b[1])) : 0;
|
|
1028
|
+
const meta = { _source: `${r.source ?? ""} ${r.url ?? ""}`, _period: str(r.period || ""),
|
|
1029
|
+
_measure: ["measure", "geography", "geo", "area", "state", "name", "label", "item", "series", "group"].map((c) => str(r[c] || "")).join(" ") };
|
|
1030
|
+
shown.push([`published ${r.measure ?? ""}`, [b[0], kind, dec], meta]);
|
|
1031
|
+
// a figure published in thousands (or millions) may be written out: -179 thousand as "179,000"
|
|
1032
|
+
const per = /\b(thousands|millions)\b/i.exec(`${r.measure ?? ""} ${r.units ?? ""}`);
|
|
1033
|
+
if (per && kind !== "pubpct") {
|
|
1034
|
+
const k = per[1].toLowerCase() === "thousands" ? 3 : 6;
|
|
1035
|
+
shown.push([`published ${r.measure ?? ""}`, [b[0] * 10 ** k, "pubnum", Math.max(0, dec - k)], meta]);
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
const isFactName = (name) => Object.prototype.hasOwnProperty.call(facts, name);
|
|
1039
|
+
const factToks = new Set(shown.flatMap(([, , f]) => answerNumbers(str(f.text ?? "")).map((n) => n.tok.toLowerCase())));
|
|
1040
|
+
const allowed = new Set((D.audit_allow || []).map((a) => allowKey(isObj(a) ? a.value : a)));
|
|
1041
|
+
const bad = [], unsigned = [], unattributed = [], misrow = [], wrongyear = [];
|
|
1042
|
+
const unlabelled = new Map();
|
|
1043
|
+
const misnamedL = [], unbound = [], cited = new Set();
|
|
1044
|
+
const ents = entities(facts);
|
|
1045
|
+
const scan = text;
|
|
1046
|
+
const lead = blocks(text).slice(0, 2).map(([b]) => b).join("\n"); // a label in the opening covers all
|
|
1047
|
+
const sizes = tableSizes(text);
|
|
1048
|
+
for (const [block, ctx] of blocks(scan)) {
|
|
1049
|
+
const isTable = isTableBlock(block);
|
|
1050
|
+
if (isTable) {
|
|
1051
|
+
misrow.push(...periodMisfits(block, shown, facts));
|
|
1052
|
+
const first = unboundTable(block, shown, facts);
|
|
1053
|
+
if (first)
|
|
1054
|
+
unbound.push(first);
|
|
1055
|
+
}
|
|
1056
|
+
for (const n of [...answerNumbers(block), ...tableWords(block), ...subsetWords(block, sizes)]) {
|
|
1057
|
+
n.table = isTable || !!n.table; // a table under a caption line too
|
|
1058
|
+
if (allowed.has(allowKey(n.tok)))
|
|
1059
|
+
continue;
|
|
1060
|
+
// a count with its noun ("3 months", "12 states") is no percent: it cannot be a pct, pp or bp fact
|
|
1061
|
+
const counted = !n.unit && !!n.counts;
|
|
1062
|
+
const hits = shown.filter(([name, d, f]) => matches(n, ...d) && !(n.plus && below0(f))
|
|
1063
|
+
&& !(counted && isFactName(name) && ["pct", "pp", "bp"].includes(d[1]))).map(([name, , f]) => [name, f]);
|
|
1064
|
+
const tokLow = n.tok.toLowerCase();
|
|
1065
|
+
if (!hits.length && !factToks.has(tokLow))
|
|
1066
|
+
bad.push(n.tok);
|
|
1067
|
+
const fhits = hits.filter(([name]) => isFactName(name)).map(([, f]) => f);
|
|
1068
|
+
if (fhits.length === 1)
|
|
1069
|
+
for (const [name] of hits)
|
|
1070
|
+
if (isFactName(name))
|
|
1071
|
+
cited.add(name);
|
|
1072
|
+
if (hits.length && !fhits.length && !factToks.has(tokLow) && !attributed(block, n, isTable, hits.map(([, f]) => f._source ?? ""))) {
|
|
1073
|
+
unattributed.push(n.tok); // a published figure passed off as the analysis's own
|
|
1074
|
+
}
|
|
1075
|
+
const sent0 = sentences(block).filter((x) => has(x, n.tok));
|
|
1076
|
+
const spots = [];
|
|
1077
|
+
for (const x of sent0)
|
|
1078
|
+
for (const m of finditer(tokRe(n.tok), x))
|
|
1079
|
+
spots.push([x, m.index]);
|
|
1080
|
+
if (fhits.length && !isTable && !n.table && spots.some(([x, i]) => wrongYear(x.slice(i + n.tok.length).slice(0, 40), fhits)
|
|
1081
|
+
|| wrongYear(yearBefore(x.slice(0, i), x.slice(i + n.tok.length)), fhits))) {
|
|
1082
|
+
const pub = hits.filter(([name, f]) => !isFactName(name) && pubYearFits(f._period ?? "", spots, n.tok)).map(([, f]) => f._source ?? "");
|
|
1083
|
+
if (!(pub.length && attributed(block, n, false, pub)))
|
|
1084
|
+
wrongyear.push(n.tok); // a credited published figure of that year
|
|
1085
|
+
}
|
|
1086
|
+
if (fhits.length && !isTable && !n.table) {
|
|
1087
|
+
const who = misnamed(sent0, n, fhits, ents) || clauseOwner(sent0, n, fhits, facts) || measureNamed(sent0, n, fhits, facts);
|
|
1088
|
+
if (who && !hits.some(([name, f]) => !isFactName(name) && [...forms(who)].some((form) => cellKey(f._measure ?? "").includes(form)))) {
|
|
1089
|
+
misnamedL.push(`${n.tok} (next to ${who})`);
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
const row = isTable ? rowLabel(n) : null;
|
|
1093
|
+
const pubSrcs = hits.filter(([name, f]) => !isFactName(name) && (row === null ||
|
|
1094
|
+
[...forms(row), strip(row.split(/[-,(]/)[0])].some((form) => form.length >= 3 && cellKey(f._measure ?? "").includes(form))))
|
|
1095
|
+
.map(([, f]) => f._source ?? "");
|
|
1096
|
+
if (isTable && fhits.length && !rowOk(n, fhits, facts) && !(pubSrcs.length && attributed(block, n, true, pubSrcs)))
|
|
1097
|
+
misrow.push(n.tok);
|
|
1098
|
+
const dhits = fhits.length && !n.table ? dated(sent0, n, fhits) : fhits; // the sentence's year
|
|
1099
|
+
const atx = (x) => at(x, n.tok);
|
|
1100
|
+
if (fhits.length && !n.neg && !n.table && dhits.every((f) => value(f) > 0)
|
|
1101
|
+
&& (sent0.some((x) => search(FELL_BEFORE, x.slice(Math.max(0, atx(x) - 25), atx(x))))
|
|
1102
|
+
|| (dhits.every(isChange) && sent0.some((x) => match(LOWER_AFTER, x.slice(atx(x) + n.tok.length)))))
|
|
1103
|
+
&& !dhits.some((f) => search(SIZE_LABEL, str(f.label ?? "") + " " + str(f.derivation ?? "")))) {
|
|
1104
|
+
unsigned.push(n.tok); // "down 0.8" for a rise of 0.8
|
|
1105
|
+
}
|
|
1106
|
+
if (fhits.length && !n.neg && !n.table && dhits.every(negative) && n.num !== 0
|
|
1107
|
+
&& !(n.unit === "" && n.num <= 12 && Number.isInteger(n.num))) {
|
|
1108
|
+
const sl = sentences(block).filter((x) => has(x, n.tok));
|
|
1109
|
+
const sent = sl.length ? sl : [block];
|
|
1110
|
+
// the words just before the number say its direction ("Virginia rose 25,600"); else the sentence
|
|
1111
|
+
const near = sent.filter((x) => has(x, n.tok)).map((x) => x.slice(Math.max(0, atx(x) - 60), atx(x)));
|
|
1112
|
+
// or just after it ("426 fewer pages", "0.3 points lower")
|
|
1113
|
+
const after = sent.filter((x) => has(x, n.tok)).map((x) => x.slice(atx(x) + n.tok.length, atx(x) + n.tok.length + 30));
|
|
1114
|
+
const upNear = near.some((w) => search(UP_WORDS, w) && !search(DOWN_WORDS, w));
|
|
1115
|
+
let down = [...near, ...after].some((w) => search(DOWN_WORDS, w));
|
|
1116
|
+
if (!down) { // a fall word earlier in the sentence with no rise word after it
|
|
1117
|
+
for (const x of sent) {
|
|
1118
|
+
const pre = x.slice(0, Math.max(atx(x), 0));
|
|
1119
|
+
const last = finditer(DOWN_WORDS, pre);
|
|
1120
|
+
if (last.length && !search(UP_WORDS, pre.slice(end(last[last.length - 1]))))
|
|
1121
|
+
down = true;
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
if (!down && near.some((w) => /\bby\s*$/i.test(w)))
|
|
1125
|
+
down = sent.some((x) => search(DOWN_WORDS, x));
|
|
1126
|
+
const cmpAfter = after.some((w) => match(CMP_AFTER, w));
|
|
1127
|
+
const size = near.some((w) => search(SIZE_BEFORE, w));
|
|
1128
|
+
const upAfter = after.some((w) => match(UP_AFTER, w));
|
|
1129
|
+
if ((upNear && !cmpAfter) || upAfter || (!down && !size))
|
|
1130
|
+
unsigned.push(n.tok); // "rose 4.2" for -4.2
|
|
1131
|
+
}
|
|
1132
|
+
else if (fhits.length && !n.neg && !n.table && n.num !== 0
|
|
1133
|
+
&& dhits.every((f) => below0(f) && String(f.fmt || "").split(":").includes("abs"))) {
|
|
1134
|
+
// a fall printed without its sign (fmt ...:abs) may carry no fall word, but never a rise word
|
|
1135
|
+
const close = sentences(block).filter((x) => has(x, n.tok)).map((x) => x.slice(Math.max(0, atx(x) - 25), atx(x)));
|
|
1136
|
+
if (close.some((w) => search(UP_WORDS, w) && !search(DOWN_WORDS, w)))
|
|
1137
|
+
unsigned.push(n.tok);
|
|
1138
|
+
}
|
|
1139
|
+
if (fhits.length && n.neg && !n.table && sent0.some((x) => search(FELL_MINUS, x.slice(Math.max(0, atx(x) - 30), atx(x))))) {
|
|
1140
|
+
unsigned.push(n.tok); // "fell -8.3 percent": a fall word and a minus sign
|
|
1141
|
+
}
|
|
1142
|
+
for (const [flag, pat] of Object.entries(FLAG_WORDS)) { // every reading of this number is flagged: say so here
|
|
1143
|
+
const near = ctx + (flag === "preliminary" ? "\n" + lead : "");
|
|
1144
|
+
const agg = (xs) => (flag === "preliminary" ? xs.some((f) => f[flag]) : xs.every((f) => f[flag]));
|
|
1145
|
+
let flagged = agg(fhits);
|
|
1146
|
+
const pub = hits.filter(([name]) => !isFactName(name)).map(([, f]) => f._source ?? "");
|
|
1147
|
+
if (pub.length && attributed(block, n, !!n.table, pub)) {
|
|
1148
|
+
// another publisher's figure, credited: our facts count only when they print at the number's own precision
|
|
1149
|
+
const exact = fhits.filter((f) => ((factDisplay(f) || [0, "", 0])[2] || 0) <= (n.dec ?? 0));
|
|
1150
|
+
flagged = exact.length ? agg(exact) : false;
|
|
1151
|
+
}
|
|
1152
|
+
if (fhits.length && flagged && !search(pat, near)) {
|
|
1153
|
+
if (!unlabelled.has(flag))
|
|
1154
|
+
unlabelled.set(flag, []);
|
|
1155
|
+
unlabelled.get(flag).push(n.tok);
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
const okTies = new Set((D.rounding_ties_ok || []).map((t) => str(isObj(t) ? t.fact : t)));
|
|
1161
|
+
const ties = sorted([...cited].filter((n) => !okTies.has(n) && (facts[n] || {}).source && toFloat(facts[n].value) !== null
|
|
1162
|
+
&& roundingTie(toFloat(facts[n].value), pyFloatRepr(toFloat(facts[n].value)), str(facts[n].fmt || ""))));
|
|
1163
|
+
const uniq = (xs) => [...new Set(xs)].join(", ");
|
|
1164
|
+
out.push(["answer", !ties.length, !ties.length ? "no cited figure sits on an unexplained rounding tie"
|
|
1165
|
+
: `answer.md cites ${ties.join(", ")}, stored on a rounding tie (one digit past the format, ending in 5): the printed ` +
|
|
1166
|
+
"figure depends on digits the CSV dropped. Save the exhibit at full precision (no rounding step) and register again, " +
|
|
1167
|
+
"or, when the value is exact at that precision (published so), list it in decisions.json " +
|
|
1168
|
+
'"rounding_ties_ok": [{"fact": "<name>", "reason": "..."}]']);
|
|
1169
|
+
out.push(["answer", !bad.length, !bad.length ? "every number in answer.md is a registered fact or allowed"
|
|
1170
|
+
: `answer.md: ${uniq(bad)} not found among the facts (as their formats show them): register each number with the ` +
|
|
1171
|
+
'facts tool (a count in words such as "three of the four" is a number too: count it with compute; a difference in ' +
|
|
1172
|
+
"percentage points takes fmt pp or pp100, in basis points bp or bp100; a number written \"44,347 thousand\" or " +
|
|
1173
|
+
'"$1.2 billion" is read in base units, so a fact stored in thousands takes fmt thousands:0, and a published figure ' +
|
|
1174
|
+
'counts as benchmark.csv gives it), fix the typo, or list it in decisions.json audit_allow {"value", "reason"} when ' +
|
|
1175
|
+
'it is not a statistic (the value as written, with its unit: "90 percent")']);
|
|
1176
|
+
unsigned.push(...fromToMisreads(blocks(scan).filter(([b]) => !isTableBlock(b)).map(([b]) => b).join("\n")));
|
|
1177
|
+
out.push(["answer", !unsigned.length, !unsigned.length ? "every change reads in its own direction"
|
|
1178
|
+
: `answer.md: ${uniq(unsigned)}: the direction words disagree with the sign of the facts they match (a negative fact ` +
|
|
1179
|
+
"needs fell, declined, down, lower, negative ... next to it; a positive one cannot be written as down or fell): give " +
|
|
1180
|
+
"the sign or fix the direction"]);
|
|
1181
|
+
out.push(["answer", !wrongyear.length, !wrongyear.length ? "every number dated in the text is a fact of that date"
|
|
1182
|
+
: `answer.md: ${uniq(wrongyear)}: the year the text gives next to the number is not one any of the facts it matches is ` +
|
|
1183
|
+
"about (those facts are keyed to other periods): check the year or the number"]);
|
|
1184
|
+
if (unbound.length) {
|
|
1185
|
+
MA.note(`answer.md: the table starting with row ${unbound.map(repr).join(", ")} has no fact registered by its row ` +
|
|
1186
|
+
"labels, so a value moved between rows would pass: register each row's facts with where = the row's label as printed");
|
|
1187
|
+
}
|
|
1188
|
+
out.push(["answer", !misnamedL.length, !misnamedL.length ? "every number beside a name is that name's fact"
|
|
1189
|
+
: `answer.md: ${uniq(misnamedL)}: the number matches only facts of other entries of the same kind (another state, ` +
|
|
1190
|
+
"industry, group): check the pairing, or register the fact with where = that name"]);
|
|
1191
|
+
out.push(["answer", !misrow.length, !misrow.length ? "every table cell is a fact of its own row"
|
|
1192
|
+
: `answer.md: table cells ${uniq(misrow)} match facts of other rows only (their labels name another row): check the ` +
|
|
1193
|
+
"row, or register the cell's fact with where = the row's label"]);
|
|
1194
|
+
const miscol = blocks(scan).filter(([b]) => isTableBlock(b)).flatMap(([b]) => columnMisfits(b, facts));
|
|
1195
|
+
out.push(["answer", !miscol.length, !miscol.length ? "every table cell is a fact of its own column"
|
|
1196
|
+
: `answer.md: table cells ${uniq(miscol)} hold the measure of another column (their facts' labels match that column's ` +
|
|
1197
|
+
"other cells): check the column order"]);
|
|
1198
|
+
out.push(["answer", !unattributed.length, !unattributed.length ? "every published figure is credited to its source"
|
|
1199
|
+
: `answer.md: ${uniq(unattributed)} match only a published figure in benchmark.csv, not one of this analysis's facts: ` +
|
|
1200
|
+
'say whose figure it is ("BLS publishes ...", a "Published" column), or cite the computed fact']);
|
|
1201
|
+
const body = norm(text);
|
|
1202
|
+
const opening = norm(blocks(text).slice(0, 2).map(([b]) => b).join("\n"));
|
|
1203
|
+
const blocking = new Set((D.questions || []).filter((q) => isObj(q) && q.blocking && !(q.asked ?? false)).map((q) => q.id));
|
|
1204
|
+
for (const [k, line] of Object.entries(D.disclosures || {})) {
|
|
1205
|
+
if (blocking.has(k) && body.includes(norm(line)) && !opening.includes(norm(line))) {
|
|
1206
|
+
out.push(["disclosures", false, `${k}: a blocking question decided without the user: put its disclosure in the ` +
|
|
1207
|
+
"answer's opening (title and first paragraph)"]);
|
|
1208
|
+
}
|
|
1209
|
+
const ok = body.includes(norm(line));
|
|
1210
|
+
out.push(["disclosures", ok, ok ? `${k}: carried in answer.md` : `${k}: answer.md must carry this disclosure word for word: ${str(line)}`]);
|
|
1211
|
+
}
|
|
1212
|
+
for (const [flag, toks] of unlabelled) {
|
|
1213
|
+
out.push(["answer", false, `answer.md cites ${flag} values (${uniq(toks)}) without saying so in the opening paragraph ` +
|
|
1214
|
+
"or next to them (same paragraph, or the table and its caption)"]);
|
|
1215
|
+
}
|
|
1216
|
+
const answerToks = answerNumbers(scan);
|
|
1217
|
+
const factVals = MA.factValues(facts);
|
|
1218
|
+
for (const r of benchRows) { // a gap beyond rounding on a cited figure: the answer says it and why
|
|
1219
|
+
const c = MA.benchNumber(r.computed), pb = MA.benchNumber(r.published);
|
|
1220
|
+
const isCited = c && MA.isFact(c[0], Math.max(c[1], Math.abs(c[0]) * 0.005), factVals);
|
|
1221
|
+
const sk = MA.SAYS.find((k) => r[k]);
|
|
1222
|
+
const says = sk ? strip(String(r[sk])).toLowerCase() : "";
|
|
1223
|
+
if (c && pb && isCited && Math.abs(c[0] - pb[0]) > Math.max(pb[1] * 1.0001, Math.abs(pb[0]) * 1e-6) + 1e-12
|
|
1224
|
+
&& !(says.startsWith("yes") && MA.roundingGap(c, pb, r.published, r.note || r.notes))) {
|
|
1225
|
+
const ok = answerToks.some((n) => matches(n, pb[0], "") || Math.abs(n.num - Math.abs(pb[0])) <= n.tol * 1.0001 + 1e-12);
|
|
1226
|
+
out.push(["answer", ok, ok ? `benchmark ${r.measure ?? ""}: the published ${str(r.published)} is in answer.md`
|
|
1227
|
+
: `benchmark ${r.measure ?? ""} (${r.period ?? ""}): computed ${str(r.computed)} differs from the published ` +
|
|
1228
|
+
`${str(r.published)}; answer.md must give the published figure and why (${strip(String(r.note || "")).slice(0, 80)})`]);
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
out.push(checkDollarBasis(text, D));
|
|
1232
|
+
if (benchRows.some((r) => !String(r.published ?? "").toLowerCase().includes("not checked"))) {
|
|
1233
|
+
const ok = !!search(BENCH_WORDS, text);
|
|
1234
|
+
out.push(["answer", ok, ok ? "answer.md reports the benchmark check" : "answer.md does not report the benchmark check: " +
|
|
1235
|
+
"say which published figure each headline was compared with and whether it matches (exhibits/benchmark.csv)"]);
|
|
1236
|
+
}
|
|
1237
|
+
return [out, text];
|
|
1238
|
+
}
|
|
1239
|
+
/** Tables read by hand from an agency page: each row of published/<name>.csv names its page and date. */
|
|
1240
|
+
export function checkPublished(base) {
|
|
1241
|
+
const d = path.join(base, "published");
|
|
1242
|
+
const out = [];
|
|
1243
|
+
for (const f of fs.existsSync(d) && fs.statSync(d).isDirectory() ? fs.readdirSync(d).sort() : []) {
|
|
1244
|
+
if (!f.toLowerCase().endsWith(".csv"))
|
|
1245
|
+
continue;
|
|
1246
|
+
const [head, rows] = MA.readCsv(path.join(d, f));
|
|
1247
|
+
const missing = ["source_url", "accessed"].filter((c) => !head.includes(c));
|
|
1248
|
+
const blank = rows.filter((r) => !String(r.source_url || "").startsWith("http") || !/^\d{4}-\d{2}-\d{2}$/.test(strip(String(r.accessed || "")))).length;
|
|
1249
|
+
const ok = rows.length > 0 && !missing.length && !blank;
|
|
1250
|
+
const msg = ok ? `published/${f}: ${rows.length} rows read from agency pages, each with its source_url and accessed date`
|
|
1251
|
+
: `published/${f}: every row needs source_url (the page) and accessed (YYYY-MM-DD)` +
|
|
1252
|
+
(missing.length ? `; missing columns ${repr(missing)}` : rows.length ? `; ${blank} row(s) lack them` : "; no rows");
|
|
1253
|
+
out.push([f, ok, msg, sorted(new Set(rows.map((r) => String(r.source_url || ""))))]);
|
|
1254
|
+
}
|
|
1255
|
+
return out;
|
|
1256
|
+
}
|
|
1257
|
+
export async function audit(out, online = false, rerunIt = false) {
|
|
1258
|
+
const base = path.resolve(out);
|
|
1259
|
+
const results = [];
|
|
1260
|
+
const res = (check, ok, msg) => void results.push([check, !!ok, msg]);
|
|
1261
|
+
if (!fs.existsSync(path.join(base, "decisions.json"))) {
|
|
1262
|
+
res("questions", false, "decisions.json missing: record the questions, literature searched, references and " +
|
|
1263
|
+
"assumptions (record_decisions)");
|
|
1264
|
+
}
|
|
1265
|
+
const D = MA.merged(base);
|
|
1266
|
+
const fpath = path.join(base, "facts.json");
|
|
1267
|
+
const facts = fs.existsSync(fpath) ? JSON.parse(AR.readTextFile(fpath)) || {} : {};
|
|
1268
|
+
if (!Object.keys(facts).length)
|
|
1269
|
+
res("facts", false, "facts.json missing or empty: register every number the answer cites (the facts tool)");
|
|
1270
|
+
for (const name of sorted(Object.keys(facts)))
|
|
1271
|
+
res("facts", ...MA.checkFact(name, facts[name] || {}, base));
|
|
1272
|
+
const [ans, answerText] = checkAnswer(base, facts, D);
|
|
1273
|
+
results.push(...ans, ...MA.checkMethod(D, base, facts), ...MA.checkBenchmark(base, facts));
|
|
1274
|
+
const ids = PS.usedSources(base, {});
|
|
1275
|
+
let policies = {};
|
|
1276
|
+
try {
|
|
1277
|
+
policies = PS.loadPolicies(ids);
|
|
1278
|
+
}
|
|
1279
|
+
catch (e) {
|
|
1280
|
+
if (!(e instanceof PS.SourceError))
|
|
1281
|
+
throw e;
|
|
1282
|
+
res("vintage", false, e.message);
|
|
1283
|
+
}
|
|
1284
|
+
if (!ids.length)
|
|
1285
|
+
res("vintage", false, "no source in data/<id>/: fetch the data with the fetch tool");
|
|
1286
|
+
for (const [sid, P] of Object.entries(policies)) { // each source's required statements, in the answer itself
|
|
1287
|
+
for (const [status, hid, msg] of PS.runAuditHooks(P, base, null, facts, answerText)) {
|
|
1288
|
+
res("policy", status === "PASS", `${sid} hook ${hid}: ${msg.replaceAll("report mentions", "answer.md mentions")}`);
|
|
1289
|
+
}
|
|
1290
|
+
}
|
|
1291
|
+
for (const sid of Object.keys(policies)) {
|
|
1292
|
+
const [rel] = PS.provenance(base, sid, {});
|
|
1293
|
+
res("vintage", !!rel, rel ? `${sid}: provenance ${rel}` : `${sid}: no provenance file in data/${sid}/ (request.json): the ` +
|
|
1294
|
+
"retrieval date cannot be stated");
|
|
1295
|
+
}
|
|
1296
|
+
const pubUrls = [];
|
|
1297
|
+
for (const [, ok, msg, urlsHere] of checkPublished(base)) {
|
|
1298
|
+
res("vintage", ok, msg);
|
|
1299
|
+
pubUrls.push(...urlsHere);
|
|
1300
|
+
}
|
|
1301
|
+
const bp = path.join(base, "exhibits", "benchmark.csv"); // the published figures' pages are links too
|
|
1302
|
+
const benchUrls = [...(fs.existsSync(bp) ? MA.readCsv(bp)[1].map((r) => String(r.url || "")) : []), ...pubUrls];
|
|
1303
|
+
const urls = sorted(new Set([...AR.allStrings(D), ...benchUrls].flatMap(AR.findUrls).map((u) => u.replace(/[.,;:]+$/, ""))));
|
|
1304
|
+
for (const u of urls)
|
|
1305
|
+
res("links", ...(online ? await AR.checkUrl(u) : [true, `${u}: not requested (use online: true to verify)`]));
|
|
1306
|
+
const hits = relHits(scanTree([base], knownKeys(), echoSources(policies).length > 0)[0], base);
|
|
1307
|
+
res("secrets", !hits.length, !hits.length ? "no API key or credential in any output file" : hits.slice(0, 10).join("; "));
|
|
1308
|
+
if (rerunIt) {
|
|
1309
|
+
const exdir = path.join(base, "exhibits");
|
|
1310
|
+
const tracked = fs.existsSync(exdir) ? fs.readdirSync(exdir).filter((f) => f.endsWith(".csv")).sort().map((f) => path.join(exdir, f)) : [];
|
|
1311
|
+
results.push(...(await MA.rerun(base, fpath, tracked)));
|
|
1312
|
+
}
|
|
1313
|
+
return results;
|
|
1314
|
+
}
|
|
1315
|
+
/** The lines audit_answer.py prints: one [PASS]/[FAIL] line per check, the failures (and the benchmark counts),
|
|
1316
|
+
* notes on what the answer must disclose, then AUDIT PASSED or AUDIT FAILED (n problems). */
|
|
1317
|
+
export function printed(results, checks, answerText = "", notes = []) {
|
|
1318
|
+
const lines = [];
|
|
1319
|
+
const fails = results.filter(([, ok]) => !ok);
|
|
1320
|
+
for (const name of [...new Set([...checks, ...results.map(([c]) => c)])]) {
|
|
1321
|
+
const rs = results.filter(([c]) => c === name);
|
|
1322
|
+
if (!rs.length)
|
|
1323
|
+
continue;
|
|
1324
|
+
const bad = rs.filter(([, ok]) => !ok);
|
|
1325
|
+
lines.push(`[${bad.length ? "FAIL" : "PASS"}] ${name}: ${rs.length - bad.length}/${rs.length} ok`);
|
|
1326
|
+
if (name === "benchmark" && !bad.length)
|
|
1327
|
+
for (const r of rs)
|
|
1328
|
+
lines.push(` ${redact(r[2])}`);
|
|
1329
|
+
for (const r of bad)
|
|
1330
|
+
lines.push(` - ${redact(r[2])}`);
|
|
1331
|
+
}
|
|
1332
|
+
const low = answerText.toLowerCase();
|
|
1333
|
+
for (const [, ok, msg] of results) { // what the answer must disclose
|
|
1334
|
+
if (ok && (msg.includes("disclose") || msg.includes("opening paragraph of the answer"))) {
|
|
1335
|
+
const line = msg.includes("disclose in the answer: ") ? strip(msg.split("disclose in the answer: ").slice(1).join("disclose in the answer: ")).toLowerCase() : "";
|
|
1336
|
+
if (line && low.includes(line))
|
|
1337
|
+
continue;
|
|
1338
|
+
if (msg.includes("opening paragraph of the answer") && low && !results.some(([c, o]) => c === "disclosures" && !o))
|
|
1339
|
+
continue;
|
|
1340
|
+
lines.push(`note: ${redact(msg)}`);
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
for (const n of notes)
|
|
1344
|
+
lines.push(`note: ${redact(n)}`);
|
|
1345
|
+
const verdict = "AUDIT " + (!fails.length ? "PASSED" : `FAILED (${fails.length} problems)`);
|
|
1346
|
+
lines.push(verdict);
|
|
1347
|
+
return { lines, verdict, passed: !fails.length };
|
|
1348
|
+
}
|
|
1349
|
+
//# sourceMappingURL=audit_answer.js.map
|