data-analyst-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +261 -2
- package/dist/adapters/_acf_tidy.d.ts +18 -0
- package/dist/adapters/_acf_tidy.js +432 -0
- package/dist/adapters/_acf_tidy.js.map +1 -0
- package/dist/adapters/_actuaries_read.d.ts +38 -0
- package/dist/adapters/_actuaries_read.js +728 -0
- package/dist/adapters/_actuaries_read.js.map +1 -0
- package/dist/adapters/_census_micro.d.ts +70 -0
- package/dist/adapters/_census_micro.js +622 -0
- package/dist/adapters/_census_micro.js.map +1 -0
- package/dist/adapters/_census_util.d.ts +40 -0
- package/dist/adapters/_census_util.js +275 -0
- package/dist/adapters/_census_util.js.map +1 -0
- package/dist/adapters/_dol_states.d.ts +6 -0
- package/dist/adapters/_dol_states.js +58 -0
- package/dist/adapters/_dol_states.js.map +1 -0
- package/dist/adapters/_govinfo_entities.d.ts +4 -0
- package/dist/adapters/_govinfo_entities.js +6 -0
- package/dist/adapters/_govinfo_entities.js.map +1 -0
- package/dist/adapters/_govinfo_json.d.ts +13 -0
- package/dist/adapters/_govinfo_json.js +107 -0
- package/dist/adapters/_govinfo_json.js.map +1 -0
- package/dist/adapters/_govinfo_py.d.ts +22 -0
- package/dist/adapters/_govinfo_py.js +272 -0
- package/dist/adapters/_govinfo_py.js.map +1 -0
- package/dist/adapters/_govinfo_xml.d.ts +22 -0
- package/dist/adapters/_govinfo_xml.js +233 -0
- package/dist/adapters/_govinfo_xml.js.map +1 -0
- package/dist/adapters/_hud_excel.d.ts +2 -0
- package/dist/adapters/_hud_excel.js +166 -0
- package/dist/adapters/_hud_excel.js.map +1 -0
- package/dist/adapters/_omb_grid.d.ts +17 -0
- package/dist/adapters/_omb_grid.js +132 -0
- package/dist/adapters/_omb_grid.js.map +1 -0
- package/dist/adapters/_omb_util.d.ts +40 -0
- package/dist/adapters/_omb_util.js +192 -0
- package/dist/adapters/_omb_util.js.map +1 -0
- package/dist/adapters/_usaspending_state.d.ts +6 -0
- package/dist/adapters/_usaspending_state.js +54 -0
- package/dist/adapters/_usaspending_state.js.map +1 -0
- package/dist/adapters/_usda-fna_book.d.ts +59 -0
- package/dist/adapters/_usda-fna_book.js +869 -0
- package/dist/adapters/_usda-fna_book.js.map +1 -0
- package/dist/adapters/_usda-fna_pdf.d.ts +20 -0
- package/dist/adapters/_usda-fna_pdf.js +361 -0
- package/dist/adapters/_usda-fna_pdf.js.map +1 -0
- package/dist/adapters/_usda-outlook_read.d.ts +70 -0
- package/dist/adapters/_usda-outlook_read.js +362 -0
- package/dist/adapters/_usda-outlook_read.js.map +1 -0
- package/dist/adapters/acf.d.ts +3 -0
- package/dist/adapters/acf.js +187 -0
- package/dist/adapters/acf.js.map +1 -0
- package/dist/adapters/actuaries.d.ts +3 -0
- package/dist/adapters/actuaries.js +293 -0
- package/dist/adapters/actuaries.js.map +1 -0
- package/dist/adapters/bts.d.ts +3 -0
- package/dist/adapters/bts.js +26 -0
- package/dist/adapters/bts.js.map +1 -0
- package/dist/adapters/census-geocoder.d.ts +10 -0
- package/dist/adapters/census-geocoder.js +47 -0
- package/dist/adapters/census-geocoder.js.map +1 -0
- package/dist/adapters/census.d.ts +17 -0
- package/dist/adapters/census.js +983 -0
- package/dist/adapters/census.js.map +1 -0
- package/dist/adapters/cfpb.d.ts +7 -0
- package/dist/adapters/cfpb.js +348 -0
- package/dist/adapters/cfpb.js.map +1 -0
- package/dist/adapters/clinical-trials.d.ts +3 -0
- package/dist/adapters/clinical-trials.js +135 -0
- package/dist/adapters/clinical-trials.js.map +1 -0
- package/dist/adapters/cms.d.ts +3 -0
- package/dist/adapters/cms.js +292 -0
- package/dist/adapters/cms.js.map +1 -0
- package/dist/adapters/college-scorecard.d.ts +3 -0
- package/dist/adapters/college-scorecard.js +122 -0
- package/dist/adapters/college-scorecard.js.map +1 -0
- package/dist/adapters/congress.d.ts +33 -0
- package/dist/adapters/congress.js +589 -0
- package/dist/adapters/congress.js.map +1 -0
- package/dist/adapters/dhs-ohss.d.ts +13 -0
- package/dist/adapters/dhs-ohss.js +598 -0
- package/dist/adapters/dhs-ohss.js.map +1 -0
- package/dist/adapters/doj-news.d.ts +7 -0
- package/dist/adapters/doj-news.js +286 -0
- package/dist/adapters/doj-news.js.map +1 -0
- package/dist/adapters/dol.d.ts +4 -0
- package/dist/adapters/dol.js +634 -0
- package/dist/adapters/dol.js.map +1 -0
- package/dist/adapters/eia.d.ts +15 -0
- package/dist/adapters/eia.js +456 -0
- package/dist/adapters/eia.js.map +1 -0
- package/dist/adapters/epa-aqs.d.ts +3 -0
- package/dist/adapters/epa-aqs.js +357 -0
- package/dist/adapters/epa-aqs.js.map +1 -0
- package/dist/adapters/epa.d.ts +3 -0
- package/dist/adapters/epa.js +298 -0
- package/dist/adapters/epa.js.map +1 -0
- package/dist/adapters/fbi.d.ts +3 -0
- package/dist/adapters/fbi.js +192 -0
- package/dist/adapters/fbi.js.map +1 -0
- package/dist/adapters/fda.d.ts +5 -0
- package/dist/adapters/fda.js +184 -0
- package/dist/adapters/fda.js.map +1 -0
- package/dist/adapters/fdic.d.ts +4 -0
- package/dist/adapters/fdic.js +95 -0
- package/dist/adapters/fdic.js.map +1 -0
- package/dist/adapters/fec.d.ts +3 -0
- package/dist/adapters/fec.js +142 -0
- package/dist/adapters/fec.js.map +1 -0
- package/dist/adapters/fed-board.d.ts +12 -0
- package/dist/adapters/fed-board.js +581 -0
- package/dist/adapters/fed-board.js.map +1 -0
- package/dist/adapters/federal-register.d.ts +11 -0
- package/dist/adapters/federal-register.js +43 -0
- package/dist/adapters/federal-register.js.map +1 -0
- package/dist/adapters/fhfa.d.ts +3 -0
- package/dist/adapters/fhfa.js +253 -0
- package/dist/adapters/fhfa.js.map +1 -0
- package/dist/adapters/fsa.d.ts +9 -0
- package/dist/adapters/fsa.js +335 -0
- package/dist/adapters/fsa.js.map +1 -0
- package/dist/adapters/govinfo.d.ts +30 -0
- package/dist/adapters/govinfo.js +636 -0
- package/dist/adapters/govinfo.js.map +1 -0
- package/dist/adapters/gsa-calc.d.ts +10 -0
- package/dist/adapters/gsa-calc.js +318 -0
- package/dist/adapters/gsa-calc.js.map +1 -0
- package/dist/adapters/hud.d.ts +3 -0
- package/dist/adapters/hud.js +200 -0
- package/dist/adapters/hud.js.map +1 -0
- package/dist/adapters/index.d.ts +3 -0
- package/dist/adapters/index.js +25 -0
- package/dist/adapters/index.js.map +1 -0
- package/dist/adapters/irs-soi.d.ts +6 -0
- package/dist/adapters/irs-soi.js +576 -0
- package/dist/adapters/irs-soi.js.map +1 -0
- package/dist/adapters/naep.d.ts +15 -0
- package/dist/adapters/naep.js +341 -0
- package/dist/adapters/naep.js.map +1 -0
- package/dist/adapters/nhtsa.d.ts +10 -0
- package/dist/adapters/nhtsa.js +239 -0
- package/dist/adapters/nhtsa.js.map +1 -0
- package/dist/adapters/nih.d.ts +5 -0
- package/dist/adapters/nih.js +196 -0
- package/dist/adapters/nih.js.map +1 -0
- package/dist/adapters/noaa.d.ts +3 -0
- package/dist/adapters/noaa.js +148 -0
- package/dist/adapters/noaa.js.map +1 -0
- package/dist/adapters/nrel.d.ts +3 -0
- package/dist/adapters/nrel.js +186 -0
- package/dist/adapters/nrel.js.map +1 -0
- package/dist/adapters/nws.d.ts +3 -0
- package/dist/adapters/nws.js +119 -0
- package/dist/adapters/nws.js.map +1 -0
- package/dist/adapters/omb.d.ts +15 -0
- package/dist/adapters/omb.js +517 -0
- package/dist/adapters/omb.js.map +1 -0
- package/dist/adapters/open-payments.d.ts +5 -0
- package/dist/adapters/open-payments.js +149 -0
- package/dist/adapters/open-payments.js.map +1 -0
- package/dist/adapters/pbgc.d.ts +13 -0
- package/dist/adapters/pbgc.js +283 -0
- package/dist/adapters/pbgc.js.map +1 -0
- package/dist/adapters/philfed.d.ts +12 -0
- package/dist/adapters/philfed.js +59 -0
- package/dist/adapters/philfed.js.map +1 -0
- package/dist/adapters/regulations.d.ts +6 -0
- package/dist/adapters/regulations.js +58 -0
- package/dist/adapters/regulations.js.map +1 -0
- package/dist/adapters/sec.d.ts +13 -0
- package/dist/adapters/sec.js +504 -0
- package/dist/adapters/sec.js.map +1 -0
- package/dist/adapters/senate-lobbying.d.ts +3 -0
- package/dist/adapters/senate-lobbying.js +181 -0
- package/dist/adapters/senate-lobbying.js.map +1 -0
- package/dist/adapters/snapqc.d.ts +32 -0
- package/dist/adapters/snapqc.js +701 -0
- package/dist/adapters/snapqc.js.map +1 -0
- package/dist/adapters/treasury-yields.d.ts +3 -0
- package/dist/adapters/treasury-yields.js +170 -0
- package/dist/adapters/treasury-yields.js.map +1 -0
- package/dist/adapters/usaspending.d.ts +3 -0
- package/dist/adapters/usaspending.js +340 -0
- package/dist/adapters/usaspending.js.map +1 -0
- package/dist/adapters/usda-fna.d.ts +7 -0
- package/dist/adapters/usda-fna.js +707 -0
- package/dist/adapters/usda-fna.js.map +1 -0
- package/dist/adapters/usda-fooddata.d.ts +3 -0
- package/dist/adapters/usda-fooddata.js +101 -0
- package/dist/adapters/usda-fooddata.js.map +1 -0
- package/dist/adapters/usda-nass.d.ts +3 -0
- package/dist/adapters/usda-nass.js +108 -0
- package/dist/adapters/usda-nass.js.map +1 -0
- package/dist/adapters/usda-outlook.d.ts +12 -0
- package/dist/adapters/usda-outlook.js +696 -0
- package/dist/adapters/usda-outlook.js.map +1 -0
- package/dist/adapters/usgs.d.ts +9 -0
- package/dist/adapters/usgs.js +265 -0
- package/dist/adapters/usgs.js.map +1 -0
- package/dist/adapters/usitc.d.ts +18 -0
- package/dist/adapters/usitc.js +170 -0
- package/dist/adapters/usitc.js.map +1 -0
- package/dist/adapters/uspto.d.ts +3 -0
- package/dist/adapters/uspto.js +47 -0
- package/dist/adapters/uspto.js.map +1 -0
- package/dist/adapters/world-bank.d.ts +3 -0
- package/dist/adapters/world-bank.js +163 -0
- package/dist/adapters/world-bank.js.map +1 -0
- package/dist/analysis/cite.d.ts +31 -0
- package/dist/analysis/cite.js +229 -0
- package/dist/analysis/cite.js.map +1 -0
- package/dist/analysis/combine.d.ts +27 -0
- package/dist/analysis/combine.js +198 -0
- package/dist/analysis/combine.js.map +1 -0
- package/dist/analysis/compute.d.ts +62 -0
- package/dist/analysis/compute.js +583 -0
- package/dist/analysis/compute.js.map +1 -0
- package/dist/analysis/expr.d.ts +69 -0
- package/dist/analysis/expr.js +780 -0
- package/dist/analysis/expr.js.map +1 -0
- package/dist/analysis/factfmt.d.ts +16 -0
- package/dist/analysis/factfmt.js +187 -0
- package/dist/analysis/factfmt.js.map +1 -0
- package/dist/analysis/facts.d.ts +63 -0
- package/dist/analysis/facts.js +355 -0
- package/dist/analysis/facts.js.map +1 -0
- package/dist/analysis/handoff.d.ts +58 -0
- package/dist/analysis/handoff.js +208 -0
- package/dist/analysis/handoff.js.map +1 -0
- package/dist/analysis/pyfmt.d.ts +16 -0
- package/dist/analysis/pyfmt.js +156 -0
- package/dist/analysis/pyfmt.js.map +1 -0
- package/dist/analysis/rerun.d.ts +13 -0
- package/dist/analysis/rerun.js +125 -0
- package/dist/analysis/rerun.js.map +1 -0
- package/dist/analysis/table.d.ts +18 -0
- package/dist/analysis/table.js +109 -0
- package/dist/analysis/table.js.map +1 -0
- package/dist/analysis/tools.d.ts +67 -0
- package/dist/analysis/tools.js +217 -0
- package/dist/analysis/tools.js.map +1 -0
- package/dist/analysis/write_table.d.ts +13 -0
- package/dist/analysis/write_table.js +59 -0
- package/dist/analysis/write_table.js.map +1 -0
- package/dist/bin/cli.d.ts +10 -0
- package/dist/bin/cli.js +177 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/core/cache.d.ts +14 -0
- package/dist/core/cache.js +90 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/context.d.ts +12 -0
- package/dist/core/context.js +15 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +14 -0
- package/dist/core/env.js +52 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/generic-rest.d.ts +61 -0
- package/dist/core/generic-rest.js +639 -0
- package/dist/core/generic-rest.js.map +1 -0
- package/dist/core/http.d.ts +120 -0
- package/dist/core/http.js +514 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/keys.d.ts +43 -0
- package/dist/core/keys.js +174 -0
- package/dist/core/keys.js.map +1 -0
- package/dist/core/manual.d.ts +41 -0
- package/dist/core/manual.js +218 -0
- package/dist/core/manual.js.map +1 -0
- package/dist/core/package.d.ts +58 -0
- package/dist/core/package.js +228 -0
- package/dist/core/package.js.map +1 -0
- package/dist/core/packs.d.ts +139 -0
- package/dist/core/packs.js +173 -0
- package/dist/core/packs.js.map +1 -0
- package/dist/core/policy.d.ts +69 -0
- package/dist/core/policy.js +189 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/py.d.ts +40 -0
- package/dist/core/py.js +171 -0
- package/dist/core/py.js.map +1 -0
- package/dist/core/routing.d.ts +10 -0
- package/dist/core/routing.js +39 -0
- package/dist/core/routing.js.map +1 -0
- package/dist/core/tabular.d.ts +21 -0
- package/dist/core/tabular.js +229 -0
- package/dist/core/tabular.js.map +1 -0
- package/dist/deep/_bea_api.d.ts +102 -0
- package/dist/deep/_bea_api.js +457 -0
- package/dist/deep/_bea_api.js.map +1 -0
- package/dist/deep/_bea_fetch.d.ts +151 -0
- package/dist/deep/_bea_fetch.js +1274 -0
- package/dist/deep/_bea_fetch.js.map +1 -0
- package/dist/deep/_bea_util.d.ts +14 -0
- package/dist/deep/_bea_util.js +156 -0
- package/dist/deep/_bea_util.js.map +1 -0
- package/dist/deep/_bls_api.d.ts +84 -0
- package/dist/deep/_bls_api.js +231 -0
- package/dist/deep/_bls_api.js.map +1 -0
- package/dist/deep/_bls_ids.d.ts +43 -0
- package/dist/deep/_bls_ids.js +188 -0
- package/dist/deep/_bls_ids.js.map +1 -0
- package/dist/deep/_bls_lib.d.ts +61 -0
- package/dist/deep/_bls_lib.js +442 -0
- package/dist/deep/_bls_lib.js.map +1 -0
- package/dist/deep/_fred_api.d.ts +148 -0
- package/dist/deep/_fred_api.js +545 -0
- package/dist/deep/_fred_api.js.map +1 -0
- package/dist/deep/_fred_pack.d.ts +162 -0
- package/dist/deep/_fred_pack.js +1027 -0
- package/dist/deep/_fred_pack.js.map +1 -0
- package/dist/deep/_fred_py.d.ts +40 -0
- package/dist/deep/_fred_py.js +199 -0
- package/dist/deep/_fred_py.js.map +1 -0
- package/dist/deep/_ipums_bundle.d.ts +29 -0
- package/dist/deep/_ipums_bundle.js +194 -0
- package/dist/deep/_ipums_bundle.js.map +1 -0
- package/dist/deep/_ipums_ddi.d.ts +23 -0
- package/dist/deep/_ipums_ddi.js +162 -0
- package/dist/deep/_ipums_ddi.js.map +1 -0
- package/dist/deep/_opm_io.d.ts +26 -0
- package/dist/deep/_opm_io.js +383 -0
- package/dist/deep/_opm_io.js.map +1 -0
- package/dist/deep/_opm_parquet.d.ts +31 -0
- package/dist/deep/_opm_parquet.js +744 -0
- package/dist/deep/_opm_parquet.js.map +1 -0
- package/dist/deep/_opm_py.d.ts +6 -0
- package/dist/deep/_opm_py.js +74 -0
- package/dist/deep/_opm_py.js.map +1 -0
- package/dist/deep/_treasury_api.d.ts +48 -0
- package/dist/deep/_treasury_api.js +221 -0
- package/dist/deep/_treasury_api.js.map +1 -0
- package/dist/deep/_treasury_context.d.ts +34 -0
- package/dist/deep/_treasury_context.js +142 -0
- package/dist/deep/_treasury_context.js.map +1 -0
- package/dist/deep/_treasury_lib.d.ts +109 -0
- package/dist/deep/_treasury_lib.js +686 -0
- package/dist/deep/_treasury_lib.js.map +1 -0
- package/dist/deep/bea.d.ts +3 -0
- package/dist/deep/bea.js +248 -0
- package/dist/deep/bea.js.map +1 -0
- package/dist/deep/bls.d.ts +3 -0
- package/dist/deep/bls.js +220 -0
- package/dist/deep/bls.js.map +1 -0
- package/dist/deep/fred.d.ts +3 -0
- package/dist/deep/fred.js +325 -0
- package/dist/deep/fred.js.map +1 -0
- package/dist/deep/ipums.d.ts +59 -0
- package/dist/deep/ipums.js +744 -0
- package/dist/deep/ipums.js.map +1 -0
- package/dist/deep/opm.d.ts +56 -0
- package/dist/deep/opm.js +1033 -0
- package/dist/deep/opm.js.map +1 -0
- package/dist/deep/treasury.d.ts +3 -0
- package/dist/deep/treasury.js +218 -0
- package/dist/deep/treasury.js.map +1 -0
- package/dist/envelope.d.ts +47 -0
- package/dist/envelope.js +88 -0
- package/dist/envelope.js.map +1 -0
- package/dist/index.d.ts +277 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/remote/app.d.ts +21 -0
- package/dist/remote/app.js +218 -0
- package/dist/remote/app.js.map +1 -0
- package/dist/remote/oauth.d.ts +14 -0
- package/dist/remote/oauth.js +90 -0
- package/dist/remote/oauth.js.map +1 -0
- package/dist/remote/pages.d.ts +8 -0
- package/dist/remote/pages.js +56 -0
- package/dist/remote/pages.js.map +1 -0
- package/dist/remote/sign.d.ts +4 -0
- package/dist/remote/sign.js +40 -0
- package/dist/remote/sign.js.map +1 -0
- package/dist/remote/store.d.ts +31 -0
- package/dist/remote/store.js +129 -0
- package/dist/remote/store.js.map +1 -0
- package/dist/report/audit_answer.d.ts +94 -0
- package/dist/report/audit_answer.js +1349 -0
- package/dist/report/audit_answer.js.map +1 -0
- package/dist/report/audit_report.d.ts +23 -0
- package/dist/report/audit_report.js +637 -0
- package/dist/report/audit_report.js.map +1 -0
- package/dist/report/build_report.d.ts +27 -0
- package/dist/report/build_report.js +1168 -0
- package/dist/report/build_report.js.map +1 -0
- package/dist/report/method_audit.d.ts +47 -0
- package/dist/report/method_audit.js +716 -0
- package/dist/report/method_audit.js.map +1 -0
- package/dist/report/policyset.d.ts +50 -0
- package/dist/report/policyset.js +421 -0
- package/dist/report/policyset.js.map +1 -0
- package/dist/report/py.d.ts +58 -0
- package/dist/report/py.js +311 -0
- package/dist/report/py.js.map +1 -0
- package/dist/report/scan_secrets.d.ts +24 -0
- package/dist/report/scan_secrets.js +265 -0
- package/dist/report/scan_secrets.js.map +1 -0
- package/dist/report/style_check.d.ts +29 -0
- package/dist/report/style_check.js +173 -0
- package/dist/report/style_check.js.map +1 -0
- package/dist/report/tools.d.ts +87 -0
- package/dist/report/tools.js +167 -0
- package/dist/report/tools.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.js +247 -0
- package/dist/server.js.map +1 -0
- package/dist/session.d.ts +47 -0
- package/dist/session.js +107 -0
- package/dist/session.js.map +1 -0
- package/dist/toolkit.d.ts +21 -0
- package/dist/toolkit.js +2 -0
- package/dist/toolkit.js.map +1 -0
- package/dist/tools.d.ts +299 -0
- package/dist/tools.js +491 -0
- package/dist/tools.js.map +1 -0
- package/guide/WORKFLOW.md +134 -0
- package/guide/assets/disclaimer.txt +1 -0
- package/guide/assets/download_data.ps1 +162 -0
- package/guide/assets/download_data.sh +143 -0
- package/guide/assets/report_template.html +597 -0
- package/guide/references/report_patterns.md +199 -0
- package/guide/references/routing.md +171 -0
- package/guide/references/style_guide.md +398 -0
- package/package.json +54 -4
- package/packs/acf/endpoints.json +114 -0
- package/packs/acf/policy.json +104 -0
- package/packs/acf/source.md +138 -0
- package/packs/actuaries/endpoints.json +82 -0
- package/packs/actuaries/policy.json +109 -0
- package/packs/actuaries/source.md +137 -0
- package/packs/bea/assets/download_bea.ps1 +182 -0
- package/packs/bea/assets/download_bea.sh +144 -0
- package/packs/bea/docs/bea_api.md +461 -0
- package/packs/bea/docs/datasets.md +157 -0
- package/packs/bea/docs/report_patterns.md +112 -0
- package/packs/bea/policy.json +170 -0
- package/packs/bea/source.md +152 -0
- package/packs/bls/assets/download_bls.ps1 +152 -0
- package/packs/bls/assets/download_bls.sh +167 -0
- package/packs/bls/docs/analysis_patterns.md +108 -0
- package/packs/bls/docs/bls_api.md +73 -0
- package/packs/bls/docs/series_ids.md +149 -0
- package/packs/bls/policy.json +238 -0
- package/packs/bls/series_ids.json +1375 -0
- package/packs/bls/source.md +156 -0
- package/packs/bts/endpoints.json +347 -0
- package/packs/bts/policy.json +94 -0
- package/packs/bts/source.md +97 -0
- package/packs/cdc/endpoints.json +197 -0
- package/packs/cdc/policy.json +83 -0
- package/packs/cdc/source.md +74 -0
- package/packs/census/endpoints.json +99 -0
- package/packs/census/policy.json +238 -0
- package/packs/census/source.md +304 -0
- package/packs/census-geocoder/endpoints.json +70 -0
- package/packs/census-geocoder/policy.json +80 -0
- package/packs/census-geocoder/source.md +53 -0
- package/packs/cfpb/endpoints.json +327 -0
- package/packs/cfpb/policy.json +119 -0
- package/packs/cfpb/source.md +96 -0
- package/packs/clinical-trials/endpoints.json +167 -0
- package/packs/clinical-trials/policy.json +84 -0
- package/packs/clinical-trials/source.md +82 -0
- package/packs/cms/endpoints.json +69 -0
- package/packs/cms/policy.json +96 -0
- package/packs/cms/source.md +112 -0
- package/packs/college-scorecard/endpoints.json +253 -0
- package/packs/college-scorecard/policy.json +106 -0
- package/packs/college-scorecard/source.md +92 -0
- package/packs/congress/endpoints.json +1701 -0
- package/packs/congress/policy.json +117 -0
- package/packs/congress/source.md +190 -0
- package/packs/dhs-ohss/endpoints.json +66 -0
- package/packs/dhs-ohss/policy.json +107 -0
- package/packs/dhs-ohss/source.md +120 -0
- package/packs/doj-news/endpoints.json +59 -0
- package/packs/doj-news/policy.json +96 -0
- package/packs/doj-news/source.md +99 -0
- package/packs/dol/endpoints.json +818 -0
- package/packs/dol/policy.json +135 -0
- package/packs/dol/source.md +181 -0
- package/packs/eia/endpoints.json +718 -0
- package/packs/eia/policy.json +112 -0
- package/packs/eia/source.md +112 -0
- package/packs/epa/endpoints.json +292 -0
- package/packs/epa/policy.json +99 -0
- package/packs/epa/source.md +99 -0
- package/packs/epa-aqs/endpoints.json +174 -0
- package/packs/epa-aqs/policy.json +96 -0
- package/packs/epa-aqs/source.md +97 -0
- package/packs/fbi/endpoints.json +1842 -0
- package/packs/fbi/policy.json +118 -0
- package/packs/fbi/source.md +108 -0
- package/packs/fda/endpoints.json +238 -0
- package/packs/fda/policy.json +121 -0
- package/packs/fda/source.md +117 -0
- package/packs/fdic/endpoints.json +417 -0
- package/packs/fdic/policy.json +83 -0
- package/packs/fdic/source.md +92 -0
- package/packs/fec/endpoints.json +708 -0
- package/packs/fec/policy.json +93 -0
- package/packs/fec/source.md +90 -0
- package/packs/fed-board/endpoints.json +89 -0
- package/packs/fed-board/policy.json +136 -0
- package/packs/fed-board/source.md +118 -0
- package/packs/federal-register/endpoints.json +143 -0
- package/packs/federal-register/policy.json +86 -0
- package/packs/federal-register/source.md +98 -0
- package/packs/fema/endpoints.json +646 -0
- package/packs/fema/policy.json +103 -0
- package/packs/fema/source.md +103 -0
- package/packs/fhfa/endpoints.json +105 -0
- package/packs/fhfa/policy.json +109 -0
- package/packs/fhfa/source.md +108 -0
- package/packs/fred/assets/download_fred.ps1 +122 -0
- package/packs/fred/assets/download_fred.sh +123 -0
- package/packs/fred/docs/analysis_patterns.md +124 -0
- package/packs/fred/docs/fred_api.md +94 -0
- package/packs/fred/docs/series.md +188 -0
- package/packs/fred/policy.json +253 -0
- package/packs/fred/source.md +142 -0
- package/packs/fsa/endpoints.json +150 -0
- package/packs/fsa/policy.json +99 -0
- package/packs/fsa/source.md +102 -0
- package/packs/govinfo/endpoints.json +520 -0
- package/packs/govinfo/policy.json +116 -0
- package/packs/govinfo/source.md +241 -0
- package/packs/gsa-calc/endpoints.json +93 -0
- package/packs/gsa-calc/policy.json +97 -0
- package/packs/gsa-calc/source.md +108 -0
- package/packs/hud/endpoints.json +147 -0
- package/packs/hud/policy.json +84 -0
- package/packs/hud/source.md +97 -0
- package/packs/ipums/assets/download_ipums.ps1 +215 -0
- package/packs/ipums/assets/download_ipums.sh +192 -0
- package/packs/ipums/docs/ipums_api.md +67 -0
- package/packs/ipums/docs/report_patterns.md +117 -0
- package/packs/ipums/docs/variables.md +284 -0
- package/packs/ipums/lib/fetch_text.json +112 -0
- package/packs/ipums/policy.json +215 -0
- package/packs/ipums/source.md +169 -0
- package/packs/irs-soi/endpoints.json +114 -0
- package/packs/irs-soi/policy.json +107 -0
- package/packs/irs-soi/source.md +124 -0
- package/packs/medicaid/endpoints.json +412 -0
- package/packs/medicaid/policy.json +108 -0
- package/packs/medicaid/source.md +136 -0
- package/packs/naep/endpoints.json +217 -0
- package/packs/naep/policy.json +100 -0
- package/packs/naep/source.md +120 -0
- package/packs/nhtsa/endpoints.json +132 -0
- package/packs/nhtsa/policy.json +93 -0
- package/packs/nhtsa/source.md +109 -0
- package/packs/nih/endpoints.json +453 -0
- package/packs/nih/policy.json +86 -0
- package/packs/nih/source.md +108 -0
- package/packs/noaa/endpoints.json +432 -0
- package/packs/noaa/policy.json +95 -0
- package/packs/noaa/source.md +88 -0
- package/packs/nrel/endpoints.json +328 -0
- package/packs/nrel/policy.json +96 -0
- package/packs/nrel/source.md +91 -0
- package/packs/nws/endpoints.json +272 -0
- package/packs/nws/policy.json +82 -0
- package/packs/nws/source.md +81 -0
- package/packs/nyfed/endpoints.json +80 -0
- package/packs/nyfed/policy.json +92 -0
- package/packs/nyfed/source.md +89 -0
- package/packs/omb/endpoints.json +460 -0
- package/packs/omb/policy.json +106 -0
- package/packs/omb/source.md +130 -0
- package/packs/open-payments/endpoints.json +298 -0
- package/packs/open-payments/policy.json +101 -0
- package/packs/open-payments/source.md +99 -0
- package/packs/opm/assets/download_opm.ps1 +125 -0
- package/packs/opm/assets/download_opm.sh +85 -0
- package/packs/opm/docs/fields.md +184 -0
- package/packs/opm/docs/opm_api.md +71 -0
- package/packs/opm/docs/report_patterns.md +83 -0
- package/packs/opm/lib/data_dictionary.csv +73 -0
- package/packs/opm/policy.json +142 -0
- package/packs/opm/source.md +137 -0
- package/packs/pbgc/endpoints.json +483 -0
- package/packs/pbgc/policy.json +94 -0
- package/packs/pbgc/source.md +95 -0
- package/packs/philfed/endpoints.json +85 -0
- package/packs/philfed/policy.json +94 -0
- package/packs/philfed/source.md +125 -0
- package/packs/port.json +287 -0
- package/packs/regulations/endpoints.json +288 -0
- package/packs/regulations/policy.json +86 -0
- package/packs/regulations/source.md +84 -0
- package/packs/sec/endpoints.json +237 -0
- package/packs/sec/policy.json +86 -0
- package/packs/sec/source.md +101 -0
- package/packs/senate-lobbying/endpoints.json +554 -0
- package/packs/senate-lobbying/policy.json +99 -0
- package/packs/senate-lobbying/source.md +100 -0
- package/packs/snapqc/endpoints.json +1426 -0
- package/packs/snapqc/policy.json +92 -0
- package/packs/snapqc/source.md +96 -0
- package/packs/treasury/catalog.json +2267 -0
- package/packs/treasury/docs/api_guide.md +245 -0
- package/packs/treasury/docs/datasets.md +56 -0
- package/packs/treasury/docs/report_patterns.md +126 -0
- package/packs/treasury/policy.json +159 -0
- package/packs/treasury/source.md +170 -0
- package/packs/treasury-yields/endpoints.json +226 -0
- package/packs/treasury-yields/policy.json +86 -0
- package/packs/treasury-yields/source.md +80 -0
- package/packs/usaspending/endpoints.json +1316 -0
- package/packs/usaspending/policy.json +109 -0
- package/packs/usaspending/source.md +103 -0
- package/packs/usda-fna/endpoints.json +427 -0
- package/packs/usda-fna/policy.json +133 -0
- package/packs/usda-fna/source.md +217 -0
- package/packs/usda-fooddata/endpoints.json +181 -0
- package/packs/usda-fooddata/policy.json +94 -0
- package/packs/usda-fooddata/source.md +105 -0
- package/packs/usda-nass/endpoints.json +324 -0
- package/packs/usda-nass/policy.json +95 -0
- package/packs/usda-nass/source.md +85 -0
- package/packs/usda-outlook/endpoints.json +86 -0
- package/packs/usda-outlook/policy.json +109 -0
- package/packs/usda-outlook/source.md +144 -0
- package/packs/usgs/endpoints.json +313 -0
- package/packs/usgs/policy.json +95 -0
- package/packs/usgs/source.md +96 -0
- package/packs/usitc/endpoints.json +74 -0
- package/packs/usitc/policy.json +111 -0
- package/packs/usitc/source.md +90 -0
- package/packs/uspto/endpoints.json +141 -0
- package/packs/uspto/policy.json +95 -0
- package/packs/uspto/source.md +101 -0
- package/packs/va/endpoints.json +414 -0
- package/packs/va/policy.json +101 -0
- package/packs/va/source.md +129 -0
- package/packs/world-bank/endpoints.json +122 -0
- package/packs/world-bank/policy.json +101 -0
- package/packs/world-bank/source.md +93 -0
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Write a source's data package: <out>/data/<source>/.
|
|
3
|
+
*
|
|
4
|
+
* <name>.csv one file per request, values exactly as published (missing values left empty)
|
|
5
|
+
* data_dictionary.csv table, column, inferred type, non-missing count, example
|
|
6
|
+
* raw/ responses as received (credential-free), when an adapter passes them
|
|
7
|
+
* request.json spec, fingerprint, route, retrieval time, every request made (the call's trail);
|
|
8
|
+
* "earlier_fetches" keeps the record of an earlier spec whose tables are still there
|
|
9
|
+
* README.md what is here, the citation, notices, terms, redistribution and retention rules
|
|
10
|
+
*/
|
|
11
|
+
import { createHash } from "node:crypto";
|
|
12
|
+
import fs from "node:fs";
|
|
13
|
+
import path from "node:path";
|
|
14
|
+
import * as pol from "./policy.js";
|
|
15
|
+
import { pyStr, stableJson } from "./py.js";
|
|
16
|
+
import { readCsv, writeCsv } from "./tabular.js";
|
|
17
|
+
const LABEL_ONLY = new Set(["description", "label", "labels", "notes", "source_note"]);
|
|
18
|
+
export function dataDir(out, source) {
|
|
19
|
+
return path.join(path.resolve(out), "data", source);
|
|
20
|
+
}
|
|
21
|
+
export function fingerprint(spec) {
|
|
22
|
+
const strip = (x) => {
|
|
23
|
+
if (Array.isArray(x))
|
|
24
|
+
return x.map(strip);
|
|
25
|
+
if (x && typeof x === "object") {
|
|
26
|
+
return Object.fromEntries(Object.keys(x).sort().filter((k) => !LABEL_ONLY.has(k) && !k.startsWith("_")).map((k) => [k, strip(x[k])]));
|
|
27
|
+
}
|
|
28
|
+
return x;
|
|
29
|
+
};
|
|
30
|
+
return createHash("sha256").update(stableJson(strip(spec))).digest("hex").slice(0, 16);
|
|
31
|
+
}
|
|
32
|
+
export function existing(out, source, spec) {
|
|
33
|
+
const rq = path.join(dataDir(out, source), "request.json");
|
|
34
|
+
if (!fs.existsSync(rq))
|
|
35
|
+
return null;
|
|
36
|
+
let saved;
|
|
37
|
+
try {
|
|
38
|
+
saved = JSON.parse(fs.readFileSync(rq, "utf8"));
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
const files = (saved.tables || []).map((t) => path.join(dataDir(out, source), t.file));
|
|
44
|
+
const t = Date.parse(saved.retrieved || "");
|
|
45
|
+
return { path: dataDir(out, source), matches: saved.fingerprint === fingerprint(spec),
|
|
46
|
+
complete: files.length > 0 && files.every((f) => fs.existsSync(f)), retrieved: saved.retrieved || "",
|
|
47
|
+
age_days: Number.isFinite(t) ? Math.floor((Date.now() - t) / 86400000) : null, route: saved.route,
|
|
48
|
+
problems: saved.problems || [] };
|
|
49
|
+
}
|
|
50
|
+
function infer(vals) {
|
|
51
|
+
const v = vals.filter((x) => x !== "" && x !== null && x !== undefined).map(pyStr);
|
|
52
|
+
if (!v.length)
|
|
53
|
+
return "empty";
|
|
54
|
+
if (v.every((x) => x.trim() !== "" && Number.isFinite(Number(x.replaceAll(",", "")))))
|
|
55
|
+
return "number";
|
|
56
|
+
if (v.every((x) => /^(?:1[6-9]|20|21)\d\d-(?:0[1-9]|1[0-2])(?:-\d\d)?(?:[T ]|$)/.test(x)))
|
|
57
|
+
return "date";
|
|
58
|
+
return "text";
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* A table already written to a CSV file (header row first), for microdata too large to hold as rows: an adapter
|
|
62
|
+
* streams the rows to `path` (a temporary file outside the package) and returns this in place of a list of rows;
|
|
63
|
+
* write() moves the file into data/<source>/ and builds the dictionary from `sample` and `nonMissing`.
|
|
64
|
+
*/
|
|
65
|
+
export class FileTable {
|
|
66
|
+
path;
|
|
67
|
+
columns;
|
|
68
|
+
rows;
|
|
69
|
+
sample;
|
|
70
|
+
nonMissing;
|
|
71
|
+
constructor(path, columns, rows, sample = [], nonMissing = {}) {
|
|
72
|
+
this.path = path;
|
|
73
|
+
this.columns = columns;
|
|
74
|
+
this.rows = rows;
|
|
75
|
+
this.sample = sample;
|
|
76
|
+
this.nonMissing = nonMissing;
|
|
77
|
+
}
|
|
78
|
+
get length() {
|
|
79
|
+
return this.rows;
|
|
80
|
+
}
|
|
81
|
+
discard() {
|
|
82
|
+
fs.rmSync(this.path, { force: true });
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
function rowsAndCols(table) {
|
|
86
|
+
if (table instanceof FileTable)
|
|
87
|
+
return [table.sample, table.columns];
|
|
88
|
+
// a JS object lists integer-like keys ("1940", "2024") first, where Python keeps insertion order: a table can
|
|
89
|
+
// carry its column order as `columns` (withColumns); any key it does not name follows, in first-seen order
|
|
90
|
+
const cols = [...(columnsOf(table) || [])];
|
|
91
|
+
const seen = new Set(cols);
|
|
92
|
+
for (const r of table)
|
|
93
|
+
for (const k of Object.keys(r))
|
|
94
|
+
if (!seen.has(k))
|
|
95
|
+
(seen.add(k), cols.push(k));
|
|
96
|
+
return [table, cols];
|
|
97
|
+
}
|
|
98
|
+
const COLUMNS = Symbol("columns");
|
|
99
|
+
/** The rows, marked with the column order to write (Python's dict order, for keys JS would reorder). */
|
|
100
|
+
export function withColumns(rows, columns) {
|
|
101
|
+
Object.defineProperty(rows, COLUMNS, { value: [...columns], enumerable: false, configurable: true });
|
|
102
|
+
return rows;
|
|
103
|
+
}
|
|
104
|
+
export function columnsOf(rows) {
|
|
105
|
+
return Array.isArray(rows) ? rows[COLUMNS] : undefined;
|
|
106
|
+
}
|
|
107
|
+
const sha256 = (file) => createHash("sha256").update(fs.readFileSync(file)).digest("hex");
|
|
108
|
+
const nowIso = () => new Date().toISOString().replace(/\.\d{3}Z$/, "+00:00");
|
|
109
|
+
const pad = (n) => String(n).padStart(2, "0");
|
|
110
|
+
const today = () => {
|
|
111
|
+
const d = new Date();
|
|
112
|
+
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
|
|
113
|
+
};
|
|
114
|
+
export function write(out, pack, tables, spec, o) {
|
|
115
|
+
const P = pack.policy;
|
|
116
|
+
const d = dataDir(out, pack.id);
|
|
117
|
+
fs.mkdirSync(d, { recursive: true });
|
|
118
|
+
const retrieved = nowIso();
|
|
119
|
+
const metaTables = [];
|
|
120
|
+
let dictionary = [];
|
|
121
|
+
const endpoint = Object.fromEntries((spec?.requests || []).map((r) => [r.name, r.endpoint]));
|
|
122
|
+
const shape = {};
|
|
123
|
+
for (const [name, t] of Object.entries(tables)) {
|
|
124
|
+
const [, cols] = rowsAndCols(t);
|
|
125
|
+
if (cols.length)
|
|
126
|
+
shape[String(endpoint[name])] ??= cols;
|
|
127
|
+
}
|
|
128
|
+
for (const [name, t] of Object.entries(tables)) {
|
|
129
|
+
let [rows, cols] = rowsAndCols(t);
|
|
130
|
+
if (!cols.length)
|
|
131
|
+
cols = shape[String(endpoint[name])] || ["no_rows"];
|
|
132
|
+
const file = `${name}.csv`;
|
|
133
|
+
if (t instanceof FileTable) {
|
|
134
|
+
fs.copyFileSync(t.path, path.join(d, file));
|
|
135
|
+
t.discard();
|
|
136
|
+
}
|
|
137
|
+
else
|
|
138
|
+
fs.writeFileSync(path.join(d, file), writeCsv(rows, cols));
|
|
139
|
+
for (const c of cols) {
|
|
140
|
+
const nonmiss = rows.map((r) => r[c]).filter((v) => v !== "" && v !== null && v !== undefined);
|
|
141
|
+
dictionary.push({ table: name, column: c, type: infer(nonmiss.slice(0, 500)),
|
|
142
|
+
non_missing: t instanceof FileTable ? t.nonMissing[c] ?? nonmiss.length : nonmiss.length,
|
|
143
|
+
example: nonmiss.length ? pyStr(nonmiss[0]).slice(0, 80) : "" });
|
|
144
|
+
}
|
|
145
|
+
metaTables.push({ name, file, rows: t instanceof FileTable ? t.rows : rows.length, columns: cols,
|
|
146
|
+
sha256: sha256(path.join(d, file)) });
|
|
147
|
+
}
|
|
148
|
+
const earlier = earlierFetches(d, new Set(metaTables.map((t) => t.file)));
|
|
149
|
+
const kept = new Set(earlier.flatMap((e) => e.tables.map((t) => t.name)));
|
|
150
|
+
const dictPath = path.join(d, "data_dictionary.csv");
|
|
151
|
+
if (kept.size && fs.existsSync(dictPath)) {
|
|
152
|
+
dictionary = dictionary.concat(readCsv(fs.readFileSync(dictPath, "utf8")).filter((r) => kept.has(String(r.table))));
|
|
153
|
+
}
|
|
154
|
+
if (dictionary.length)
|
|
155
|
+
fs.writeFileSync(dictPath, writeCsv(dictionary, ["table", "column", "type", "non_missing", "example"]));
|
|
156
|
+
if (o.raw) {
|
|
157
|
+
fs.mkdirSync(path.join(d, "raw"), { recursive: true });
|
|
158
|
+
for (const [rel, data] of Object.entries(o.raw))
|
|
159
|
+
fs.writeFileSync(path.join(d, "raw", path.basename(rel)), data);
|
|
160
|
+
}
|
|
161
|
+
const req = { source: pack.id, spec, fingerprint: fingerprint(spec), route: o.route, acquired: o.acquired ?? "api",
|
|
162
|
+
retrieved, vintage: o.vintage ?? "", requests: o.requests || [], tables: metaTables, problems: o.problems || [],
|
|
163
|
+
notes: o.notes || [], earlier_fetches: earlier,
|
|
164
|
+
policy: { redistribution: P.redistribution.mode, cache: P.cache.mode } };
|
|
165
|
+
fs.writeFileSync(path.join(d, "request.json"), JSON.stringify(req, (k, v) => (k.startsWith("_") ? undefined : v), 2));
|
|
166
|
+
const cite = pol.citation(P, o.route, { retrieved: today(), vintage: o.vintage ?? "", dataset: spec?.description ?? "" });
|
|
167
|
+
let extra = o.extraReadme ?? "";
|
|
168
|
+
if (earlier.length) {
|
|
169
|
+
const listed = earlier.map((e) => `- ${pol.localStamp(e.retrieved)} (${e.route}): ` + e.tables.map((t) => `\`${t.file}\``).join(", ") +
|
|
170
|
+
"; spec and requests in request.json `earlier_fetches`" + (e.problems || []).map((p) => `\n - **Problem**: ${p}`).join(""));
|
|
171
|
+
extra = [extra, "## Earlier fetches into this folder\n\n" + listed.join("\n")].filter(Boolean).join("\n\n");
|
|
172
|
+
}
|
|
173
|
+
writeReadme(d, pack, metaTables, cite, retrieved, o.acquired ?? "api", o.notes || [], o.problems || [], extra);
|
|
174
|
+
return { data_dir: d, tables: metaTables, retrieved, citation: cite };
|
|
175
|
+
}
|
|
176
|
+
/** The provenance of earlier fetches whose tables are still in the folder and not replaced now. */
|
|
177
|
+
export function earlierFetches(d, newFiles) {
|
|
178
|
+
let old;
|
|
179
|
+
try {
|
|
180
|
+
old = JSON.parse(fs.readFileSync(path.join(d, "request.json"), "utf8"));
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
return [];
|
|
184
|
+
}
|
|
185
|
+
const out = [];
|
|
186
|
+
let files = new Set(newFiles);
|
|
187
|
+
for (const e of [...(old.earlier_fetches || []), old]) {
|
|
188
|
+
const keep = (e.tables || []).filter((t) => !files.has(t.file) && fs.existsSync(path.join(d, t.file || "")));
|
|
189
|
+
if (!keep.length)
|
|
190
|
+
continue;
|
|
191
|
+
const mine = keep.filter((t) => t.name).map((t) => `${t.name}:`);
|
|
192
|
+
const pick = (k) => (e[k] || []).map(String).filter((n) => mine.some((m) => n.startsWith(m)));
|
|
193
|
+
out.push({ spec: e.spec, fingerprint: e.fingerprint, route: e.route, acquired: e.acquired, retrieved: e.retrieved,
|
|
194
|
+
vintage: e.vintage, requests: e.requests, tables: keep, problems: pick("problems"), notes: pick("notes") });
|
|
195
|
+
files = new Set([...files, ...keep.map((t) => t.file)]);
|
|
196
|
+
}
|
|
197
|
+
return out;
|
|
198
|
+
}
|
|
199
|
+
function writeReadme(d, pack, tables, cite, retrieved, acquired, notes, problems, extra) {
|
|
200
|
+
const P = pack.policy;
|
|
201
|
+
const rd = P.redistribution;
|
|
202
|
+
const lines = [`# ${P.title}: downloaded data`, "",
|
|
203
|
+
`Retrieved ${pol.localStamp(retrieved)} by the \`${acquired}\` route. Values are exactly as published; missing values are ` +
|
|
204
|
+
"empty cells. Column types and examples are in `data_dictionary.csv`; every request is in `request.json` (URLs without keys).",
|
|
205
|
+
"", "## Files", "",
|
|
206
|
+
...tables.map((t) => `- \`${t.file}\`: ${t.rows.toLocaleString("en-US")} rows; columns ${t.columns.slice(0, 12).join(", ")}` +
|
|
207
|
+
(t.columns.length > 12 ? " …" : "")),
|
|
208
|
+
"", "## Citation", "", cite, ""];
|
|
209
|
+
if (P.notices?.length)
|
|
210
|
+
lines.push("## Required notices", "", ...P.notices.map((n) => `- ${n}`), "");
|
|
211
|
+
lines.push("## Use and redistribution", "", `Rule: **${rd.mode}**. ${rd.why || ""}`);
|
|
212
|
+
if (rd.mode === "definition_only") {
|
|
213
|
+
lines.push("", "These files must not be shared, attached to a report or put in a replication package. Share the request " +
|
|
214
|
+
"definition and the reproduce scripts instead; anyone rerunning the work downloads the data under their own account.");
|
|
215
|
+
}
|
|
216
|
+
if (P.retention)
|
|
217
|
+
lines.push("", `Retention: ${P.retention.note || ""}`);
|
|
218
|
+
if (P.terms_url)
|
|
219
|
+
lines.push("", `Terms: ${P.terms_url}`);
|
|
220
|
+
if (notes.length)
|
|
221
|
+
lines.push("", "## Notes", "", ...notes.map((n) => `- ${n}`));
|
|
222
|
+
if (problems.length)
|
|
223
|
+
lines.push("", "## Problems", "", ...problems.map((p) => `- ${p}`));
|
|
224
|
+
if (extra)
|
|
225
|
+
lines.push("", extra);
|
|
226
|
+
fs.writeFileSync(path.join(d, "README.md"), lines.join("\n") + "\n");
|
|
227
|
+
}
|
|
228
|
+
//# sourceMappingURL=package.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"package.js","sourceRoot":"","sources":["../../src/core/package.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,KAAK,GAAG,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAEjD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,aAAa,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC;AAEvF,MAAM,UAAU,OAAO,CAAC,GAAW,EAAE,MAAc;IACjD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAa;IACvC,MAAM,KAAK,GAAG,CAAC,CAAM,EAAO,EAAE;QAC5B,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxI,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;IACF,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACzF,CAAC;AAKD,MAAM,UAAU,QAAQ,CAAC,GAAW,EAAE,MAAc,EAAE,IAAU;IAC9D,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,cAAc,CAAC,CAAC;IAC3D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IACpC,IAAI,KAAU,CAAC;IACf,IAAI,CAAC;QACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5F,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAC5C,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,WAAW,KAAK,WAAW,CAAC,IAAI,CAAC;QACnF,QAAQ,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;QAC5G,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK;QACjG,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,KAAK,CAAC,IAAe;IAC5B,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACnF,IAAI,CAAC,CAAC,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IAC9B,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,QAAQ,CAAC;IACvG,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,6DAA6D,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,MAAM,CAAC;IACzG,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,OAAO,SAAS;IACD;IAAqB;IAA0B;IAAqB;IAC9E;IADT,YAAmB,IAAY,EAAS,OAAiB,EAAS,IAAY,EAAS,SAAgB,EAAE,EAChG,aAAqC,EAAE;QAD7B,SAAI,GAAJ,IAAI,CAAQ;QAAS,YAAO,GAAP,OAAO,CAAU;QAAS,SAAI,GAAJ,IAAI,CAAQ;QAAS,WAAM,GAAN,MAAM,CAAY;QAChG,eAAU,GAAV,UAAU,CAA6B;IAAG,CAAC;IACpD,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IACD,OAAO;QACL,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACxC,CAAC;CACF;AAID,SAAS,WAAW,CAAC,KAAY;IAC/B,IAAI,KAAK,YAAY,SAAS;QAAE,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACrE,8GAA8G;IAC9G,2GAA2G;IAC3G,MAAM,IAAI,GAAa,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACrD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACrG,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AACvB,CAAC;AAED,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;AAElC,wGAAwG;AACxG,MAAM,UAAU,WAAW,CAAkB,IAAO,EAAE,OAAiB;IACrE,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;IACrG,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAa;IACrC,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,IAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAClE,CAAC;AAED,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAClG,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;AAC7E,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACtD,MAAM,KAAK,GAAG,GAAG,EAAE;IACjB,MAAM,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;IACrB,OAAO,GAAG,CAAC,CAAC,WAAW,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;AAC3E,CAAC,CAAC;AAKF,MAAM,UAAU,KAAK,CAAC,GAAW,EAAE,IAAU,EAAE,MAA6B,EAAE,IAAU,EAAE,CAGzF;IACC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;IAChC,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC;IAC3B,MAAM,UAAU,GAAgB,EAAE,CAAC;IACnC,IAAI,UAAU,GAAU,EAAE,CAAC;IAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC7F,MAAM,KAAK,GAA6B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC/C,MAAM,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,IAAI,CAAC,MAAM;YAAE,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC;IAC1D,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC/C,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtE,MAAM,IAAI,GAAG,GAAG,IAAI,MAAM,CAAC;QAC3B,IAAI,CAAC,YAAY,SAAS,EAAE,CAAC;YAC3B,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;YAC5C,CAAC,CAAC,OAAO,EAAE,CAAC;QACd,CAAC;;YAAM,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAClE,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC;YAC/F,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC1E,WAAW,EAAE,CAAC,YAAY,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;gBACxF,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,YAAY,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI;YAC9F,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC;IACD,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC;IACrD,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzC,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACtH,CAAC;IACD,IAAI,UAAU,CAAC,MAAM;QAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAC/H,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;QACV,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACvD,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;YAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACnH,CAAC;IACD,MAAM,GAAG,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,KAAK;QAChH,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,EAAE;QAC/G,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE,EAAE,eAAe,EAAE,OAAO;QAC9C,MAAM,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;IAC3E,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACtH,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,IAAI,EAAE,EAAE,CAAC,CAAC;IAC1H,IAAI,KAAK,GAAG,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC;IAChC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACxI,uDAAuD,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACvI,KAAK,GAAG,CAAC,KAAK,EAAE,yCAAyC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9G,CAAC;IACD,WAAW,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;IAC/G,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;AACxE,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,cAAc,CAAC,CAAS,EAAE,QAAqB;IAC7D,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,GAAG,GAAU,EAAE,CAAC;IACtB,IAAI,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QACtD,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAClH,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAS;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;QAC3E,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtH,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS;YAC/G,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC9G,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,WAAW,CAAC,CAAS,EAAE,IAAU,EAAE,MAAmB,EAAE,IAAY,EAAE,SAAiB,EAAE,QAAgB,EAChH,KAAe,EAAE,QAAkB,EAAE,KAAa;IAClD,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,MAAM,EAAE,GAAG,CAAC,CAAC,cAAc,CAAC;IAC5B,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,mBAAmB,EAAE,EAAE;QAChD,aAAa,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,aAAa,QAAQ,gEAAgE;YAC3H,8HAA8H;QAC9H,EAAE,EAAE,UAAU,EAAE,EAAE;QAClB,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAC1H,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACtC,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IACnC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACpG,KAAK,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,IAAI,OAAO,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC;IACrF,IAAI,EAAE,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,0GAA0G;YACvH,qHAAqH,CAAC,CAAC;IAC3H,CAAC;IACD,IAAI,CAAC,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;IACxE,IAAI,CAAC,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;IACzD,IAAI,KAAK,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;IAChF,IAAI,QAAQ,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;IACzF,IAAI,KAAK;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;IACjC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,WAAW,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;AACvE,CAAC"}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import type { Client } from "./http.js";
|
|
2
|
+
import type { Policy } from "./policy.js";
|
|
3
|
+
export type Row = Record<string, unknown>;
|
|
4
|
+
export type Tables = Record<string, Row[] | import("./package.js").FileTable>;
|
|
5
|
+
export type SpecRequest = {
|
|
6
|
+
name: string;
|
|
7
|
+
endpoint?: string;
|
|
8
|
+
dataset?: string;
|
|
9
|
+
params?: Record<string, unknown>;
|
|
10
|
+
max_rows?: number;
|
|
11
|
+
allow_empty?: boolean;
|
|
12
|
+
for_each?: any;
|
|
13
|
+
[k: string]: any;
|
|
14
|
+
};
|
|
15
|
+
export type Spec = {
|
|
16
|
+
description?: string;
|
|
17
|
+
requests: SpecRequest[];
|
|
18
|
+
[k: string]: any;
|
|
19
|
+
};
|
|
20
|
+
/** Hooks a standard pack's adapter may define (the Python adapter.py hook names, in camelCase). */
|
|
21
|
+
export type Adapter = {
|
|
22
|
+
fetch?: (spec: Spec, pack: Pack, route: string) => Promise<[Tables, string[]]>;
|
|
23
|
+
checkSpec?: (spec: Spec, pack: Pack) => string[];
|
|
24
|
+
beforeRequest?: (req: SpecRequest, ep: any, params: Record<string, unknown>) => Record<string, unknown> | void;
|
|
25
|
+
normalize?: (req: SpecRequest, ep: any, rows: Row[]) => Row[];
|
|
26
|
+
requestNote?: (req: SpecRequest, ep: any, params: Record<string, unknown>) => string | null | undefined;
|
|
27
|
+
checkError?: (status: number, body: Buffer) => string | null | undefined;
|
|
28
|
+
manualItems?: (spec: Spec, pack: Pack) => any[];
|
|
29
|
+
/** The full manual items when they need lookups (names, "latest"); the tools prefer it over manualItems. */
|
|
30
|
+
manualItemsAsync?: (spec: Spec, pack: Pack) => Promise<any[]>;
|
|
31
|
+
parseUpload?: (pack: Pack, req: SpecRequest, files: string[]) => Row[] | Promise<Row[]>;
|
|
32
|
+
search?: (query: string, pack: Pack) => any[] | Promise<any[]>;
|
|
33
|
+
ping?: (client: Client) => Promise<{
|
|
34
|
+
auth_ok: boolean;
|
|
35
|
+
kind: string;
|
|
36
|
+
detail: string;
|
|
37
|
+
}>;
|
|
38
|
+
/** Extra one-line request summaries for source_brief (beside endpoints.json's). */
|
|
39
|
+
describe?: (pack: Pack) => {
|
|
40
|
+
name: string;
|
|
41
|
+
summary: string;
|
|
42
|
+
}[];
|
|
43
|
+
keyOnlyNeeds?: (spec: Spec) => string[];
|
|
44
|
+
refusedHosts?: (spec: Spec) => string | null | undefined;
|
|
45
|
+
keylessFiles?: (spec: Spec, pack: Pack) => any[];
|
|
46
|
+
[k: string]: unknown;
|
|
47
|
+
};
|
|
48
|
+
/** What a deep pack's fetch or ingest returns (it writes <workspace>/data/<id>/ itself, usually with package.write). */
|
|
49
|
+
export type DeepResult = {
|
|
50
|
+
data_dir: string;
|
|
51
|
+
tables: {
|
|
52
|
+
name: string;
|
|
53
|
+
file: string;
|
|
54
|
+
rows: number;
|
|
55
|
+
columns: string[];
|
|
56
|
+
}[];
|
|
57
|
+
notes: string[];
|
|
58
|
+
problems?: string[];
|
|
59
|
+
retrieved: string;
|
|
60
|
+
citation: string;
|
|
61
|
+
vintage?: string;
|
|
62
|
+
route: string;
|
|
63
|
+
files?: string[];
|
|
64
|
+
};
|
|
65
|
+
export type DeepContext = {
|
|
66
|
+
pack: Pack;
|
|
67
|
+
spec: any;
|
|
68
|
+
workspace: string;
|
|
69
|
+
force: boolean;
|
|
70
|
+
};
|
|
71
|
+
/**
|
|
72
|
+
* A deep pack (its own spec format and operations; the Python version had its own fetch.py).
|
|
73
|
+
* operations the request index source_brief shows: names and one-line summaries
|
|
74
|
+
* schema full schemas of named operations (request_schema)
|
|
75
|
+
* uses which operations a spec uses (the gate checks that their schemas were read)
|
|
76
|
+
* checkSpec problems with a spec ([] = valid), before any request
|
|
77
|
+
*/
|
|
78
|
+
export type DeepPack = {
|
|
79
|
+
operations: {
|
|
80
|
+
name: string;
|
|
81
|
+
summary: string;
|
|
82
|
+
}[];
|
|
83
|
+
schema: (names: string[], pack: Pack) => any[] | Promise<any[]>;
|
|
84
|
+
uses: (spec: any) => string[];
|
|
85
|
+
specExample?: any;
|
|
86
|
+
search?: (query: string, pack: Pack) => any[] | Promise<any[]>;
|
|
87
|
+
checkSpec?: (spec: any, pack: Pack) => string[];
|
|
88
|
+
fetch: (ctx: DeepContext) => Promise<DeepResult>;
|
|
89
|
+
ingest?: (ctx: DeepContext & {
|
|
90
|
+
files: string[];
|
|
91
|
+
}) => Promise<DeepResult>;
|
|
92
|
+
manualItems?: (spec: any, pack: Pack) => any[];
|
|
93
|
+
keylessFiles?: (spec: any, pack: Pack) => any[];
|
|
94
|
+
};
|
|
95
|
+
export type Pack = {
|
|
96
|
+
id: string;
|
|
97
|
+
dir: string;
|
|
98
|
+
policy: Policy;
|
|
99
|
+
endpoints: any | null;
|
|
100
|
+
adapter: Adapter | null;
|
|
101
|
+
deep: DeepPack | null;
|
|
102
|
+
/** Python code this pack needs that is not ported yet ([] = ready). */
|
|
103
|
+
missing: string[];
|
|
104
|
+
sourceMd: string;
|
|
105
|
+
has: (hook: keyof Adapter) => boolean;
|
|
106
|
+
};
|
|
107
|
+
export declare function isPack(dir: string): boolean;
|
|
108
|
+
/** Every pack folder, enabled or not. */
|
|
109
|
+
export declare function allPackIds(): string[];
|
|
110
|
+
/**
|
|
111
|
+
* Selective loading: the operator can enable only some sources (DATA_ANALYST_SOURCES=fred,treasury, the CLI's
|
|
112
|
+
* --sources) or domains (DATA_ANALYST_DOMAINS=economy,health, --domains). A source outside them is not listed,
|
|
113
|
+
* briefed or fetched.
|
|
114
|
+
*/
|
|
115
|
+
export declare function packEnabled(id: string): boolean;
|
|
116
|
+
/** The enabled packs (see packEnabled). */
|
|
117
|
+
export declare function packIds(): string[];
|
|
118
|
+
/**
|
|
119
|
+
* DATA_ANALYST_HIDE_UNCONFIGURED=1 (--hide-unconfigured): leave out of listings a source that requires a key
|
|
120
|
+
* none is set up for, here and now (inside a signed-in user's call their own keys count).
|
|
121
|
+
*/
|
|
122
|
+
export declare function hiddenForKeys(p: Pack): boolean;
|
|
123
|
+
export declare class UnknownSource extends Error {
|
|
124
|
+
}
|
|
125
|
+
export declare function load(pid: string): Promise<Pack>;
|
|
126
|
+
export declare function allPacks(): Promise<Pack[]>;
|
|
127
|
+
/** Lower-case words without stop words and a plural s; hyphenated or dotted terms also stay whole. */
|
|
128
|
+
export declare function tokens(s: string): string[];
|
|
129
|
+
export type RouteHit = {
|
|
130
|
+
source: string;
|
|
131
|
+
title: string;
|
|
132
|
+
score: number;
|
|
133
|
+
depth: string;
|
|
134
|
+
auth: string;
|
|
135
|
+
matched: string[];
|
|
136
|
+
ready: boolean;
|
|
137
|
+
};
|
|
138
|
+
/** Rank packs for a question by weighted keyword overlap (a hint, not a decision). */
|
|
139
|
+
export declare function route(question: string, limit?: number): Promise<RouteHit[]>;
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Find and load source packs (packs/<id>/: policy.json, source.md, endpoints.json, docs/, assets/).
|
|
3
|
+
*
|
|
4
|
+
* A standard pack is run by core (generic-rest + package) from its endpoints.json, plus an optional adapter
|
|
5
|
+
* (src/adapters/<id>.ts) for what endpoints.json cannot express. A deep pack (src/deep/<id>.ts) has its own
|
|
6
|
+
* operations. packs/port.json lists the packs whose Python version had code beyond endpoints.json; such a pack
|
|
7
|
+
* is "not ported" until its TypeScript code is registered, and the server says so instead of guessing.
|
|
8
|
+
*/
|
|
9
|
+
import fs from "node:fs";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { findAdapter, findDeep } from "../adapters/index.js";
|
|
12
|
+
import { PACKS_DIR } from "./env.js";
|
|
13
|
+
import { findAuthKey } from "./keys.js";
|
|
14
|
+
import * as pol from "./policy.js";
|
|
15
|
+
let portList = null;
|
|
16
|
+
function port() {
|
|
17
|
+
if (!portList) {
|
|
18
|
+
const p = path.join(PACKS_DIR, "port.json");
|
|
19
|
+
portList = fs.existsSync(p) ? JSON.parse(fs.readFileSync(p, "utf8")) : {};
|
|
20
|
+
}
|
|
21
|
+
return portList;
|
|
22
|
+
}
|
|
23
|
+
export function isPack(dir) {
|
|
24
|
+
return fs.existsSync(path.join(dir, "policy.json"));
|
|
25
|
+
}
|
|
26
|
+
/** Every pack folder, enabled or not. */
|
|
27
|
+
export function allPackIds() {
|
|
28
|
+
if (!fs.existsSync(PACKS_DIR))
|
|
29
|
+
return [];
|
|
30
|
+
return fs.readdirSync(PACKS_DIR).filter((d) => isPack(path.join(PACKS_DIR, d))).sort();
|
|
31
|
+
}
|
|
32
|
+
const listSetting = (name) => (process.env[name] || "").split(",").map((x) => x.trim().toLowerCase()).filter(Boolean);
|
|
33
|
+
/**
|
|
34
|
+
* Selective loading: the operator can enable only some sources (DATA_ANALYST_SOURCES=fred,treasury, the CLI's
|
|
35
|
+
* --sources) or domains (DATA_ANALYST_DOMAINS=economy,health, --domains). A source outside them is not listed,
|
|
36
|
+
* briefed or fetched.
|
|
37
|
+
*/
|
|
38
|
+
export function packEnabled(id) {
|
|
39
|
+
const only = listSetting("DATA_ANALYST_SOURCES");
|
|
40
|
+
if (only.length && !only.includes(id))
|
|
41
|
+
return false;
|
|
42
|
+
const domains = listSetting("DATA_ANALYST_DOMAINS");
|
|
43
|
+
if (domains.length) {
|
|
44
|
+
const p = pol.readJson(path.join(PACKS_DIR, id), "policy.json");
|
|
45
|
+
if (!(p?.domains || []).some((d) => domains.includes(String(d).toLowerCase())))
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
/** The enabled packs (see packEnabled). */
|
|
51
|
+
export function packIds() {
|
|
52
|
+
return allPackIds().filter(packEnabled);
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* DATA_ANALYST_HIDE_UNCONFIGURED=1 (--hide-unconfigured): leave out of listings a source that requires a key
|
|
56
|
+
* none is set up for, here and now (inside a signed-in user's call their own keys count).
|
|
57
|
+
*/
|
|
58
|
+
export function hiddenForKeys(p) {
|
|
59
|
+
return /^(1|true|yes)$/i.test(process.env.DATA_ANALYST_HIDE_UNCONFIGURED || "") &&
|
|
60
|
+
p.policy.auth.required === "required" && !findAuthKey(p.policy.auth)[0];
|
|
61
|
+
}
|
|
62
|
+
export class UnknownSource extends Error {
|
|
63
|
+
}
|
|
64
|
+
const cache = new Map();
|
|
65
|
+
export async function load(pid) {
|
|
66
|
+
if (cache.has(pid) && packEnabled(pid))
|
|
67
|
+
return cache.get(pid);
|
|
68
|
+
const dir = path.join(PACKS_DIR, pid);
|
|
69
|
+
if (!/^[a-z0-9-]+$/.test(pid) || !isPack(dir)) {
|
|
70
|
+
throw new UnknownSource(`unknown source ${JSON.stringify(pid)}. Known sources: ${packIds().join(", ")}`);
|
|
71
|
+
}
|
|
72
|
+
if (!packEnabled(pid)) {
|
|
73
|
+
throw new UnknownSource(`source ${JSON.stringify(pid)} is not enabled on this server (enabled: ${packIds().join(", ")})`);
|
|
74
|
+
}
|
|
75
|
+
const policy = pol.load(dir);
|
|
76
|
+
const endpoints = pol.readJson(dir, "endpoints.json");
|
|
77
|
+
const la = findAdapter(pid), ld = findDeep(pid);
|
|
78
|
+
const adapter = la ? await la() : null;
|
|
79
|
+
const deep = ld ? await ld() : null;
|
|
80
|
+
const need = port()[pid]?.python || [];
|
|
81
|
+
const missing = need.filter((f) => (f === "adapter.py" ? !adapter : f === "fetch.py" ? !deep : false));
|
|
82
|
+
const pack = {
|
|
83
|
+
id: pid, dir, policy, endpoints, adapter, deep, missing, sourceMd: path.join(dir, "source.md"),
|
|
84
|
+
has: (hook) => !!adapter && typeof adapter[hook] === "function",
|
|
85
|
+
};
|
|
86
|
+
cache.set(pid, pack);
|
|
87
|
+
return pack;
|
|
88
|
+
}
|
|
89
|
+
export async function allPacks() {
|
|
90
|
+
return Promise.all(packIds().map(load));
|
|
91
|
+
}
|
|
92
|
+
// ---------------------------------------------------------------------- routing a question to packs
|
|
93
|
+
const STOP = new Set(("a an the of in on for to and or by with from what how much many is are was were did does do " +
|
|
94
|
+
"since between over last year years data show chart compare trend latest per has have had than more after only " +
|
|
95
|
+
"those each who which this that its their it be been").split(" "));
|
|
96
|
+
const SAME = { weekly: "week", monthly: "month", hourly: "hour", yearly: "annual",
|
|
97
|
+
annually: "annual", quarterly: "quarter", earn: "earning", earned: "earning", paid: "pay", salarie: "salary",
|
|
98
|
+
employed: "employment" };
|
|
99
|
+
const FIELDS = [["keywords", 3], ["questions", 2], ["domains", 2], ["title", 2], ["agency", 1]];
|
|
100
|
+
/** Lower-case words without stop words and a plural s; hyphenated or dotted terms also stay whole. */
|
|
101
|
+
export function tokens(s) {
|
|
102
|
+
const out = [];
|
|
103
|
+
for (const term of s.toLowerCase().match(/[a-z0-9]+(?:[-.][a-z0-9]+)*/g) || []) {
|
|
104
|
+
const parts = term.match(/[a-z0-9]+/g) || [];
|
|
105
|
+
for (let w of [...(parts.length > 1 ? [term] : []), ...parts]) {
|
|
106
|
+
if (STOP.has(w) || w.length < 2)
|
|
107
|
+
continue;
|
|
108
|
+
if (w.length > 3 && w.endsWith("s") && /^[a-z]+$/.test(w))
|
|
109
|
+
w = w.slice(0, -1);
|
|
110
|
+
out.push(SAME[w] ?? w);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return out;
|
|
114
|
+
}
|
|
115
|
+
function vocab(P) {
|
|
116
|
+
const out = {};
|
|
117
|
+
for (const [f] of FIELDS) {
|
|
118
|
+
const vals = P[f] || [];
|
|
119
|
+
out[f] = (typeof vals === "string" ? [vals] : vals).map((v) => [v, new Set(tokens(v))]);
|
|
120
|
+
}
|
|
121
|
+
return out;
|
|
122
|
+
}
|
|
123
|
+
const subset = (a, b) => [...a].every((x) => b.has(x));
|
|
124
|
+
/** Rank packs for a question by weighted keyword overlap (a hint, not a decision). */
|
|
125
|
+
export async function route(question, limit = 6) {
|
|
126
|
+
const q = new Set(tokens(question));
|
|
127
|
+
const packs = (await allPacks()).filter((p) => !hiddenForKeys(p));
|
|
128
|
+
const V = new Map(packs.map((p) => [p.id, vocab(p.policy)]));
|
|
129
|
+
const df = new Map();
|
|
130
|
+
for (const v of V.values()) {
|
|
131
|
+
const seen = new Set();
|
|
132
|
+
for (const entries of Object.values(v))
|
|
133
|
+
for (const [, toks] of entries)
|
|
134
|
+
toks.forEach((t) => seen.add(t));
|
|
135
|
+
seen.forEach((w) => df.set(w, (df.get(w) || 0) + 1));
|
|
136
|
+
}
|
|
137
|
+
const idf = new Map([...df].map(([w, n]) => [w, Math.log((packs.length + 1) / (n + 0.5))]));
|
|
138
|
+
const ranked = [];
|
|
139
|
+
for (const p of packs) {
|
|
140
|
+
const v = V.get(p.id);
|
|
141
|
+
let score = 0;
|
|
142
|
+
const why = [];
|
|
143
|
+
for (const [field, w] of FIELDS) {
|
|
144
|
+
const all = new Set(v[field].flatMap(([, t]) => [...t]));
|
|
145
|
+
const hit = [...q].filter((t) => all.has(t));
|
|
146
|
+
if (!hit.length)
|
|
147
|
+
continue;
|
|
148
|
+
const whole = new Set(v[field].filter(([, t]) => t.size === 1 || subset(t, q)).flatMap(([, t]) => [...t]));
|
|
149
|
+
score += w * hit.reduce((s, t) => s + (idf.get(t) || 0) * (whole.has(t) ? 1 : /^\d+$/.test(t) ? 0 : 0.5), 0);
|
|
150
|
+
for (const [text, toks] of v[field]) {
|
|
151
|
+
if (toks.size > 1 && subset(toks, q)) {
|
|
152
|
+
score += (w * [...toks].reduce((s, t) => s + (idf.get(t) || 0), 0)) / 2;
|
|
153
|
+
why.unshift(`${field}: ${text}`);
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
const some = [...toks].filter((t) => q.has(t) && !/^\d+$/.test(t));
|
|
157
|
+
if (some.length) {
|
|
158
|
+
why.push(toks.size === 1 ? `${field}: ${text}` :
|
|
159
|
+
`${field}: ${[...toks].filter((t) => q.has(t)).sort().join(" ")} (one word of ${JSON.stringify(text)})`);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (q.has(p.id) || question.toLowerCase().includes(p.id.replaceAll("-", " ")))
|
|
165
|
+
score += 5;
|
|
166
|
+
if (score) {
|
|
167
|
+
ranked.push({ source: p.id, title: p.policy.title, score: Math.round(score * 10) / 10, depth: p.policy.depth,
|
|
168
|
+
auth: p.policy.auth.required, matched: why.slice(0, 4), ready: !p.missing.length });
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
return ranked.sort((a, b) => b.score - a.score).slice(0, limit);
|
|
172
|
+
}
|
|
173
|
+
//# sourceMappingURL=packs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"packs.js","sourceRoot":"","sources":["../../src/core/packs.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,KAAK,GAAG,MAAM,aAAa,CAAC;AAkEnC,IAAI,QAAQ,GAAgD,IAAI,CAAC;AACjE,SAAS,IAAI;IACX,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC5C,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5E,CAAC;IACD,OAAO,QAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,GAAW;IAChC,OAAO,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,UAAU;IACxB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IACzC,OAAO,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACzF,CAAC;AAED,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,EAAE,CACnC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAE1F;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,MAAM,IAAI,GAAG,WAAW,CAAC,sBAAsB,CAAC,CAAC;IACjD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAAE,OAAO,KAAK,CAAC;IACpD,MAAM,OAAO,GAAG,WAAW,CAAC,sBAAsB,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,aAAa,CAAC,CAAC;QACrE,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;IACvG,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,2CAA2C;AAC3C,MAAM,UAAU,OAAO;IACrB,OAAO,UAAU,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;AAC1C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,CAAO;IACnC,OAAO,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,EAAE,CAAC;QAC7E,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,KAAK,UAAU,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,OAAO,aAAc,SAAQ,KAAK;CAAG;AAE3C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAgB,CAAC;AAEtC,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,GAAW;IACpC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;IAC/D,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACtC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,aAAa,CAAC,kBAAkB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,oBAAoB,OAAO,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC3G,CAAC;IACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,aAAa,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,4CAA4C,OAAO,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5H,CAAC;IACD,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7B,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;IACtD,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACvC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACvG,MAAM,IAAI,GAAS;QACjB,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC;QAC9F,GAAG,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,UAAU;KAChE,CAAC;IACF,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACrB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ;IAC5B,OAAO,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;AAC1C,CAAC;AAED,qGAAqG;AACrG,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,8FAA8F;IAClH,gHAAgH;IAChH,qDAAqD,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;AACrE,MAAM,IAAI,GAA2B,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ;IACvG,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ;IAC5G,QAAQ,EAAE,YAAY,EAAE,CAAC;AAC3B,MAAM,MAAM,GAAuB,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;AAEpH,sGAAsG;AACtG,MAAM,UAAU,MAAM,CAAC,CAAS;IAC9B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,8BAA8B,CAAC,IAAI,EAAE,EAAE,CAAC;QAC/E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAC7C,KAAK,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC;YAC9D,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YAC1C,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9E,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAGD,SAAS,KAAK,CAAC,CAAM;IACnB,MAAM,GAAG,GAAU,EAAE,CAAC;IACtB,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACxB,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,MAAM,GAAG,CAAC,CAAc,EAAE,CAAc,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AAKjF,sFAAsF;AACtF,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,QAAgB,EAAE,KAAK,GAAG,CAAC;IACrD,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YAAE,KAAK,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,OAAO;gBAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACzG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5F,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAE,CAAC;QACvB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACzD,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7C,IAAI,CAAC,GAAG,CAAC,MAAM;gBAAE,SAAS;YAC1B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3G,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC7G,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpC,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;oBACrC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oBACxE,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,KAAK,IAAI,EAAE,CAAC,CAAC;gBACnC,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBACnE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;wBAChB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,IAAI,EAAE,CAAC,CAAC;4BAC9C,GAAG,KAAK,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,iBAAiB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAC7G,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAAE,KAAK,IAAI,CAAC,CAAC;QAC1F,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK;gBAC1G,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAClE,CAAC"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import type { Auth } from "./keys.js";
|
|
2
|
+
export type Policy = {
|
|
3
|
+
id: string;
|
|
4
|
+
title: string;
|
|
5
|
+
agency: string;
|
|
6
|
+
depth: "deep" | "standard";
|
|
7
|
+
domains: string[];
|
|
8
|
+
keywords: string[];
|
|
9
|
+
questions: string[];
|
|
10
|
+
homepage: string;
|
|
11
|
+
terms_url?: string;
|
|
12
|
+
auth: Auth & {
|
|
13
|
+
required: "required" | "optional" | "none";
|
|
14
|
+
placement: "query" | "header" | "body" | "none";
|
|
15
|
+
};
|
|
16
|
+
limits: {
|
|
17
|
+
rps: number;
|
|
18
|
+
burst: number;
|
|
19
|
+
max_rows?: number;
|
|
20
|
+
errors_per_min?: number;
|
|
21
|
+
bytes_per_min?: number;
|
|
22
|
+
[k: string]: unknown;
|
|
23
|
+
};
|
|
24
|
+
routes: Record<string, any>;
|
|
25
|
+
cache: {
|
|
26
|
+
mode: "none" | "run_only" | "persistent";
|
|
27
|
+
ttl_hours?: number;
|
|
28
|
+
why?: string;
|
|
29
|
+
};
|
|
30
|
+
redistribution: {
|
|
31
|
+
mode: "allowed" | "aggregates_only" | "definition_only";
|
|
32
|
+
microdata_globs: string[];
|
|
33
|
+
package_include: string[];
|
|
34
|
+
withhold: any;
|
|
35
|
+
why?: string;
|
|
36
|
+
};
|
|
37
|
+
notices: string[];
|
|
38
|
+
audit_hooks: any[];
|
|
39
|
+
clarify: string[];
|
|
40
|
+
formats: string[];
|
|
41
|
+
retention: any;
|
|
42
|
+
citation: string;
|
|
43
|
+
aliases?: string[];
|
|
44
|
+
use_instead?: {
|
|
45
|
+
question: string;
|
|
46
|
+
source: string;
|
|
47
|
+
terms: string[];
|
|
48
|
+
why: string;
|
|
49
|
+
}[];
|
|
50
|
+
[k: string]: any;
|
|
51
|
+
};
|
|
52
|
+
export declare const REDIST_ORDER: Record<string, number>;
|
|
53
|
+
export declare class PolicyError extends Error {
|
|
54
|
+
}
|
|
55
|
+
export declare function readJson<T = any>(dir: string, name: string): T | null;
|
|
56
|
+
export declare function load(packDir: string): Policy;
|
|
57
|
+
/** What a pack says it covers, lower case: policy fields plus source.md and docs/*.md without "Not here" bullets. */
|
|
58
|
+
export declare function coverageText(packDir: string): string | null;
|
|
59
|
+
/** Problems with a policy (empty = valid). */
|
|
60
|
+
export declare function validate(p: any, packDir?: string): string[];
|
|
61
|
+
export declare function checkCache(p: Policy, want: "none" | "run_only" | "persistent"): void;
|
|
62
|
+
export declare function strictest(policies: Policy[]): string;
|
|
63
|
+
export declare function notices(policies: Policy[]): string[];
|
|
64
|
+
/** The user's calendar date of a retrieval time; a plain date is kept. */
|
|
65
|
+
export declare function localDay(stamp: unknown): string;
|
|
66
|
+
/** "2026-10-27 23:10 local time (2026-10-28 03:10 UTC)"; a plain date is kept. */
|
|
67
|
+
export declare function localStamp(stamp: unknown): string;
|
|
68
|
+
/** The source's citation text; data that came by the keyless_file route cite that file when the pack says how. */
|
|
69
|
+
export declare function citation(p: Policy, route?: string, vals?: Record<string, string>): string;
|