argo-search 1.0.1 → 2.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -21
- package/README.en.md +822 -0
- package/README.es.md +864 -0
- package/README.ja.md +864 -0
- package/README.ko.md +864 -0
- package/README.md +867 -320
- package/SKILL.md +135 -0
- package/assets/readme/hero.svg +121 -0
- package/assets/readme/made-with-beautify.svg +10 -0
- package/assets/readme/proof-routes.svg +73 -0
- package/assets/readme/settings-keys.png +0 -0
- package/assets/readme/settings-tools.png +0 -0
- package/assets/readme/why-better.svg +104 -0
- package/assets/readme/workflow.svg +83 -0
- package/backends/domain_profiles.json +2308 -96
- package/backends/engine_registry.yaml +3314 -503
- package/backends/query_synonyms_cn.json +35 -0
- package/backends/quota_profiles.json +2409 -161
- package/backends/source_types_cn.json +83 -0
- package/backends/text_lexicon.json +44 -0
- package/bin/argo +703 -0
- package/bin/argo.js +71 -39
- package/config.yaml +4976 -363
- package/cordis.patch.yml +21 -0
- package/data/countries.json +40 -0
- package/data/lang_matrix/lang_capability.json +842 -0
- package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
- package/data/regions_cn.json +1 -0
- package/docs/ADDING_NEW_ENGINE.md +184 -0
- package/docs/ARGO_INTRO.md +124 -0
- package/docs/DATA_SOURCE_GAPS.md +81 -0
- package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
- package/docs/ENGINE_CATALOG.md +564 -0
- package/docs/MCP_SETUP.md +106 -0
- package/docs/PLATFORM_PATHS.md +111 -0
- package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
- package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
- package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
- package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
- package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
- package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
- package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
- package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
- package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
- package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
- package/docs/SEARCH_ARCHIVE.md +115 -0
- package/docs/assets/architecture.svg +104 -0
- package/docs/assets/hero.svg +98 -0
- package/docs/engines/bangumi.md +27 -0
- package/docs/engines/biorxiv.md +27 -0
- package/docs/engines/clawhub.md +27 -0
- package/docs/engines/crt_sh.md +27 -0
- package/docs/engines/datacite.md +27 -0
- package/docs/engines/deps_dev.md +27 -0
- package/docs/engines/douban_movie.md +27 -0
- package/docs/engines/endoflife.md +27 -0
- package/docs/engines/flk_law.md +27 -0
- package/docs/engines/fxtwitter.md +27 -0
- package/docs/engines/google_news.md +27 -0
- package/docs/engines/gov_regulations.md +27 -0
- package/docs/engines/guqian.md +27 -0
- package/docs/engines/hackernews.md +27 -0
- package/docs/engines/iplant.md +27 -0
- package/docs/engines/keenable.md +27 -0
- package/docs/engines/met_museum.md +27 -0
- package/docs/engines/nasa_images.md +27 -0
- package/docs/engines/netease_music.md +27 -0
- package/docs/engines/npm.md +27 -0
- package/docs/engines/nvd.md +27 -0
- package/docs/engines/openreview.md +27 -0
- package/docs/engines/openstd.md +27 -0
- package/docs/engines/parallel_free.md +27 -0
- package/docs/engines/people_daily.md +27 -0
- package/docs/engines/realtime_index.md +27 -0
- package/docs/engines/sec_edgar.md +27 -0
- package/docs/engines/seltz.md +27 -0
- package/docs/engines/skillsmp.md +27 -0
- package/docs/engines/sspai.md +27 -0
- package/docs/engines/std_samr.md +27 -0
- package/docs/engines/train.md +27 -0
- package/docs/engines/un_comtrade.md +27 -0
- package/docs/engines/wikisource.md +27 -0
- package/docs/engines/zdic.md +27 -0
- package/docs/engines/zenodo.md +27 -0
- package/docs/engines/zhihu_hot_app.md +27 -0
- package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
- package/engines/README.md +10 -0
- package/engines/_template_cli.yaml +39 -0
- package/engines/_template_http.yaml +37 -0
- package/engines/plugins/_template_plugin.py +50 -0
- package/engines/specs/_cisa_kev.yaml +54 -0
- package/engines/specs/_osv.yaml +63 -0
- package/engines/specs/aljazeera.yaml +43 -0
- package/engines/specs/antara.yaml +43 -0
- package/engines/specs/artic.yaml +20 -0
- package/engines/specs/aviation_weather.yaml +39 -0
- package/engines/specs/bangumi.yaml +21 -0
- package/engines/specs/biorxiv.yaml +24 -0
- package/engines/specs/carbon_intensity.yaml +32 -0
- package/engines/specs/cinii.yaml +44 -0
- package/engines/specs/clawhub.yaml +49 -0
- package/engines/specs/cleveland.yaml +19 -0
- package/engines/specs/cn_ai_news.yaml +43 -0
- package/engines/specs/core.yaml +35 -0
- package/engines/specs/crt_sh.yaml +38 -0
- package/engines/specs/ct24.yaml +43 -0
- package/engines/specs/datacite.yaml +27 -0
- package/engines/specs/datagov_uk.yaml +41 -0
- package/engines/specs/ddgs_images.yaml +45 -0
- package/engines/specs/deepseek_search.yaml +74 -0
- package/engines/specs/deezer.yaml +20 -0
- package/engines/specs/deps_dev.yaml +21 -0
- package/engines/specs/digup.yaml +36 -0
- package/engines/specs/doi.yaml +39 -0
- package/engines/specs/douban_movie.yaml +22 -0
- package/engines/specs/dw_news.yaml +47 -0
- package/engines/specs/egov_law.yaml +21 -0
- package/engines/specs/elpais.yaml +43 -0
- package/engines/specs/endoflife.yaml +21 -0
- package/engines/specs/energy_charts.yaml +21 -0
- package/engines/specs/factcheck_org.yaml +52 -0
- package/engines/specs/faz.yaml +43 -0
- package/engines/specs/federal_register.yaml +57 -0
- package/engines/specs/firecrawl.yaml +53 -0
- package/engines/specs/flk_law.yaml +24 -0
- package/engines/specs/folha.yaml +43 -0
- package/engines/specs/france24.yaml +44 -0
- package/engines/specs/full_fact.yaml +47 -0
- package/engines/specs/fxtwitter.yaml +31 -0
- package/engines/specs/gbfs_nyc.yaml +32 -0
- package/engines/specs/gdacs.yaml +21 -0
- package/engines/specs/gdelt.yaml +64 -0
- package/engines/specs/google_news.yaml +22 -0
- package/engines/specs/gov_regulations.yaml +38 -0
- package/engines/specs/guardian_rss.yaml +59 -0
- package/engines/specs/hurriyet.yaml +43 -0
- package/engines/specs/iplant.yaml +25 -0
- package/engines/specs/jikan.yaml +24 -0
- package/engines/specs/jolpica.yaml +20 -0
- package/engines/specs/k10plus.yaml +20 -0
- package/engines/specs/keenable.yaml +44 -0
- package/engines/specs/lefigaro.yaml +43 -0
- package/engines/specs/listenbrainz.yaml +19 -0
- package/engines/specs/matichon.yaml +43 -0
- package/engines/specs/met_museum.yaml +21 -0
- package/engines/specs/nasa_images.yaml +45 -0
- package/engines/specs/netease_music.yaml +38 -0
- package/engines/specs/nhk.yaml +43 -0
- package/engines/specs/nhtsa_vpic.yaml +20 -0
- package/engines/specs/noaa_swpc.yaml +20 -0
- package/engines/specs/nvd.yaml +38 -0
- package/engines/specs/obis.yaml +20 -0
- package/engines/specs/opencitations.yaml +52 -0
- package/engines/specs/openf1.yaml +21 -0
- package/engines/specs/openfoodfacts.yaml +45 -0
- package/engines/specs/openligadb.yaml +20 -0
- package/engines/specs/openreview.yaml +37 -0
- package/engines/specs/openstd.yaml +21 -0
- package/engines/specs/people_daily.yaml +22 -0
- package/engines/specs/realtime_index.yaml +34 -0
- package/engines/specs/ror.yaml +20 -0
- package/engines/specs/satnogs.yaml +20 -0
- package/engines/specs/sec_edgar.yaml +46 -0
- package/engines/specs/skillsmp.yaml +50 -0
- package/engines/specs/sogou_weixin_browser.yaml +30 -0
- package/engines/specs/soilgrids.yaml +25 -0
- package/engines/specs/sspai.yaml +36 -0
- package/engines/specs/stackexchange.yaml +46 -0
- package/engines/specs/std_samr.yaml +21 -0
- package/engines/specs/tass.yaml +43 -0
- package/engines/specs/tle_mirror.yaml +21 -0
- package/engines/specs/train.yaml +37 -0
- package/engines/specs/tvmaze.yaml +20 -0
- package/engines/specs/twitter_syndication.yaml +31 -0
- package/engines/specs/un_comtrade.yaml +23 -0
- package/engines/specs/unpaywall.yaml +54 -0
- package/engines/specs/vnpress.yaml +43 -0
- package/engines/specs/weather.yaml +32 -0
- package/engines/specs/who_don.yaml +21 -0
- package/engines/specs/who_gho.yaml +20 -0
- package/engines/specs/wikimedia_commons.yaml +34 -0
- package/engines/specs/wikisource.yaml +22 -0
- package/engines/specs/worms.yaml +19 -0
- package/engines/specs/x_trending.yaml +34 -0
- package/engines/specs/yna.yaml +43 -0
- package/engines/specs/ynet.yaml +43 -0
- package/engines/specs/zbmath.yaml +40 -0
- package/engines/specs/zdic.yaml +22 -0
- package/engines/specs/zenodo.yaml +27 -0
- package/engines/specs/zhihu_hot_app.yaml +35 -0
- package/mcp/clients.yaml +74 -0
- package/package.json +51 -12
- package/packages/dsh-plugin/README.md +134 -0
- package/packages/dsh-plugin/cordis.patch.yml +48 -0
- package/packages/dsh-plugin/dsh/client.js +1244 -0
- package/packages/dsh-plugin/dsh/index.js +2290 -0
- package/packages/dsh-plugin/dsh/native-tools.mjs +743 -0
- package/packages/dsh-plugin/package.json +38 -0
- package/packages/dsh-plugin/test/client.test.js +232 -0
- package/packages/dsh-plugin/test/config.test.js +30 -0
- package/packages/dsh-plugin/test/index.test.js +1338 -0
- package/references/query-templates.md +96 -0
- package/references/query-templates.yaml +118 -0
- package/scripts/ab_eval.py +219 -0
- package/scripts/ab_eval_p0p1.py +266 -0
- package/scripts/adaptive.py +271 -70
- package/scripts/answer.py +154 -0
- package/scripts/archive_run.py +591 -0
- package/scripts/argo_engine_registry.py +172 -0
- package/scripts/argo_paths.py +728 -0
- package/scripts/article.py +136 -0
- package/scripts/arxiv_source.py +201 -0
- package/scripts/badcase_ledger.py +252 -0
- package/scripts/batch_probe.py +241 -0
- package/scripts/benchmark.py +19 -0
- package/scripts/bounded_run.py +144 -0
- package/scripts/browser_auth.py +323 -0
- package/scripts/burstiness_detector.py +147 -0
- package/scripts/cache.py +1078 -67
- package/scripts/cache_guard.py +271 -0
- package/scripts/cache_key_vdom.py +56 -0
- package/scripts/cache_similarity.py +170 -0
- package/scripts/caliber_isolation.py +195 -0
- package/scripts/calibrate_lowquality.py +210 -0
- package/scripts/candidate_envelope.py +366 -0
- package/scripts/chrome_cdp.py +871 -0
- package/scripts/circuit_breaker.py +389 -0
- package/scripts/citations.py +334 -0
- package/scripts/cite_cli.py +59 -0
- package/scripts/cjk_tokens.py +95 -0
- package/scripts/clarify.py +47 -24
- package/scripts/cli_io.py +240 -0
- package/scripts/config.py +683 -45
- package/scripts/config_cache.py +402 -0
- package/scripts/conn_pool.py +220 -0
- package/scripts/content_security.py +685 -0
- package/scripts/content_signals.py +730 -0
- package/scripts/crawl.py +74 -35
- package/scripts/daemon.py +866 -0
- package/scripts/digup_bridge.py +214 -0
- package/scripts/engine_admission.py +366 -0
- package/scripts/engine_dispatch.py +1019 -0
- package/scripts/engine_env.py +534 -0
- package/scripts/engine_failure.py +411 -0
- package/scripts/engine_families.py +732 -0
- package/scripts/engine_policy.py +218 -0
- package/scripts/engine_requires.py +190 -0
- package/scripts/engine_status.py +322 -0
- package/scripts/engine_validate.py +638 -0
- package/scripts/engines.py +778 -294
- package/scripts/engines_base.py +1521 -0
- package/scripts/engines_builders.py +298 -0
- package/scripts/engines_builders_batch10.py +159 -0
- package/scripts/engines_builders_batch11.py +174 -0
- package/scripts/engines_builders_batch12.py +103 -0
- package/scripts/engines_builders_batch13.py +340 -0
- package/scripts/engines_builders_batch9.py +1468 -0
- package/scripts/engines_builders_cn.py +2283 -0
- package/scripts/engines_builders_data.py +2923 -0
- package/scripts/engines_builders_data_macro.py +775 -0
- package/scripts/engines_builders_feeds.py +255 -0
- package/scripts/engines_builders_intl.py +1239 -0
- package/scripts/engines_builders_search.py +440 -0
- package/scripts/engines_builders_tech.py +1360 -0
- package/scripts/evidence.py +588 -162
- package/scripts/evidence_loop.py +592 -0
- package/scripts/evidence_tier.py +298 -0
- package/scripts/except_sets.py +25 -0
- package/scripts/extract.py +55 -16
- package/scripts/fact_align.py +207 -0
- package/scripts/fact_crosscheck.py +230 -0
- package/scripts/fetch.py +25 -12
- package/scripts/fetch_browser.py +87 -0
- package/scripts/fetch_jsrun.py +122 -0
- package/scripts/fetch_quality.py +225 -0
- package/scripts/fetch_render_tinyfish.py +117 -0
- package/scripts/fetch_v3.py +1779 -0
- package/scripts/focus_extract.py +350 -0
- package/scripts/fulltext_store.py +169 -0
- package/scripts/fxtwitter_api.py +325 -0
- package/scripts/gen_engine_catalog.py +364 -0
- package/scripts/gen_native_tools.py +91 -0
- package/scripts/gen_routing_snapshot.py +184 -0
- package/scripts/health_check.py +140 -402
- package/scripts/health_probe.py +23 -8
- package/scripts/hot_state.py +134 -0
- package/scripts/http_client.py +1053 -0
- package/scripts/image_ops.py +430 -0
- package/scripts/image_vision/vision_probe +0 -0
- package/scripts/image_vision/vision_probe.swift +126 -0
- package/scripts/install.ps1 +148 -0
- package/scripts/install.sh +206 -0
- package/scripts/job.py +1186 -0
- package/scripts/jump_resolver.py +235 -0
- package/scripts/lang_capability.py +150 -0
- package/scripts/lang_detect.py +289 -0
- package/scripts/lang_matrix_probe.py +329 -0
- package/scripts/lang_pref.py +311 -0
- package/scripts/link_source.py +340 -0
- package/scripts/local_engines.py +321 -0
- package/scripts/local_image.py +970 -0
- package/scripts/local_seek.py +366 -0
- package/scripts/macro_countries.py +82 -0
- package/scripts/matrix_search_eval.py +1105 -0
- package/scripts/mcp_diag.py +8 -3
- package/scripts/mcp_handlers.py +1074 -0
- package/scripts/mcp_handlers_surface.py +157 -0
- package/scripts/mcp_launch.sh +122 -0
- package/scripts/mcp_server.py +118 -417
- package/scripts/mcp_setup.py +407 -0
- package/scripts/mcp_tools.py +330 -0
- package/scripts/mcp_transport.py +191 -0
- package/scripts/min_version_scan.py +335 -0
- package/scripts/net_proxy.py +258 -0
- package/scripts/network_aware.py +152 -0
- package/scripts/paper.py +274 -0
- package/scripts/pdf_extract.py +518 -0
- package/scripts/phrase_filter.py +66 -0
- package/scripts/plan.py +422 -0
- package/scripts/postprocess_signals.py +150 -0
- package/scripts/pre-commit +48 -0
- package/scripts/query_enhance.py +157 -0
- package/scripts/query_operators.py +196 -0
- package/scripts/query_rewriter.py +116 -7
- package/scripts/query_signals.py +300 -0
- package/scripts/query_templates.py +167 -0
- package/scripts/query_understanding.py +411 -0
- package/scripts/query_variants.py +298 -0
- package/scripts/quota.py +488 -76
- package/scripts/rank_signals.py +775 -0
- package/scripts/ranking_eval.py +319 -0
- package/scripts/readability_extract.py +433 -0
- package/scripts/recompute.py +415 -0
- package/scripts/recovery.py +616 -0
- package/scripts/recovery_l0.py +160 -0
- package/scripts/redskill/redskill_engine.py +332 -0
- package/scripts/regression_p0p1.py +318 -0
- package/scripts/replay_eval.py +640 -0
- package/scripts/rerank_tfidf.py +225 -0
- package/scripts/research.py +591 -403
- package/scripts/research_cli.py +361 -0
- package/scripts/research_dossier.py +492 -0
- package/scripts/research_expand.py +146 -0
- package/scripts/research_gates.py +272 -0
- package/scripts/research_report.py +209 -0
- package/scripts/research_strategy.py +37 -0
- package/scripts/research_work_packages.py +195 -0
- package/scripts/result_lang.py +311 -0
- package/scripts/robots_guard.py +259 -0
- package/scripts/route.py +921 -254
- package/scripts/route_bangs.py +40 -0
- package/scripts/route_cache.py +225 -0
- package/scripts/route_combo.py +910 -0
- package/scripts/route_domains.py +328 -0
- package/scripts/route_lang.py +605 -0
- package/scripts/route_log.py +63 -0
- package/scripts/route_policy.py +250 -0
- package/scripts/rrf_classic.py +66 -0
- package/scripts/screenshot.py +73 -0
- package/scripts/search.py +897 -405
- package/scripts/search_attribution.py +88 -0
- package/scripts/search_benchmark.py +407 -0
- package/scripts/search_cli.py +627 -0
- package/scripts/search_entry.py +389 -0
- package/scripts/search_output.py +606 -0
- package/scripts/search_pipeline.py +1077 -0
- package/scripts/search_rank.py +991 -0
- package/scripts/search_types.py +27 -12
- package/scripts/seek_locator.py +68 -0
- package/scripts/semantic_evidence.py +215 -0
- package/scripts/serp_guard.py +228 -0
- package/scripts/serp_kill_log.py +51 -0
- package/scripts/single_flight.py +120 -0
- package/scripts/social_engines/__init__.py +10 -0
- package/scripts/social_engines/bilibili_engine.py +131 -0
- package/scripts/social_engines/hackernews_engine.py +113 -0
- package/scripts/social_engines/reddit_engine.py +172 -0
- package/scripts/social_engines/twitter_engine.py +397 -0
- package/scripts/social_engines/v2ex_engine.py +101 -0
- package/scripts/social_engines/weibo_engine.py +128 -0
- package/scripts/social_engines/xiaohongshu_engine.py +137 -0
- package/scripts/social_engines/zhihu_engine.py +139 -0
- package/scripts/social_research.py +201 -0
- package/scripts/sogou_weixin_browser.py +318 -0
- package/scripts/source_ledger.py +249 -0
- package/scripts/stage_timing.py +58 -0
- package/scripts/stats_cli.py +95 -0
- package/scripts/straggler_policy.py +60 -0
- package/scripts/stylometry_detector.py +231 -0
- package/scripts/sync_backends.py +353 -0
- package/scripts/sync_installs.py +29 -0
- package/scripts/template_cli.py +127 -0
- package/scripts/text_lexicon.py +72 -0
- package/scripts/tfidf_router.py +109 -37
- package/scripts/time_utils.py +233 -0
- package/scripts/topic_research_profiles.py +643 -0
- package/scripts/train.py +737 -0
- package/scripts/tweet.py +596 -0
- package/scripts/url_canon.py +217 -0
- package/scripts/url_optimize.py +56 -0
- package/scripts/url_safety.py +236 -0
- package/scripts/usage_log.py +116 -0
- package/scripts/usage_log_clean.py +113 -0
- package/scripts/v2ex_nodes.py +311 -0
- package/scripts/variant_recall.py +188 -0
- package/scripts/watch.py +205 -0
- package/scripts/wechat_article.py +258 -0
- package/scripts/wide_research.py +402 -0
- package/scripts/wx.py +349 -0
- package/scripts/x_article_fetch.py +141 -0
- package/scripts/x_search.py +234 -0
- package/scripts/x_trending.py +189 -0
- package/scripts/yaml_load.py +67 -0
- package/sub-skills/ego-search/SKILL.md +253 -0
- package/sub-skills/ego-search/references/browser-runtime.md +43 -0
- package/sub-skills/ego-search/references/install.md +67 -0
- package/sub-skills/ego-search/references/original-ego-upgrade.md +51 -0
- package/sub-skills/ego-search/scripts/ego_search.py +800 -0
- package/sub-skills/ego-search/scripts/merge.py +226 -0
- package/sub-skills/ego-search/scripts/quality.py +64 -0
- package/sub-skills/ego-search/scripts/runtime.py +172 -0
- package/sub-skills/ego-search/scripts/safety.py +109 -0
- package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
- package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
- package/sub-skills/ego-search/tests/test_query_normalize.py +68 -0
- package/sub-skills/js-run/SKILL.md +67 -0
- package/sub-skills/js-run/scripts/jsrun.py +112 -0
- package/sub-skills/js-run/scripts/shim.js +188 -0
- package/sub-skills/js-run/tests/fixtures/challenge_like.js +27 -0
- package/sub-skills/js-run/tests/test_jsrun.py +165 -0
- package/sub-skills/local-search/SKILL.md +50 -53
- package/sub-skills/local-search/bm25.py +162 -0
- package/sub-skills/local-search/config.yaml +152 -55
- package/sub-skills/local-search/engine_registry.py +91 -26
- package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
- package/sub-skills/local-search/local_search_adapter.py +13 -2
- package/sub-skills/local-search/parse_maps.yaml +40 -42
- package/sub-skills/local-search/search_v3.py +1182 -65
- package/sub-skills/local-search/smart_router.py +47 -7
- package/sub-skills/local-seek/SKILL.md +110 -0
- package/sub-skills/local-seek/config/domains.yaml +20 -0
- package/sub-skills/local-seek/references/auto-behaviors.md +24 -0
- package/sub-skills/local-seek/references/fd-mdfind-recipes.md +62 -0
- package/sub-skills/local-seek/references/git-integration.md +54 -0
- package/sub-skills/local-seek/references/remote-handoff.md +46 -0
- package/sub-skills/local-seek/references/rg-recipes.md +96 -0
- package/sub-skills/local-seek/references/strategies.md +52 -0
- package/sub-skills/local-seek/references/structural-search.md +59 -0
- package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
- package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
- package/sub-skills/local-seek/scripts/seek.py +1465 -0
- package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
- package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
- package/sub-skills/local-seek/scripts/seek_rank.py +193 -0
- package/sub-skills/local-seek/tests/test_content_rank.py +222 -0
- package/sub-skills/local-seek/tests/test_determinism_20261006.py +145 -0
- package/sub-skills/local-seek/tests/test_noise_tier.py +101 -0
- package/sub-skills/local-seek/tests/test_pcre2_probe.py +136 -0
- package/sub-skills/local-seek/tests/test_rank_pinyin.py +77 -0
- package/sub-skills/local-seek/tests/test_regression_20261003.py +169 -0
- package/sub-skills/local-seek/tests/test_regression_20261006.py +146 -0
- package/sub-skills/local-seek/tests/test_seek_degradation.py +111 -0
- package/sub-skills/local-seek/tests/test_seek_fts.py +313 -0
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""gen_engine_catalog.py — 生成搜索源使用文档(docs/ENGINE_CATALOG.md)。
|
|
3
|
+
|
|
4
|
+
为什么要生成而不是手写:引擎清单、密钥要求、可用状态、费用档位分散在
|
|
5
|
+
config.yaml / engines/specs/*.yaml / quota_profiles.json / engine_env 四处,
|
|
6
|
+
手写文档必然与事实脱节(本仓 2026-09-12 就出现过「文档说 12 个 MCP 工具、
|
|
7
|
+
实际 14 个」「文档说 150+ 引擎、分不清收录与可用」)。这里的做法是:
|
|
8
|
+
文档从**运行时同一批函数**取事实,`--check` 比对磁盘上的文档,
|
|
9
|
+
`tests/test_engine_catalog.py` 把 `--check` 挂进检查——文档改不动也漂不掉。
|
|
10
|
+
|
|
11
|
+
用法:
|
|
12
|
+
python3 scripts/gen_engine_catalog.py # 生成/覆盖文档
|
|
13
|
+
python3 scripts/gen_engine_catalog.py --check # 只校验,过期退出码 1
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import argparse
|
|
19
|
+
import json
|
|
20
|
+
import sys
|
|
21
|
+
from collections import defaultdict
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
SKILL_DIR = Path(__file__).resolve().parent.parent
|
|
26
|
+
SCRIPTS_DIR = SKILL_DIR / "scripts"
|
|
27
|
+
DOC_PATH = SKILL_DIR / "docs" / "ENGINE_CATALOG.md"
|
|
28
|
+
if str(SCRIPTS_DIR) not in sys.path:
|
|
29
|
+
sys.path.insert(0, str(SCRIPTS_DIR))
|
|
30
|
+
|
|
31
|
+
# 文档只用「声明事实」分档,不用运行时状态(密钥是否已配、是否被熔断/封锁
|
|
32
|
+
# 都是本机瞬时状态):文档要进仓库、要被别的机器生成复核,一旦写进
|
|
33
|
+
# 「本机此刻缺哪些密钥」,换个环境生成就与磁盘不符,检查随机变红。
|
|
34
|
+
# 运行时状态交给 `argo search --list-engines --detail` 现场查。
|
|
35
|
+
DOC_STATUS_KEY = "需自备密钥"
|
|
36
|
+
DOC_STATUS_DEP = "需装后端工具"
|
|
37
|
+
DOC_STATUS_DISABLED = "已停用"
|
|
38
|
+
DOC_STATUS_EXPLICIT = "显式专用"
|
|
39
|
+
DOC_STATUS_USABLE = "可直接用"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _doc_status(row: dict[str, Any]) -> str:
|
|
43
|
+
if not row["enabled"]:
|
|
44
|
+
return DOC_STATUS_DISABLED
|
|
45
|
+
parts = []
|
|
46
|
+
if row["required_env"]:
|
|
47
|
+
parts.append(DOC_STATUS_KEY)
|
|
48
|
+
if row["requires"]:
|
|
49
|
+
parts.append("需装后端工具" if not parts else "后端工具")
|
|
50
|
+
if row.get("explicit_only"):
|
|
51
|
+
parts.append(DOC_STATUS_EXPLICIT)
|
|
52
|
+
return " + ".join(parts) if parts else DOC_STATUS_USABLE
|
|
53
|
+
# 费用档位 → 人话
|
|
54
|
+
COST_LABEL = {
|
|
55
|
+
"free": "免费",
|
|
56
|
+
"low": "低价计费",
|
|
57
|
+
"api": "按调用计费",
|
|
58
|
+
"paid": "付费",
|
|
59
|
+
}
|
|
60
|
+
# 能力族 → 文档分组标题
|
|
61
|
+
FAMILY_LABEL = {
|
|
62
|
+
"web_general": "全网搜索",
|
|
63
|
+
"academic": "学术文献",
|
|
64
|
+
"code": "代码 / 包 / 文档",
|
|
65
|
+
"finance_market": "行情 / 资金",
|
|
66
|
+
"finance_macro": "宏观数据",
|
|
67
|
+
"news_flash": "快讯 / 电报",
|
|
68
|
+
"social": "社区 UGC",
|
|
69
|
+
"hot_trending": "热榜",
|
|
70
|
+
"world_news": "本地新闻流",
|
|
71
|
+
"knowledge": "百科 / 实体",
|
|
72
|
+
"science_chem": "化学 / 药学",
|
|
73
|
+
"science_bio": "生物 / 蛋白",
|
|
74
|
+
"science_geo": "地球 / 空间",
|
|
75
|
+
"legal": "法律判例",
|
|
76
|
+
"media_book": "媒体 / 图书",
|
|
77
|
+
"sports": "体育",
|
|
78
|
+
"archive": "归档 / 历史",
|
|
79
|
+
"structured_card": "垂直结构化卡",
|
|
80
|
+
"personal_data": "个人数据",
|
|
81
|
+
"misc_vertical": "其他垂直",
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _collect() -> dict[str, Any]:
|
|
86
|
+
"""从运行时取事实(与 CLI/MCP 同一批函数,不另建计算方式)。"""
|
|
87
|
+
from config import load_config, get_domains
|
|
88
|
+
from engine_families import family_of
|
|
89
|
+
from engine_policy import GENERAL_FREE_FALLBACK, get_engine_tier
|
|
90
|
+
from engine_status import list_engines_detail
|
|
91
|
+
import research as _research
|
|
92
|
+
|
|
93
|
+
cfg = load_config()
|
|
94
|
+
specs = cfg.get("engines", {})
|
|
95
|
+
rows = list_engines_detail()
|
|
96
|
+
quota = json.loads((SKILL_DIR / "backends" / "quota_profiles.json")
|
|
97
|
+
.read_text(encoding="utf-8"))
|
|
98
|
+
|
|
99
|
+
# 分发路径:本引擎被哪些域/清单引用(文档要能回答「什么时候会用到它」)
|
|
100
|
+
dispatch: dict[str, list[str]] = defaultdict(list)
|
|
101
|
+
for d in get_domains(cfg):
|
|
102
|
+
for e in d.get("engines_combo") or []:
|
|
103
|
+
dispatch[e].append(f"域 {d['name']}")
|
|
104
|
+
for e in GENERAL_FREE_FALLBACK:
|
|
105
|
+
dispatch[e].append("通用兜底链")
|
|
106
|
+
for e in (set(_research._RESEARCH_EN_BOOSTS)
|
|
107
|
+
| set(_research._RESEARCH_ACADEMIC_BOOSTS)
|
|
108
|
+
| set(_research._RESEARCH_JA_KO_BOOSTS)):
|
|
109
|
+
dispatch[e].append("深度研究 boost")
|
|
110
|
+
# 第 5 条路径:TF-IDF 语义画像(domain_profiles 里 documents 非空即参与语义路由)
|
|
111
|
+
try:
|
|
112
|
+
dp = json.loads((SKILL_DIR / "backends" / "domain_profiles.json")
|
|
113
|
+
.read_text(encoding="utf-8"))
|
|
114
|
+
for e, v in dp.items():
|
|
115
|
+
if isinstance(v, dict) and (v.get("documents") or []):
|
|
116
|
+
dispatch[e].append("语义画像命中")
|
|
117
|
+
except Exception:
|
|
118
|
+
pass
|
|
119
|
+
for r in rows:
|
|
120
|
+
spec = specs.get(r["engine_id"]) or {}
|
|
121
|
+
q = quota.get(r["engine_id"]) or {}
|
|
122
|
+
r["family"] = family_of(r["engine_id"], spec)
|
|
123
|
+
r["family_label"] = FAMILY_LABEL.get(r["family"], r["family"])
|
|
124
|
+
r["tier"] = get_engine_tier(r["engine_id"], spec)
|
|
125
|
+
r["cost_label"] = COST_LABEL.get(q.get("cost_tier", "free"), q.get("cost_tier", "免费"))
|
|
126
|
+
r["limit"] = q.get("limit")
|
|
127
|
+
r["period"] = q.get("period")
|
|
128
|
+
r["enabled"] = bool(spec.get("enabled", True))
|
|
129
|
+
# required_env / requires 是「声明要求」,与机器上是否已配无关
|
|
130
|
+
r["required_env"] = list(r.get("required_env") or [])
|
|
131
|
+
r["requires"] = list(r.get("requires") or [])
|
|
132
|
+
r["dispatch"] = sorted(set(dispatch.get(r["engine_id"], [])))
|
|
133
|
+
# desc 取引擎声明自述(engine_status 的 detail 不带 desc)
|
|
134
|
+
r["desc"] = (spec.get("desc") or spec.get("label") or "").strip()
|
|
135
|
+
r["doc_status"] = _doc_status(r)
|
|
136
|
+
return {"rows": rows, "specs": specs, "cfg": cfg,
|
|
137
|
+
"domains": get_domains(cfg)}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _fmt_quota(row: dict[str, Any]) -> str:
|
|
141
|
+
limit, period = row.get("limit"), row.get("period")
|
|
142
|
+
if not limit:
|
|
143
|
+
return "不限"
|
|
144
|
+
unit = {"second": "秒", "minute": "分", "hour": "小时",
|
|
145
|
+
"day": "天", "month": "月"}.get(str(period), str(period))
|
|
146
|
+
return f"{limit}/{unit}"
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _where(row: dict[str, Any]) -> str:
|
|
150
|
+
"""这源什么时候会被用到(分发路径)。"""
|
|
151
|
+
if row["dispatch"]:
|
|
152
|
+
return "、".join(row["dispatch"])
|
|
153
|
+
if row.get("explicit_only"):
|
|
154
|
+
return "显式调用(--engine)"
|
|
155
|
+
if row["engine_id"].startswith("local_"):
|
|
156
|
+
return "经 local_search 展开"
|
|
157
|
+
if not row["enabled"]:
|
|
158
|
+
return "已停用"
|
|
159
|
+
return "—"
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _table(rows: list[dict[str, Any]]) -> list[str]:
|
|
163
|
+
out = ["| 引擎 | 状态 | 费用 | 频率上限 | 需自备密钥 | 什么时候用到 | 说明 |",
|
|
164
|
+
"|---|---|---|---|---|---|---|"]
|
|
165
|
+
for r in rows:
|
|
166
|
+
env = "、".join(r["required_env"]) if r["required_env"] else "—"
|
|
167
|
+
desc = (r.get("desc") or "").replace("|", "/").strip() or "—"
|
|
168
|
+
out.append(
|
|
169
|
+
f"| `{r['engine_id']}` | {r['doc_status']} "
|
|
170
|
+
f"| {r['cost_label']} | {_fmt_quota(r)} | {env} | {_where(r)} | {desc} |")
|
|
171
|
+
return out
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def render() -> str:
|
|
175
|
+
d = _collect()
|
|
176
|
+
rows = d["rows"]
|
|
177
|
+
disabled = [r for r in rows if not r["enabled"]]
|
|
178
|
+
keyed = [r for r in rows if r["enabled"] and r["required_env"]]
|
|
179
|
+
needs_dep = [r for r in rows if r["enabled"] and r["requires"]]
|
|
180
|
+
usable = [r for r in rows if r["enabled"] and not r["required_env"]
|
|
181
|
+
and not r["requires"]]
|
|
182
|
+
cost_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
183
|
+
for r in rows:
|
|
184
|
+
cost_groups[r["cost_label"]].append(r)
|
|
185
|
+
explicit = [r for r in rows if r.get("explicit_only")]
|
|
186
|
+
by_family: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
187
|
+
for r in columns_sorted(rows):
|
|
188
|
+
by_family[r["family_label"]].append(r)
|
|
189
|
+
|
|
190
|
+
d_domains = d["domains"]
|
|
191
|
+
L: list[str] = []
|
|
192
|
+
L.append("# Argo 搜索源使用文档(自动生成)")
|
|
193
|
+
L.append("")
|
|
194
|
+
L.append("> 本文件由 `scripts/gen_engine_catalog.py` 生成,**勿手改**。")
|
|
195
|
+
L.append("> 改引擎声明后重新生成:`python3 scripts/gen_engine_catalog.py`;")
|
|
196
|
+
L.append("> 过期会被 `tests/test_engine_catalog.py` 拦下。")
|
|
197
|
+
L.append("")
|
|
198
|
+
L.append("## 一、总量与口径")
|
|
199
|
+
L.append("")
|
|
200
|
+
L.append(f"- **收录 {len(rows)} 个源**(config.yaml + `engines/specs/*.yaml` 声明合并后的总数)")
|
|
201
|
+
L.append(f"- **开箱可用 {len(usable)} 个**:不需要你配任何密钥或装额外工具,"
|
|
202
|
+
"自动路由就会用上")
|
|
203
|
+
L.append(f"- **需自备密钥 {len(keyed)} 个**:{_names(keyed)}"
|
|
204
|
+
"(没配也不影响搜索,路由会跳过)")
|
|
205
|
+
L.append(f"- **需装后端工具 {len(needs_dep)} 个**:{_names(needs_dep)}"
|
|
206
|
+
"(装好并登录后即可用)")
|
|
207
|
+
L.append(f"- **已停用 {len(disabled)} 个**:{_names(disabled)}")
|
|
208
|
+
L.append(f"- **显式专用 {len(explicit)} 个**:{_names(explicit)}"
|
|
209
|
+
"(设计上不进自动路由,按 `--engine` 或交接提示调用)")
|
|
210
|
+
L.append("")
|
|
211
|
+
L.append("自己核一遍(口径不同,别混用):")
|
|
212
|
+
L.append("")
|
|
213
|
+
L.append("```bash")
|
|
214
|
+
L.append("argo search --list-engines | wc -l # 有实现的源(可实例化)")
|
|
215
|
+
L.append("argo search --list-engines --detail | wc -l # 收录总数(含已停用)")
|
|
216
|
+
L.append("argo search --list-engines --routable-only | wc -l # 本机此刻真能路由的")
|
|
217
|
+
L.append("argo search --list-engines --detail # 逐源状态/密钥/依赖")
|
|
218
|
+
L.append("```")
|
|
219
|
+
L.append("")
|
|
220
|
+
L.append("本文件的分档是**声明口径**(照引擎自己的声明算,换台机器也一样);"
|
|
221
|
+
"最后一行是**本机口径**——它把已配好密钥、已装好后端工具的源也算进来,"
|
|
222
|
+
"再扣掉当前被熔断/封锁的源。两个数字不相等是正常的;"
|
|
223
|
+
"想知道本机此刻到底哪些就绪,看 `--detail` 里的 `missing_env`(空 = 已配)。")
|
|
224
|
+
L.append("")
|
|
225
|
+
L.append("## 二、费用与密钥:哪些白用、哪些要钱")
|
|
226
|
+
L.append("")
|
|
227
|
+
billable = [(r["engine_id"], r["cost_label"], bool(r["dispatch"]), r["required_env"])
|
|
228
|
+
for r in rows if r["cost_label"] != "免费"]
|
|
229
|
+
free_rows = cost_groups.get("免费") or []
|
|
230
|
+
free_keyed = sorted(r["engine_id"] for r in free_rows if r["required_env"])
|
|
231
|
+
L.append(f"- **免费档 {len(free_rows)} 个**(含已停用):无需密钥,或只需免费注册的密钥"
|
|
232
|
+
+ (f";其中 {len(free_keyed)} 个要自备密钥(免费额度):"
|
|
233
|
+
f"{'、'.join(free_keyed)}" if free_keyed else ""))
|
|
234
|
+
L.append(f"- **计费档 {len(billable)} 个**(下表逐个列出,"
|
|
235
|
+
"档位取自各源自己的 `cost_tier` 声明)")
|
|
236
|
+
L.append("")
|
|
237
|
+
L.append("| 引擎 | 档位 | 是否进自动路由 | 需自备密钥 |")
|
|
238
|
+
L.append("|---|---|---|---|")
|
|
239
|
+
for name, label, on_path, env in sorted(billable, key=lambda x: (x[1], x[0])):
|
|
240
|
+
L.append(f"| `{name}` | {label} | {'是' if on_path else '否,仅显式调用'} "
|
|
241
|
+
f"| {'、'.join(env) if env else '—'} |")
|
|
242
|
+
L.append("")
|
|
243
|
+
paid_on_path = [n for n, lab, on, _ in billable if lab == "付费" and on]
|
|
244
|
+
L.append("**结论**:"
|
|
245
|
+
+ ("付费档(paid)目前**没有**接进自动路由,只有显式 `--engine` 才会用到;"
|
|
246
|
+
if not paid_on_path else
|
|
247
|
+
f"付费档里有 {len(paid_on_path)} 个进了自动路由({', '.join(paid_on_path)}),"
|
|
248
|
+
"用之前先确认额度;")
|
|
249
|
+
+ "低价/按量计费的源有 "
|
|
250
|
+
f"{len([1 for _, lab, on, _ in billable if lab in ('低价计费', '按调用计费') and on])} "
|
|
251
|
+
"个在自动路由路径上,多数带免费额度或已配密钥。"
|
|
252
|
+
"额度记在本地配额表(`backends/quota_profiles.json` 的 limit / period,"
|
|
253
|
+
"用量存在本机状态库),用尽后该源在语义路由里被降权;"
|
|
254
|
+
"本地表统计的是 argo 自己的调用,若同一密钥还被别的工具用,"
|
|
255
|
+
"实际额度以服务商侧为准。想彻底避开计费源,用 `--engine` 显式指定免费源,"
|
|
256
|
+
"或走 `--mode budget`。")
|
|
257
|
+
L.append("")
|
|
258
|
+
L.append("## 三、特别能力(不是普通网页搜索)")
|
|
259
|
+
L.append("")
|
|
260
|
+
L.append("| 能力 | 怎么用 | 说明 |")
|
|
261
|
+
L.append("|---|---|---|")
|
|
262
|
+
L.append("| 垂直结构化卡 | 直接问「北京到上海高铁」「今天油价」「黄金价格」"
|
|
263
|
+
"「2026 年历」「上海车牌摇号」等 | 火车票 / 油价 / 贵金属 / 万年历 / 星座 / 手机 / 汽车 / 挂号"
|
|
264
|
+
"等有标准答案的问题,直接给答案而不是一堆链接 |")
|
|
265
|
+
L.append("| 抽取型取证 | `argo search \"<推文URL或ID>\" --engine twitter_syndication` | "
|
|
266
|
+
"按 URL 取单条推文,免登录、零密钥;关键词查询会诚实返回空(该通道没有搜索端点) |")
|
|
267
|
+
L.append("| 学术与数据集 | 问「XX 论文」走学术域(arXiv/OpenAlex/Crossref/EuropePMC/DBLP/"
|
|
268
|
+
"Semantic Scholar);问「数据集/开放数据」走数据集域(DataCite/Zenodo) | "
|
|
269
|
+
"论文与数据集分开走,互不串味 |")
|
|
270
|
+
L.append("| 美股申报原文 | 问「苹果 10-K」「SEC filing」「招股书」 | "
|
|
271
|
+
"SEC EDGAR 官方全文检索(免密钥),直出申报文件 |")
|
|
272
|
+
L.append("| 深度研究 | `argo research \"问题\"` | 问题分解 → 多源采集 → 综合报告;"
|
|
273
|
+
"注意它不套娃,worker 里不会再用研究工具 |")
|
|
274
|
+
L.append("| 证据核验 | `argo evidence \"query\"`(或管道喂搜索结果) | "
|
|
275
|
+
"对结果打分、标注事实/推断/未知 |")
|
|
276
|
+
L.append("| 公众号全文 | `argo article \"<链接>\"` | 标题 / 正文 / 图片全量抽取 |")
|
|
277
|
+
L.append("| 招聘聚合 | `argo job \"岗位\"` | BOSS / 猎聘 / 智联 / 前程无忧 / 597 / 今日招聘 |")
|
|
278
|
+
L.append("| 抓取与爬站 | `argo fetch \"<url>\"` / `argo crawl \"<url>\"` / `argo extract \"<url>\"` | "
|
|
279
|
+
"四级降级抓取(md 变体 → HTTP → TLS 指纹 → 浏览器);crawl 走 sitemap/BFS;extract 取表格/Meta/JSON-LD |")
|
|
280
|
+
L.append("| 批量预检 | `argo preflight \"url1\" \"url2\" …` | 开工前判定哪些能拿、哪些要登录、哪些已死,"
|
|
281
|
+
"三档结论 go / go_with_skips / stop |")
|
|
282
|
+
L.append("| 本地文件搜索 | `argo search \"词\" --include-local` | "
|
|
283
|
+
"并入本机文件/笔记命中(source=local_files,不参与融合评分) |")
|
|
284
|
+
L.append("| 可复算 | MCP `argo_recompute` / `--allow-recompute` | "
|
|
285
|
+
"数值重算,默认关闭,需要显式授权 |")
|
|
286
|
+
L.append("| 网页截图 / PDF | MCP `argo_screenshot` / `argo_pdf` | 长尾取证能力,默认不挂载 MCP 时用不到 |")
|
|
287
|
+
L.append("")
|
|
288
|
+
L.append("## 四、默认关闭的能力怎么打开")
|
|
289
|
+
L.append("")
|
|
290
|
+
L.append("- **MCP 19 工具面**(与 CLI 能力面对齐):默认关(工具定义会常驻注入上下文)。DSH 用户在 profile patch "
|
|
291
|
+
"里取消 `mcp-argo` 段注释后重启;其他客户端见 README「MCP 接入」。")
|
|
292
|
+
L.append("- **原生工具**:默认只注册 `argo_search` / `argo_fetch`;"
|
|
293
|
+
"`nativeTools` 配置可按需放开全部 18 个(`argo_research` 除外)。")
|
|
294
|
+
L.append("- **需密钥的源**:把密钥写进 `~/.config/argo/env`(600 权限)或环境变量,"
|
|
295
|
+
"下表中「需自备密钥」一列即变量名。")
|
|
296
|
+
L.append("- **显式专用源**:不进自动路由,用 `argo search \"词\" --engine <引擎名>` 调用。")
|
|
297
|
+
L.append("")
|
|
298
|
+
L.append("## 五、逐源清单(按能力族)")
|
|
299
|
+
L.append("")
|
|
300
|
+
L.append("状态含义:**可直接用** = 自动路由会用上;**需自配密钥 / 需装后端工具** = 配好后即可用;"
|
|
301
|
+
"**被上游封锁** = 源站当前拒绝;**已停用** = 配置层面关闭。")
|
|
302
|
+
L.append("")
|
|
303
|
+
for fam in sorted(by_family, key=lambda f: (-len(by_family[f]), f)):
|
|
304
|
+
group = by_family[fam]
|
|
305
|
+
L.append(f"### {fam}({len(group)})")
|
|
306
|
+
L.append("")
|
|
307
|
+
L.extend(_table(group))
|
|
308
|
+
L.append("")
|
|
309
|
+
L.append("## 六、分发域清单(自动生成)")
|
|
310
|
+
L.append("")
|
|
311
|
+
L.append(f"共 {len(d_domains)} 个业务域;"
|
|
312
|
+
"命中即按域内的组合取源(多意图时按配置顺序取,窄意图域排在宽泛域之前)。"
|
|
313
|
+
"自己看某个域为什么选这些源,看 `config.yaml` 的 `domains` 段。")
|
|
314
|
+
L.append("")
|
|
315
|
+
L.append("| 域 | 主源 | 组合 |")
|
|
316
|
+
L.append("|---|---|---|")
|
|
317
|
+
for dom in d_domains:
|
|
318
|
+
combo = "、".join(f"`{x}`" for x in (dom.get("engines_combo") or []))
|
|
319
|
+
L.append(f"| `{dom['name']}` | `{dom.get('primary') or '—'}` | {combo or '—'} |")
|
|
320
|
+
L.append("")
|
|
321
|
+
L.append("## 七、怎么自己查当前状态")
|
|
322
|
+
L.append("")
|
|
323
|
+
L.append("```bash")
|
|
324
|
+
L.append(f"argo search --list-engines --detail | python3 -m json.tool | less # 全部 {len(rows)} 个源的详情")
|
|
325
|
+
L.append("argo search --list-engines --detail --routable-only # 只看现在能用的")
|
|
326
|
+
L.append("python3 scripts/matrix_search_eval.py --offline # 可达性门:有没有死源")
|
|
327
|
+
L.append("python3 scripts/engine_validate.py --engine <名> --stage all # 单个源的健康+质量双阶段体检")
|
|
328
|
+
L.append("```")
|
|
329
|
+
L.append("")
|
|
330
|
+
return "\n".join(L) + "\n"
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def columns_sorted(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
334
|
+
return sorted(rows, key=lambda r: (0 if r["doc_status"] == DOC_STATUS_USABLE else 1,
|
|
335
|
+
r["engine_id"]))
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _names(rows: list[dict[str, Any]]) -> str:
|
|
339
|
+
return "、".join(f"`{r['engine_id']}`" for r in rows) if rows else "无"
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def main() -> int:
|
|
343
|
+
ap = argparse.ArgumentParser(description="生成搜索源使用文档")
|
|
344
|
+
ap.add_argument("--check", action="store_true", help="只校验文档是否过期")
|
|
345
|
+
args = ap.parse_args()
|
|
346
|
+
text = render()
|
|
347
|
+
if args.check:
|
|
348
|
+
if not DOC_PATH.exists():
|
|
349
|
+
print(f"❌ 文档不存在:{DOC_PATH}", file=sys.stderr)
|
|
350
|
+
return 1
|
|
351
|
+
if DOC_PATH.read_text(encoding="utf-8") != text:
|
|
352
|
+
print("❌ 搜索源文档已过期,重新生成:python3 scripts/gen_engine_catalog.py",
|
|
353
|
+
file=sys.stderr)
|
|
354
|
+
return 1
|
|
355
|
+
print(f"✅ 搜索源文档与当前引擎声明一致({len(_collect()['rows'])} 个源)")
|
|
356
|
+
return 0
|
|
357
|
+
DOC_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
358
|
+
DOC_PATH.write_text(text, encoding="utf-8")
|
|
359
|
+
print(f"已生成 {DOC_PATH.relative_to(SKILL_DIR)}({len(_collect()['rows'])} 个源)")
|
|
360
|
+
return 0
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
if __name__ == "__main__":
|
|
364
|
+
sys.exit(main())
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""gen_native_tools.py — 从 schema 唯一来源生成 DSH 原生工具规格表。
|
|
3
|
+
|
|
4
|
+
schema 来源是 scripts/mcp_tools.py(模型可见 inputSchema 只在该文件维护)。
|
|
5
|
+
本脚本把它翻译为 packages/dsh-plugin/dsh/native-tools.mjs(纯 ESM 常量,
|
|
6
|
+
Node 18 可直接 import),使 DSH 原生一等工具(nativeTools 配置)无需手写
|
|
7
|
+
第二份 schema——此前 index.js 手写 argo_search/argo_fetch 两个规格,与
|
|
8
|
+
Python 侧已经出现措辞/参数漂移空间。
|
|
9
|
+
|
|
10
|
+
生成的规格覆盖除 argo_research 外的全部工具:research 是分钟级调度,
|
|
11
|
+
CLI 单发默认 60s 超时会拦腰杀研究,只应经 MCP 或 wide_research 入口使用。
|
|
12
|
+
|
|
13
|
+
漂移检查:tests/test_native_tools_sync.py 重算本脚本输出并与仓库内文件
|
|
14
|
+
逐字节比对——改 mcp_tools.py 后必须重新生成,否则测试红。
|
|
15
|
+
|
|
16
|
+
用法:
|
|
17
|
+
python3 scripts/gen_native_tools.py # 写文件
|
|
18
|
+
python3 scripts/gen_native_tools.py --check # 只校验不写(CI/测试用)
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import sys
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from cli_io import dumps_pretty
|
|
26
|
+
|
|
27
|
+
SCRIPT_DIR = Path(__file__).resolve().parent
|
|
28
|
+
sys.path.insert(0, str(SCRIPT_DIR))
|
|
29
|
+
|
|
30
|
+
from mcp_tools import TOOLS # noqa: E402
|
|
31
|
+
|
|
32
|
+
OUT_PATH = SCRIPT_DIR.parent / "packages" / "dsh-plugin" / "dsh" / "native-tools.mjs"
|
|
33
|
+
|
|
34
|
+
# CLI 单发形态排除项(见模块 docstring)
|
|
35
|
+
_EXCLUDED = {"argo_research"}
|
|
36
|
+
|
|
37
|
+
HEADER = """\
|
|
38
|
+
// 本文件由 scripts/gen_native_tools.py 从 schema 唯一真源
|
|
39
|
+
// scripts/mcp_tools.py 生成 —— 勿手改;改 schema 后重新生成,
|
|
40
|
+
// tests/test_native_tools_sync.py 负责漂移门禁。
|
|
41
|
+
//
|
|
42
|
+
// 说明:
|
|
43
|
+
// - description / parameters 与 MCP 模型可见 schema 完全一致
|
|
44
|
+
// - allowed = parameters.properties 键序(原生侧 pickArgs 白名单)
|
|
45
|
+
// - 不含 argo_research:分钟级编排不适合 CLI 单发(默认超时会中断),
|
|
46
|
+
// 走 MCP 形态或 wide_research 入口
|
|
47
|
+
//
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def build_native_tools() -> dict:
|
|
52
|
+
out: dict[str, dict] = {}
|
|
53
|
+
for tool in TOOLS:
|
|
54
|
+
name = tool["name"]
|
|
55
|
+
if name in _EXCLUDED:
|
|
56
|
+
continue
|
|
57
|
+
schema = tool["inputSchema"]
|
|
58
|
+
out[name] = {
|
|
59
|
+
"description": tool["description"],
|
|
60
|
+
"parameters": {
|
|
61
|
+
**schema,
|
|
62
|
+
"additionalProperties": False,
|
|
63
|
+
},
|
|
64
|
+
"allowed": list(schema.get("properties", {}).keys()),
|
|
65
|
+
}
|
|
66
|
+
return out
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def render(native_tools: dict) -> str:
|
|
70
|
+
body = dumps_pretty(native_tools)
|
|
71
|
+
return HEADER + "export const NATIVE_TOOLS = " + body + ";\n"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def main() -> int:
|
|
75
|
+
native_tools = build_native_tools()
|
|
76
|
+
rendered = render(native_tools)
|
|
77
|
+
if "--check" in sys.argv:
|
|
78
|
+
current = OUT_PATH.read_text(encoding="utf-8") if OUT_PATH.exists() else ""
|
|
79
|
+
if current != rendered:
|
|
80
|
+
print(f"FAIL: {OUT_PATH.name} 与 mcp_tools.py 漂移——请运行 "
|
|
81
|
+
f"python3 scripts/gen_native_tools.py 重新生成", file=sys.stderr)
|
|
82
|
+
return 1
|
|
83
|
+
print(f"OK: {len(native_tools)} 个原生工具规格与 mcp_tools.py 一致")
|
|
84
|
+
return 0
|
|
85
|
+
OUT_PATH.write_text(rendered, encoding="utf-8")
|
|
86
|
+
print(f"已生成 {OUT_PATH}({len(native_tools)} 个工具)")
|
|
87
|
+
return 0
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
if __name__ == "__main__":
|
|
91
|
+
sys.exit(main())
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""gen_routing_snapshot.py — 路由决策快照:生成与对账(--check 供测试调用)。
|
|
3
|
+
|
|
4
|
+
治的是慢性病:近 90 天 `fix(route)` 13 次——每次动域正则都可能碰翻另一条
|
|
5
|
+
查询的路由,而现有防线只有手工维护的 ~100 条标注用例(ROUTE_MATRIX +
|
|
6
|
+
precision 测试),没有「这次改动翻转了哪些查询」的总视图。本脚本把路由
|
|
7
|
+
决策变成可 diff 的快照:
|
|
8
|
+
|
|
9
|
+
语料(三处标注集的并集,去重):
|
|
10
|
+
1. matrix_search_eval.ROUTE_MATRIX(59 条,带期望域)
|
|
11
|
+
2. test_domain_pattern_precision.LEGITIMATE_ROUTES(41 条,期望域)
|
|
12
|
+
3. 同文件 SUBSTRING_ACCIDENTS(~15 条,不得命中的域)
|
|
13
|
+
每条记录 {query, domain, kind, expect, engines}。**比对键只有 domain 与
|
|
14
|
+
kind**(regex/tfidf):引擎组合随熔断/配额/健康态波动,参与比对会让门禁
|
|
15
|
+
被环境噪声天天假红,直到被人 --no-verify——那比没有门禁更坏。
|
|
16
|
+
|
|
17
|
+
生成(改完路由先跑它 review diff 再提交):
|
|
18
|
+
python3 scripts/gen_routing_snapshot.py
|
|
19
|
+
对账(tests/test_routing_snapshot.py 调用;diff 即 exit 1):
|
|
20
|
+
python3 scripts/gen_routing_snapshot.py --check
|
|
21
|
+
本地附加(不入快照、不对账):把本机 usage_log 的 Top-N 真实查询过一遍
|
|
22
|
+
路由,只看不锁——生产形态查询的漂移观察窗:
|
|
23
|
+
python3 scripts/gen_routing_snapshot.py --local 50
|
|
24
|
+
|
|
25
|
+
与 gen_engine_catalog.py 同一模式:真源在 config.yaml 与 route,快照是投影,
|
|
26
|
+
过期由测试拦下。
|
|
27
|
+
"""
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import argparse
|
|
31
|
+
import json
|
|
32
|
+
import sys
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
|
|
35
|
+
ROOT = Path(__file__).resolve().parent.parent
|
|
36
|
+
SCRIPTS = ROOT / "scripts"
|
|
37
|
+
TESTS = ROOT / "tests"
|
|
38
|
+
SNAPSHOT = TESTS / "golden" / "routing_snapshot.json"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _collect_corpus() -> list[dict]:
|
|
42
|
+
"""三处标注集合并去重。期望域/禁入域作上下文记录,不参与对账。"""
|
|
43
|
+
sys.path.insert(0, str(SCRIPTS))
|
|
44
|
+
sys.path.insert(0, str(TESTS))
|
|
45
|
+
entries: dict[str, dict] = {}
|
|
46
|
+
|
|
47
|
+
from matrix_search_eval import ROUTE_MATRIX
|
|
48
|
+
for c in ROUTE_MATRIX:
|
|
49
|
+
q = (c.get("q") or "").strip()
|
|
50
|
+
if q:
|
|
51
|
+
entries[q] = {"query": q, "expect": c.get("domain"),
|
|
52
|
+
"src": "ROUTE_MATRIX"}
|
|
53
|
+
|
|
54
|
+
import test_domain_pattern_precision as precision
|
|
55
|
+
for q, want in precision.LEGITIMATE_ROUTES:
|
|
56
|
+
q = q.strip()
|
|
57
|
+
if q:
|
|
58
|
+
entries.setdefault(q, {"query": q, "expect": want,
|
|
59
|
+
"src": "LEGITIMATE_ROUTES"})
|
|
60
|
+
for q, forbidden in precision.SUBSTRING_ACCIDENTS:
|
|
61
|
+
q = q.strip()
|
|
62
|
+
if q:
|
|
63
|
+
e = entries.get(q) or {"query": q, "src": "SUBSTRING_ACCIDENTS"}
|
|
64
|
+
e["forbid"] = forbidden
|
|
65
|
+
entries[q] = e
|
|
66
|
+
return [entries[k] for k in sorted(entries)]
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _route_one(query: str) -> dict:
|
|
70
|
+
from route import route_query
|
|
71
|
+
d = route_query(query, mode="auto", depth="fast")
|
|
72
|
+
reason = d.get("reason") or ""
|
|
73
|
+
kind = "tfidf" if "TF-IDF" in reason else "regex"
|
|
74
|
+
return {
|
|
75
|
+
"domain": d.get("domain"),
|
|
76
|
+
"kind": kind,
|
|
77
|
+
"engines": (d.get("engines_combo") or [])[:2],
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _build() -> list[dict]:
|
|
82
|
+
out = []
|
|
83
|
+
for e in _collect_corpus():
|
|
84
|
+
r = _route_one(e["query"])
|
|
85
|
+
out.append({"query": e["query"], **r,
|
|
86
|
+
"expect": e.get("expect"), "forbid": e.get("forbid"),
|
|
87
|
+
"src": e.get("src")})
|
|
88
|
+
return out
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _load_snapshot() -> dict:
|
|
92
|
+
return json.loads(SNAPSHOT.read_text(encoding="utf-8"))
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _diff(current: list[dict], recorded: list[dict]) -> list[str]:
|
|
96
|
+
old = {e["query"]: e for e in recorded.get("entries", [])}
|
|
97
|
+
problems = []
|
|
98
|
+
for e in current:
|
|
99
|
+
prev = old.get(e["query"])
|
|
100
|
+
if prev is None:
|
|
101
|
+
problems.append(f" 新增查询未入快照: {e['query']!r}")
|
|
102
|
+
continue
|
|
103
|
+
for key in ("domain", "kind"):
|
|
104
|
+
if prev.get(key) != e.get(key):
|
|
105
|
+
problems.append(
|
|
106
|
+
f" {e['query']!r}: {key} {prev.get(key)!r} -> {e.get(key)!r}"
|
|
107
|
+
f"(期望域 {e.get('expect')!r},来源 {e.get('src')})")
|
|
108
|
+
for q in sorted(set(old) - {e["query"] for e in current}):
|
|
109
|
+
problems.append(f" 快照有而语料没有: {q!r}(语料被删?若是故意,重新生成)")
|
|
110
|
+
# 期望域失配(标注集自己抓的那层,快照顺带报告)
|
|
111
|
+
for e in current:
|
|
112
|
+
if e.get("expect") and e["domain"] != e["expect"]:
|
|
113
|
+
problems.append(f" 期望域失配: {e['query']!r} 期望 {e['expect']!r}"
|
|
114
|
+
f" 实际 {e['domain']!r}(precision 测试也应红)")
|
|
115
|
+
return problems
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _local_report(n: int) -> None:
|
|
119
|
+
"""usage_log Top-N 真实查询的路由观察窗(不入快照,仅打印)。"""
|
|
120
|
+
import collections
|
|
121
|
+
import glob
|
|
122
|
+
counts: collections.Counter = collections.Counter()
|
|
123
|
+
for f in glob.glob(str(Path.home() / ".cache" / "unified-search"
|
|
124
|
+
/ "usage_log" / "*.jsonl")):
|
|
125
|
+
try:
|
|
126
|
+
for line in open(f, encoding="utf-8"):
|
|
127
|
+
try:
|
|
128
|
+
d = json.loads(line)
|
|
129
|
+
except Exception:
|
|
130
|
+
continue
|
|
131
|
+
if d.get("stream") == "query" and d.get("query"):
|
|
132
|
+
counts[d["query"].strip()] += 1
|
|
133
|
+
except OSError:
|
|
134
|
+
continue
|
|
135
|
+
top = counts.most_common(n)
|
|
136
|
+
print(f"== usage_log Top-{len(top)} 真实查询路由(仅观察,不对账)==")
|
|
137
|
+
for q, c in top:
|
|
138
|
+
r = _route_one(q)
|
|
139
|
+
print(f" ×{c:<3} {r['kind']:<6} {r['domain']:<18} {q[:40]!r}")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def main() -> int:
|
|
143
|
+
ap = argparse.ArgumentParser(description="路由决策快照生成/对账")
|
|
144
|
+
ap.add_argument("--check", action="store_true",
|
|
145
|
+
help="对账模式:与已提交快照 diff,有漂移即 exit 1")
|
|
146
|
+
ap.add_argument("--local", type=int, metavar="N", default=0,
|
|
147
|
+
help="附加:把本机 usage_log Top-N 查询过一遍路由(仅观察)")
|
|
148
|
+
args = ap.parse_args()
|
|
149
|
+
|
|
150
|
+
current = _build()
|
|
151
|
+
if args.local:
|
|
152
|
+
_local_report(args.local)
|
|
153
|
+
return 0
|
|
154
|
+
|
|
155
|
+
if args.check:
|
|
156
|
+
if not SNAPSHOT.exists():
|
|
157
|
+
print(f"快照不存在: {SNAPSHOT}\n先跑: python3 scripts/gen_routing_snapshot.py")
|
|
158
|
+
return 1
|
|
159
|
+
problems = _diff(current, _load_snapshot())
|
|
160
|
+
if problems:
|
|
161
|
+
print(f"路由快照漂移 {len(problems)} 处(改路由请重新生成并 review diff):")
|
|
162
|
+
print("\n".join(problems))
|
|
163
|
+
print("\n重新生成: python3 scripts/gen_routing_snapshot.py")
|
|
164
|
+
return 1
|
|
165
|
+
print(f"路由快照对账通过({len(current)} 条查询,domain+kind 零漂移)")
|
|
166
|
+
return 0
|
|
167
|
+
|
|
168
|
+
payload = {
|
|
169
|
+
"version": 1,
|
|
170
|
+
"generated_by": "scripts/gen_routing_snapshot.py",
|
|
171
|
+
"compare_keys": ["domain", "kind"],
|
|
172
|
+
"note": "engines 仅存档不比对(随熔断/配额波动);expect/forbid 是语料"
|
|
173
|
+
"自带的标注,失配由 precision/negative 测试与 --check 双报",
|
|
174
|
+
"entries": current,
|
|
175
|
+
}
|
|
176
|
+
SNAPSHOT.parent.mkdir(parents=True, exist_ok=True)
|
|
177
|
+
SNAPSHOT.write_text(json.dumps(payload, ensure_ascii=False, indent=1) + "\n",
|
|
178
|
+
encoding="utf-8")
|
|
179
|
+
print(f"已写入 {SNAPSHOT}({len(current)} 条)")
|
|
180
|
+
return 0
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
if __name__ == "__main__":
|
|
184
|
+
sys.exit(main())
|