argo-search 1.0.1 → 2.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -21
- package/README.en.md +822 -0
- package/README.es.md +864 -0
- package/README.ja.md +864 -0
- package/README.ko.md +864 -0
- package/README.md +867 -320
- package/SKILL.md +135 -0
- package/assets/readme/hero.svg +121 -0
- package/assets/readme/made-with-beautify.svg +10 -0
- package/assets/readme/proof-routes.svg +73 -0
- package/assets/readme/settings-keys.png +0 -0
- package/assets/readme/settings-tools.png +0 -0
- package/assets/readme/why-better.svg +104 -0
- package/assets/readme/workflow.svg +83 -0
- package/backends/domain_profiles.json +2308 -96
- package/backends/engine_registry.yaml +3314 -503
- package/backends/query_synonyms_cn.json +35 -0
- package/backends/quota_profiles.json +2409 -161
- package/backends/source_types_cn.json +83 -0
- package/backends/text_lexicon.json +44 -0
- package/bin/argo +703 -0
- package/bin/argo.js +71 -39
- package/config.yaml +4976 -363
- package/cordis.patch.yml +21 -0
- package/data/countries.json +40 -0
- package/data/lang_matrix/lang_capability.json +842 -0
- package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
- package/data/regions_cn.json +1 -0
- package/docs/ADDING_NEW_ENGINE.md +184 -0
- package/docs/ARGO_INTRO.md +124 -0
- package/docs/DATA_SOURCE_GAPS.md +81 -0
- package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
- package/docs/ENGINE_CATALOG.md +564 -0
- package/docs/MCP_SETUP.md +106 -0
- package/docs/PLATFORM_PATHS.md +111 -0
- package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
- package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
- package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
- package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
- package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
- package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
- package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
- package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
- package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
- package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
- package/docs/SEARCH_ARCHIVE.md +115 -0
- package/docs/assets/architecture.svg +104 -0
- package/docs/assets/hero.svg +98 -0
- package/docs/engines/bangumi.md +27 -0
- package/docs/engines/biorxiv.md +27 -0
- package/docs/engines/clawhub.md +27 -0
- package/docs/engines/crt_sh.md +27 -0
- package/docs/engines/datacite.md +27 -0
- package/docs/engines/deps_dev.md +27 -0
- package/docs/engines/douban_movie.md +27 -0
- package/docs/engines/endoflife.md +27 -0
- package/docs/engines/flk_law.md +27 -0
- package/docs/engines/fxtwitter.md +27 -0
- package/docs/engines/google_news.md +27 -0
- package/docs/engines/gov_regulations.md +27 -0
- package/docs/engines/guqian.md +27 -0
- package/docs/engines/hackernews.md +27 -0
- package/docs/engines/iplant.md +27 -0
- package/docs/engines/keenable.md +27 -0
- package/docs/engines/met_museum.md +27 -0
- package/docs/engines/nasa_images.md +27 -0
- package/docs/engines/netease_music.md +27 -0
- package/docs/engines/npm.md +27 -0
- package/docs/engines/nvd.md +27 -0
- package/docs/engines/openreview.md +27 -0
- package/docs/engines/openstd.md +27 -0
- package/docs/engines/parallel_free.md +27 -0
- package/docs/engines/people_daily.md +27 -0
- package/docs/engines/realtime_index.md +27 -0
- package/docs/engines/sec_edgar.md +27 -0
- package/docs/engines/seltz.md +27 -0
- package/docs/engines/skillsmp.md +27 -0
- package/docs/engines/sspai.md +27 -0
- package/docs/engines/std_samr.md +27 -0
- package/docs/engines/train.md +27 -0
- package/docs/engines/un_comtrade.md +27 -0
- package/docs/engines/wikisource.md +27 -0
- package/docs/engines/zdic.md +27 -0
- package/docs/engines/zenodo.md +27 -0
- package/docs/engines/zhihu_hot_app.md +27 -0
- package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
- package/engines/README.md +10 -0
- package/engines/_template_cli.yaml +39 -0
- package/engines/_template_http.yaml +37 -0
- package/engines/plugins/_template_plugin.py +50 -0
- package/engines/specs/_cisa_kev.yaml +54 -0
- package/engines/specs/_osv.yaml +63 -0
- package/engines/specs/aljazeera.yaml +43 -0
- package/engines/specs/antara.yaml +43 -0
- package/engines/specs/artic.yaml +20 -0
- package/engines/specs/aviation_weather.yaml +39 -0
- package/engines/specs/bangumi.yaml +21 -0
- package/engines/specs/biorxiv.yaml +24 -0
- package/engines/specs/carbon_intensity.yaml +32 -0
- package/engines/specs/cinii.yaml +44 -0
- package/engines/specs/clawhub.yaml +49 -0
- package/engines/specs/cleveland.yaml +19 -0
- package/engines/specs/cn_ai_news.yaml +43 -0
- package/engines/specs/core.yaml +35 -0
- package/engines/specs/crt_sh.yaml +38 -0
- package/engines/specs/ct24.yaml +43 -0
- package/engines/specs/datacite.yaml +27 -0
- package/engines/specs/datagov_uk.yaml +41 -0
- package/engines/specs/ddgs_images.yaml +45 -0
- package/engines/specs/deepseek_search.yaml +74 -0
- package/engines/specs/deezer.yaml +20 -0
- package/engines/specs/deps_dev.yaml +21 -0
- package/engines/specs/digup.yaml +36 -0
- package/engines/specs/doi.yaml +39 -0
- package/engines/specs/douban_movie.yaml +22 -0
- package/engines/specs/dw_news.yaml +47 -0
- package/engines/specs/egov_law.yaml +21 -0
- package/engines/specs/elpais.yaml +43 -0
- package/engines/specs/endoflife.yaml +21 -0
- package/engines/specs/energy_charts.yaml +21 -0
- package/engines/specs/factcheck_org.yaml +52 -0
- package/engines/specs/faz.yaml +43 -0
- package/engines/specs/federal_register.yaml +57 -0
- package/engines/specs/firecrawl.yaml +53 -0
- package/engines/specs/flk_law.yaml +24 -0
- package/engines/specs/folha.yaml +43 -0
- package/engines/specs/france24.yaml +44 -0
- package/engines/specs/full_fact.yaml +47 -0
- package/engines/specs/fxtwitter.yaml +31 -0
- package/engines/specs/gbfs_nyc.yaml +32 -0
- package/engines/specs/gdacs.yaml +21 -0
- package/engines/specs/gdelt.yaml +64 -0
- package/engines/specs/google_news.yaml +22 -0
- package/engines/specs/gov_regulations.yaml +38 -0
- package/engines/specs/guardian_rss.yaml +59 -0
- package/engines/specs/hurriyet.yaml +43 -0
- package/engines/specs/iplant.yaml +25 -0
- package/engines/specs/jikan.yaml +24 -0
- package/engines/specs/jolpica.yaml +20 -0
- package/engines/specs/k10plus.yaml +20 -0
- package/engines/specs/keenable.yaml +44 -0
- package/engines/specs/lefigaro.yaml +43 -0
- package/engines/specs/listenbrainz.yaml +19 -0
- package/engines/specs/matichon.yaml +43 -0
- package/engines/specs/met_museum.yaml +21 -0
- package/engines/specs/nasa_images.yaml +45 -0
- package/engines/specs/netease_music.yaml +38 -0
- package/engines/specs/nhk.yaml +43 -0
- package/engines/specs/nhtsa_vpic.yaml +20 -0
- package/engines/specs/noaa_swpc.yaml +20 -0
- package/engines/specs/nvd.yaml +38 -0
- package/engines/specs/obis.yaml +20 -0
- package/engines/specs/opencitations.yaml +52 -0
- package/engines/specs/openf1.yaml +21 -0
- package/engines/specs/openfoodfacts.yaml +45 -0
- package/engines/specs/openligadb.yaml +20 -0
- package/engines/specs/openreview.yaml +37 -0
- package/engines/specs/openstd.yaml +21 -0
- package/engines/specs/people_daily.yaml +22 -0
- package/engines/specs/realtime_index.yaml +34 -0
- package/engines/specs/ror.yaml +20 -0
- package/engines/specs/satnogs.yaml +20 -0
- package/engines/specs/sec_edgar.yaml +46 -0
- package/engines/specs/skillsmp.yaml +50 -0
- package/engines/specs/sogou_weixin_browser.yaml +30 -0
- package/engines/specs/soilgrids.yaml +25 -0
- package/engines/specs/sspai.yaml +36 -0
- package/engines/specs/stackexchange.yaml +46 -0
- package/engines/specs/std_samr.yaml +21 -0
- package/engines/specs/tass.yaml +43 -0
- package/engines/specs/tle_mirror.yaml +21 -0
- package/engines/specs/train.yaml +37 -0
- package/engines/specs/tvmaze.yaml +20 -0
- package/engines/specs/twitter_syndication.yaml +31 -0
- package/engines/specs/un_comtrade.yaml +23 -0
- package/engines/specs/unpaywall.yaml +54 -0
- package/engines/specs/vnpress.yaml +43 -0
- package/engines/specs/weather.yaml +32 -0
- package/engines/specs/who_don.yaml +21 -0
- package/engines/specs/who_gho.yaml +20 -0
- package/engines/specs/wikimedia_commons.yaml +34 -0
- package/engines/specs/wikisource.yaml +22 -0
- package/engines/specs/worms.yaml +19 -0
- package/engines/specs/x_trending.yaml +34 -0
- package/engines/specs/yna.yaml +43 -0
- package/engines/specs/ynet.yaml +43 -0
- package/engines/specs/zbmath.yaml +40 -0
- package/engines/specs/zdic.yaml +22 -0
- package/engines/specs/zenodo.yaml +27 -0
- package/engines/specs/zhihu_hot_app.yaml +35 -0
- package/mcp/clients.yaml +74 -0
- package/package.json +51 -12
- package/packages/dsh-plugin/README.md +134 -0
- package/packages/dsh-plugin/cordis.patch.yml +48 -0
- package/packages/dsh-plugin/dsh/client.js +1244 -0
- package/packages/dsh-plugin/dsh/index.js +2290 -0
- package/packages/dsh-plugin/dsh/native-tools.mjs +743 -0
- package/packages/dsh-plugin/package.json +38 -0
- package/packages/dsh-plugin/test/client.test.js +232 -0
- package/packages/dsh-plugin/test/config.test.js +30 -0
- package/packages/dsh-plugin/test/index.test.js +1338 -0
- package/references/query-templates.md +96 -0
- package/references/query-templates.yaml +118 -0
- package/scripts/ab_eval.py +219 -0
- package/scripts/ab_eval_p0p1.py +266 -0
- package/scripts/adaptive.py +271 -70
- package/scripts/answer.py +154 -0
- package/scripts/archive_run.py +591 -0
- package/scripts/argo_engine_registry.py +172 -0
- package/scripts/argo_paths.py +728 -0
- package/scripts/article.py +136 -0
- package/scripts/arxiv_source.py +201 -0
- package/scripts/badcase_ledger.py +252 -0
- package/scripts/batch_probe.py +241 -0
- package/scripts/benchmark.py +19 -0
- package/scripts/bounded_run.py +144 -0
- package/scripts/browser_auth.py +323 -0
- package/scripts/burstiness_detector.py +147 -0
- package/scripts/cache.py +1078 -67
- package/scripts/cache_guard.py +271 -0
- package/scripts/cache_key_vdom.py +56 -0
- package/scripts/cache_similarity.py +170 -0
- package/scripts/caliber_isolation.py +195 -0
- package/scripts/calibrate_lowquality.py +210 -0
- package/scripts/candidate_envelope.py +366 -0
- package/scripts/chrome_cdp.py +871 -0
- package/scripts/circuit_breaker.py +389 -0
- package/scripts/citations.py +334 -0
- package/scripts/cite_cli.py +59 -0
- package/scripts/cjk_tokens.py +95 -0
- package/scripts/clarify.py +47 -24
- package/scripts/cli_io.py +240 -0
- package/scripts/config.py +683 -45
- package/scripts/config_cache.py +402 -0
- package/scripts/conn_pool.py +220 -0
- package/scripts/content_security.py +685 -0
- package/scripts/content_signals.py +730 -0
- package/scripts/crawl.py +74 -35
- package/scripts/daemon.py +866 -0
- package/scripts/digup_bridge.py +214 -0
- package/scripts/engine_admission.py +366 -0
- package/scripts/engine_dispatch.py +1019 -0
- package/scripts/engine_env.py +534 -0
- package/scripts/engine_failure.py +411 -0
- package/scripts/engine_families.py +732 -0
- package/scripts/engine_policy.py +218 -0
- package/scripts/engine_requires.py +190 -0
- package/scripts/engine_status.py +322 -0
- package/scripts/engine_validate.py +638 -0
- package/scripts/engines.py +778 -294
- package/scripts/engines_base.py +1521 -0
- package/scripts/engines_builders.py +298 -0
- package/scripts/engines_builders_batch10.py +159 -0
- package/scripts/engines_builders_batch11.py +174 -0
- package/scripts/engines_builders_batch12.py +103 -0
- package/scripts/engines_builders_batch13.py +340 -0
- package/scripts/engines_builders_batch9.py +1468 -0
- package/scripts/engines_builders_cn.py +2283 -0
- package/scripts/engines_builders_data.py +2923 -0
- package/scripts/engines_builders_data_macro.py +775 -0
- package/scripts/engines_builders_feeds.py +255 -0
- package/scripts/engines_builders_intl.py +1239 -0
- package/scripts/engines_builders_search.py +440 -0
- package/scripts/engines_builders_tech.py +1360 -0
- package/scripts/evidence.py +588 -162
- package/scripts/evidence_loop.py +592 -0
- package/scripts/evidence_tier.py +298 -0
- package/scripts/except_sets.py +25 -0
- package/scripts/extract.py +55 -16
- package/scripts/fact_align.py +207 -0
- package/scripts/fact_crosscheck.py +230 -0
- package/scripts/fetch.py +25 -12
- package/scripts/fetch_browser.py +87 -0
- package/scripts/fetch_jsrun.py +122 -0
- package/scripts/fetch_quality.py +225 -0
- package/scripts/fetch_render_tinyfish.py +117 -0
- package/scripts/fetch_v3.py +1779 -0
- package/scripts/focus_extract.py +350 -0
- package/scripts/fulltext_store.py +169 -0
- package/scripts/fxtwitter_api.py +325 -0
- package/scripts/gen_engine_catalog.py +364 -0
- package/scripts/gen_native_tools.py +91 -0
- package/scripts/gen_routing_snapshot.py +184 -0
- package/scripts/health_check.py +140 -402
- package/scripts/health_probe.py +23 -8
- package/scripts/hot_state.py +134 -0
- package/scripts/http_client.py +1053 -0
- package/scripts/image_ops.py +430 -0
- package/scripts/image_vision/vision_probe +0 -0
- package/scripts/image_vision/vision_probe.swift +126 -0
- package/scripts/install.ps1 +148 -0
- package/scripts/install.sh +206 -0
- package/scripts/job.py +1186 -0
- package/scripts/jump_resolver.py +235 -0
- package/scripts/lang_capability.py +150 -0
- package/scripts/lang_detect.py +289 -0
- package/scripts/lang_matrix_probe.py +329 -0
- package/scripts/lang_pref.py +311 -0
- package/scripts/link_source.py +340 -0
- package/scripts/local_engines.py +321 -0
- package/scripts/local_image.py +970 -0
- package/scripts/local_seek.py +366 -0
- package/scripts/macro_countries.py +82 -0
- package/scripts/matrix_search_eval.py +1105 -0
- package/scripts/mcp_diag.py +8 -3
- package/scripts/mcp_handlers.py +1074 -0
- package/scripts/mcp_handlers_surface.py +157 -0
- package/scripts/mcp_launch.sh +122 -0
- package/scripts/mcp_server.py +118 -417
- package/scripts/mcp_setup.py +407 -0
- package/scripts/mcp_tools.py +330 -0
- package/scripts/mcp_transport.py +191 -0
- package/scripts/min_version_scan.py +335 -0
- package/scripts/net_proxy.py +258 -0
- package/scripts/network_aware.py +152 -0
- package/scripts/paper.py +274 -0
- package/scripts/pdf_extract.py +518 -0
- package/scripts/phrase_filter.py +66 -0
- package/scripts/plan.py +422 -0
- package/scripts/postprocess_signals.py +150 -0
- package/scripts/pre-commit +48 -0
- package/scripts/query_enhance.py +157 -0
- package/scripts/query_operators.py +196 -0
- package/scripts/query_rewriter.py +116 -7
- package/scripts/query_signals.py +300 -0
- package/scripts/query_templates.py +167 -0
- package/scripts/query_understanding.py +411 -0
- package/scripts/query_variants.py +298 -0
- package/scripts/quota.py +488 -76
- package/scripts/rank_signals.py +775 -0
- package/scripts/ranking_eval.py +319 -0
- package/scripts/readability_extract.py +433 -0
- package/scripts/recompute.py +415 -0
- package/scripts/recovery.py +616 -0
- package/scripts/recovery_l0.py +160 -0
- package/scripts/redskill/redskill_engine.py +332 -0
- package/scripts/regression_p0p1.py +318 -0
- package/scripts/replay_eval.py +640 -0
- package/scripts/rerank_tfidf.py +225 -0
- package/scripts/research.py +591 -403
- package/scripts/research_cli.py +361 -0
- package/scripts/research_dossier.py +492 -0
- package/scripts/research_expand.py +146 -0
- package/scripts/research_gates.py +272 -0
- package/scripts/research_report.py +209 -0
- package/scripts/research_strategy.py +37 -0
- package/scripts/research_work_packages.py +195 -0
- package/scripts/result_lang.py +311 -0
- package/scripts/robots_guard.py +259 -0
- package/scripts/route.py +921 -254
- package/scripts/route_bangs.py +40 -0
- package/scripts/route_cache.py +225 -0
- package/scripts/route_combo.py +910 -0
- package/scripts/route_domains.py +328 -0
- package/scripts/route_lang.py +605 -0
- package/scripts/route_log.py +63 -0
- package/scripts/route_policy.py +250 -0
- package/scripts/rrf_classic.py +66 -0
- package/scripts/screenshot.py +73 -0
- package/scripts/search.py +897 -405
- package/scripts/search_attribution.py +88 -0
- package/scripts/search_benchmark.py +407 -0
- package/scripts/search_cli.py +627 -0
- package/scripts/search_entry.py +389 -0
- package/scripts/search_output.py +606 -0
- package/scripts/search_pipeline.py +1077 -0
- package/scripts/search_rank.py +991 -0
- package/scripts/search_types.py +27 -12
- package/scripts/seek_locator.py +68 -0
- package/scripts/semantic_evidence.py +215 -0
- package/scripts/serp_guard.py +228 -0
- package/scripts/serp_kill_log.py +51 -0
- package/scripts/single_flight.py +120 -0
- package/scripts/social_engines/__init__.py +10 -0
- package/scripts/social_engines/bilibili_engine.py +131 -0
- package/scripts/social_engines/hackernews_engine.py +113 -0
- package/scripts/social_engines/reddit_engine.py +172 -0
- package/scripts/social_engines/twitter_engine.py +397 -0
- package/scripts/social_engines/v2ex_engine.py +101 -0
- package/scripts/social_engines/weibo_engine.py +128 -0
- package/scripts/social_engines/xiaohongshu_engine.py +137 -0
- package/scripts/social_engines/zhihu_engine.py +139 -0
- package/scripts/social_research.py +201 -0
- package/scripts/sogou_weixin_browser.py +318 -0
- package/scripts/source_ledger.py +249 -0
- package/scripts/stage_timing.py +58 -0
- package/scripts/stats_cli.py +95 -0
- package/scripts/straggler_policy.py +60 -0
- package/scripts/stylometry_detector.py +231 -0
- package/scripts/sync_backends.py +353 -0
- package/scripts/sync_installs.py +29 -0
- package/scripts/template_cli.py +127 -0
- package/scripts/text_lexicon.py +72 -0
- package/scripts/tfidf_router.py +109 -37
- package/scripts/time_utils.py +233 -0
- package/scripts/topic_research_profiles.py +643 -0
- package/scripts/train.py +737 -0
- package/scripts/tweet.py +596 -0
- package/scripts/url_canon.py +217 -0
- package/scripts/url_optimize.py +56 -0
- package/scripts/url_safety.py +236 -0
- package/scripts/usage_log.py +116 -0
- package/scripts/usage_log_clean.py +113 -0
- package/scripts/v2ex_nodes.py +311 -0
- package/scripts/variant_recall.py +188 -0
- package/scripts/watch.py +205 -0
- package/scripts/wechat_article.py +258 -0
- package/scripts/wide_research.py +402 -0
- package/scripts/wx.py +349 -0
- package/scripts/x_article_fetch.py +141 -0
- package/scripts/x_search.py +234 -0
- package/scripts/x_trending.py +189 -0
- package/scripts/yaml_load.py +67 -0
- package/sub-skills/ego-search/SKILL.md +253 -0
- package/sub-skills/ego-search/references/browser-runtime.md +43 -0
- package/sub-skills/ego-search/references/install.md +67 -0
- package/sub-skills/ego-search/references/original-ego-upgrade.md +51 -0
- package/sub-skills/ego-search/scripts/ego_search.py +800 -0
- package/sub-skills/ego-search/scripts/merge.py +226 -0
- package/sub-skills/ego-search/scripts/quality.py +64 -0
- package/sub-skills/ego-search/scripts/runtime.py +172 -0
- package/sub-skills/ego-search/scripts/safety.py +109 -0
- package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
- package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
- package/sub-skills/ego-search/tests/test_query_normalize.py +68 -0
- package/sub-skills/js-run/SKILL.md +67 -0
- package/sub-skills/js-run/scripts/jsrun.py +112 -0
- package/sub-skills/js-run/scripts/shim.js +188 -0
- package/sub-skills/js-run/tests/fixtures/challenge_like.js +27 -0
- package/sub-skills/js-run/tests/test_jsrun.py +165 -0
- package/sub-skills/local-search/SKILL.md +50 -53
- package/sub-skills/local-search/bm25.py +162 -0
- package/sub-skills/local-search/config.yaml +152 -55
- package/sub-skills/local-search/engine_registry.py +91 -26
- package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
- package/sub-skills/local-search/local_search_adapter.py +13 -2
- package/sub-skills/local-search/parse_maps.yaml +40 -42
- package/sub-skills/local-search/search_v3.py +1182 -65
- package/sub-skills/local-search/smart_router.py +47 -7
- package/sub-skills/local-seek/SKILL.md +110 -0
- package/sub-skills/local-seek/config/domains.yaml +20 -0
- package/sub-skills/local-seek/references/auto-behaviors.md +24 -0
- package/sub-skills/local-seek/references/fd-mdfind-recipes.md +62 -0
- package/sub-skills/local-seek/references/git-integration.md +54 -0
- package/sub-skills/local-seek/references/remote-handoff.md +46 -0
- package/sub-skills/local-seek/references/rg-recipes.md +96 -0
- package/sub-skills/local-seek/references/strategies.md +52 -0
- package/sub-skills/local-seek/references/structural-search.md +59 -0
- package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
- package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
- package/sub-skills/local-seek/scripts/seek.py +1465 -0
- package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
- package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
- package/sub-skills/local-seek/scripts/seek_rank.py +193 -0
- package/sub-skills/local-seek/tests/test_content_rank.py +222 -0
- package/sub-skills/local-seek/tests/test_determinism_20261006.py +145 -0
- package/sub-skills/local-seek/tests/test_noise_tier.py +101 -0
- package/sub-skills/local-seek/tests/test_pcre2_probe.py +136 -0
- package/sub-skills/local-seek/tests/test_rank_pinyin.py +77 -0
- package/sub-skills/local-seek/tests/test_regression_20261003.py +169 -0
- package/sub-skills/local-seek/tests/test_regression_20261006.py +146 -0
- package/sub-skills/local-seek/tests/test_seek_degradation.py +111 -0
- package/sub-skills/local-seek/tests/test_seek_fts.py +313 -0
|
@@ -0,0 +1,616 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
recovery.py — 空结果错误恢复决策树(P0-002)
|
|
4
|
+
|
|
5
|
+
第一性原理:
|
|
6
|
+
「零结果」不是终态而是可恢复状态。用最小额外成本,按信息增益从高到低
|
|
7
|
+
逐级放宽约束,直到拿到可吸收事实或耗尽策略。
|
|
8
|
+
|
|
9
|
+
MECE 五级决策树(L1→L5,成本递增):
|
|
10
|
+
L1 放宽 : 去停用词 / 去引号 / 截断过长尾修饰
|
|
11
|
+
L2 同义 : 小同义表替换(财报↔earnings,股价↔行情,教程↔guide)
|
|
12
|
+
L3 换引擎 : 从 engines_fallback 或通用免费组合取未试引擎
|
|
13
|
+
L4 跨语言 : 非拉丁→通用英源;英文+偏好zh→中文源;中英互译启发式
|
|
14
|
+
L5 结构化 : 输出 recovery 字段告知 Agent 已降级及降级路径
|
|
15
|
+
|
|
16
|
+
模式约束:
|
|
17
|
+
mode=fast : 最多 L1 + L3,禁止 L4(控制延迟/成本)
|
|
18
|
+
|
|
19
|
+
设计为「计划生成器」:本模块只产出候选 (query, engines) 与理由,
|
|
20
|
+
不直接执行网络请求,由 search.py 用回调执行,保持职责单一。
|
|
21
|
+
|
|
22
|
+
纯本地,仅 stdlib + 一个 JSON 小表。
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import json
|
|
28
|
+
import re
|
|
29
|
+
from dataclasses import dataclass, field
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
from typing import Any, Callable, Optional
|
|
32
|
+
|
|
33
|
+
from cjk_tokens import signal_tokens
|
|
34
|
+
|
|
35
|
+
_BACKENDS_DIR = Path(__file__).parent.parent / "backends"
|
|
36
|
+
_SYNONYMS_PATH = _BACKENDS_DIR / "query_synonyms_cn.json"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# ── 停用词 / 尾修饰 ────────────────────────────────────────────────────────────
|
|
40
|
+
|
|
41
|
+
_STOPWORDS_CN = {
|
|
42
|
+
"的", "了", "呢", "吧", "啊", "呀", "吗", "哦", "嘛", "些", "个",
|
|
43
|
+
"请问", "怎么", "如何", "一下", "帮我", "我想", "想要", "关于", "有关",
|
|
44
|
+
"求", "跪求", "谁知道", "有没有", "是不是",
|
|
45
|
+
}
|
|
46
|
+
_STOPWORDS_EN = {"the", "a", "an", "of", "for", "to", "please", "how", "what"}
|
|
47
|
+
|
|
48
|
+
# 常见「过长尾修饰」:地点/时间/程度等,去掉后主体更聚焦
|
|
49
|
+
_TAIL_MODIFIERS = re.compile(
|
|
50
|
+
r"(最新|最全|详细|完整|全面|2020|2021|2022|2023|2024|2025|2026|"
|
|
51
|
+
r"知乎|贴吧|论坛|大全|汇总|排行榜|一览)")
|
|
52
|
+
|
|
53
|
+
_QUOTE_CHARS = re.compile(r'["\'“”‘’「」『』]')
|
|
54
|
+
_TOKEN_RE = re.compile(r"[\u4e00-\u9fff]|[a-zA-Z0-9]+")
|
|
55
|
+
_EN_WORD = re.compile(r"\b[a-zA-Z]{3,}\b")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# ── 数据结构 ──────────────────────────────────────────────────────────────────
|
|
59
|
+
|
|
60
|
+
@dataclass
|
|
61
|
+
class RecoveryStep:
|
|
62
|
+
"""单个恢复尝试。"""
|
|
63
|
+
|
|
64
|
+
level: str # L1 / L2 / L3 / L4
|
|
65
|
+
strategy: str # relax / synonym / switch_engine / translate
|
|
66
|
+
query: str
|
|
67
|
+
engines: list[str]
|
|
68
|
+
reason: str
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class RecoveryResult:
|
|
73
|
+
"""恢复执行的最终结构化结论(写入顶层 recovery 字段)。"""
|
|
74
|
+
|
|
75
|
+
triggered: bool = False
|
|
76
|
+
recovered: bool = False
|
|
77
|
+
level_used: str | None = None
|
|
78
|
+
strategy_used: str | None = None
|
|
79
|
+
steps_tried: list[dict[str, Any]] = field(default_factory=list)
|
|
80
|
+
final_query: str | None = None
|
|
81
|
+
note: str = ""
|
|
82
|
+
|
|
83
|
+
def to_dict(self) -> dict[str, Any]:
|
|
84
|
+
return {
|
|
85
|
+
"triggered": self.triggered,
|
|
86
|
+
"recovered": self.recovered,
|
|
87
|
+
"level_used": self.level_used,
|
|
88
|
+
"strategy_used": self.strategy_used,
|
|
89
|
+
"steps_tried": self.steps_tried,
|
|
90
|
+
"final_query": self.final_query,
|
|
91
|
+
"note": self.note,
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
# ── 同义表加载 ────────────────────────────────────────────────────────────────
|
|
96
|
+
|
|
97
|
+
_synonyms_cache: dict[str, Any] | None = None
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _load_synonyms() -> dict[str, Any]:
|
|
101
|
+
"""加载同义表(缓存)。文件缺失/损坏时返回空表,不抛异常。"""
|
|
102
|
+
global _synonyms_cache
|
|
103
|
+
if _synonyms_cache is not None:
|
|
104
|
+
return _synonyms_cache
|
|
105
|
+
if not _SYNONYMS_PATH.exists():
|
|
106
|
+
_synonyms_cache = {"synonyms": {}, "en_to_cn": {}}
|
|
107
|
+
return _synonyms_cache
|
|
108
|
+
try:
|
|
109
|
+
data = json.loads(_SYNONYMS_PATH.read_text(encoding="utf-8"))
|
|
110
|
+
_synonyms_cache = {
|
|
111
|
+
"synonyms": data.get("synonyms", {}),
|
|
112
|
+
"en_to_cn": data.get("en_to_cn", {}),
|
|
113
|
+
}
|
|
114
|
+
except (json.JSONDecodeError, OSError):
|
|
115
|
+
_synonyms_cache = {"synonyms": {}, "en_to_cn": {}}
|
|
116
|
+
return _synonyms_cache
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
# ── L1 放宽 ────────────────────────────────────────────────────────────────────
|
|
120
|
+
|
|
121
|
+
def relax_query(query: str) -> str:
|
|
122
|
+
"""L1:去引号 / 去停用词 / 去过长尾修饰。"""
|
|
123
|
+
q = _QUOTE_CHARS.sub(" ", query)
|
|
124
|
+
q = _TAIL_MODIFIERS.sub(" ", q)
|
|
125
|
+
# 去停用词(按 token)
|
|
126
|
+
tokens = re.findall(r"[\u4e00-\u9fffA-Za-z0-9]+", q)
|
|
127
|
+
kept = []
|
|
128
|
+
for t in tokens:
|
|
129
|
+
if t.lower() in _STOPWORDS_EN or t in _STOPWORDS_CN:
|
|
130
|
+
continue
|
|
131
|
+
kept.append(t)
|
|
132
|
+
relaxed = " ".join(kept).strip()
|
|
133
|
+
# 保底:去停用词后不能为空
|
|
134
|
+
return relaxed or re.sub(r"\s+", " ", q).strip() or query
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
# ── L1.5 结构化语法放宽(结果归零时逆序撤条件)─────────────────────────────────
|
|
138
|
+
|
|
139
|
+
# 平台原生结构化语法:精准检索结果归零时,按「宽松代价从小到大」逐层剥离,
|
|
140
|
+
# 每一步只撤一类、保留核心词。顺序=帖子给的逆序(排除→热度→媒体→语言→日期→
|
|
141
|
+
# 账号→短语)。跨平台通用:X from:/until:/min_faves:、GitHub repo:/is:、
|
|
142
|
+
# Web site:、Reddit subreddit: 等。
|
|
143
|
+
_STRUCTURED_RELAX_ORDER: list[tuple[str, re.Pattern]] = [
|
|
144
|
+
("exclude", re.compile(r"(?<![A-Za-z0-9])\-\S+")),
|
|
145
|
+
("heat", re.compile(r"(?:min_faves|min_retweets|min_replies|min_rt|min_followers):\d+")),
|
|
146
|
+
("media", re.compile(r"filter:[A-Za-z]+")),
|
|
147
|
+
("lang", re.compile(r"lang:[A-Za-z\-]+")),
|
|
148
|
+
("date", re.compile(r"(?:since|until):\S+")),
|
|
149
|
+
("account", re.compile(r"(?:from|to|via|list|repo|site|subreddit|user|org|is|label|author|assignee|milestone|comments|in|points|file|extension|path|filename):\S+")),
|
|
150
|
+
("phrase", re.compile(r'"[^"]+"')),
|
|
151
|
+
]
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def structured_relax_steps(query: str, max_steps: int = 6) -> list[str]:
|
|
155
|
+
"""按结构化语法逐层放宽,返回「从最保留到最宽」的候选查询序列。
|
|
156
|
+
|
|
157
|
+
只处理含平台语法的查询(高后果/精准检索);无语法时返回 []。
|
|
158
|
+
每步剥掉一类条件,保留其余,符合「归零时一次撤一个、从去排除到去短语」。
|
|
159
|
+
"""
|
|
160
|
+
if not query or not isinstance(query, str):
|
|
161
|
+
return []
|
|
162
|
+
steps: list[str] = []
|
|
163
|
+
cur = query
|
|
164
|
+
for _name, pat in _STRUCTURED_RELAX_ORDER:
|
|
165
|
+
if not pat.search(cur):
|
|
166
|
+
continue
|
|
167
|
+
if _name == "phrase":
|
|
168
|
+
relaxed = pat.sub(lambda m: m.group(0)[1:-1], cur) # 去引号保留词
|
|
169
|
+
else:
|
|
170
|
+
relaxed = pat.sub(" ", cur)
|
|
171
|
+
relaxed = re.sub(r"\s+", " ", relaxed).strip()
|
|
172
|
+
if relaxed and relaxed != cur and relaxed != query:
|
|
173
|
+
steps.append(relaxed)
|
|
174
|
+
cur = relaxed
|
|
175
|
+
if len(steps) >= max_steps:
|
|
176
|
+
break
|
|
177
|
+
return steps
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def strip_structured(query: str) -> str:
|
|
181
|
+
"""剥掉平台结构化字段,只保留核心关键词(供语义/通用引擎使用)。
|
|
182
|
+
|
|
183
|
+
语义型引擎(byted/bocha/anysearch 等)不理解 from:/repo:/site:/until: 这类
|
|
184
|
+
平台原生语法,把字段当普通词传进去反而污染相关性。此函数去掉「明确前缀字段」
|
|
185
|
+
与引号,保留核心词;无语法查询原样返回。`-排除` 不在此剥离(易误伤,交给 recovery)。
|
|
186
|
+
"""
|
|
187
|
+
if not query or not isinstance(query, str):
|
|
188
|
+
return query
|
|
189
|
+
cur = query
|
|
190
|
+
for _name, pat in _STRUCTURED_RELAX_ORDER:
|
|
191
|
+
if _name in ("exclude", "phrase"):
|
|
192
|
+
continue
|
|
193
|
+
if pat.search(cur):
|
|
194
|
+
cur = pat.sub(" ", cur)
|
|
195
|
+
cur = _QUOTE_CHARS.sub(" ", cur) # 去引号保留词
|
|
196
|
+
cur = re.sub(r"\s+", " ", cur).strip()
|
|
197
|
+
return cur or query
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
# ── L2 同义 ────────────────────────────────────────────────────────────────────
|
|
201
|
+
|
|
202
|
+
def synonym_expand(query: str) -> Optional[str]:
|
|
203
|
+
"""L2:将查询中命中的词替换为首个同义词。无命中返回 None。"""
|
|
204
|
+
table = _load_synonyms()["synonyms"]
|
|
205
|
+
for term, syns in table.items():
|
|
206
|
+
if term in query and syns:
|
|
207
|
+
return query.replace(term, syns[0])
|
|
208
|
+
return None
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
# ── L3 换引擎 ──────────────────────────────────────────────────────────────────
|
|
212
|
+
|
|
213
|
+
# 通用免费源清单:唯一来源 engine_policy.GENERAL_FREE_FALLBACK(与 route 保底共用,
|
|
214
|
+
# 避免两处清单漂移)。顺序 = 通用检索优先,百科殿后。
|
|
215
|
+
try:
|
|
216
|
+
from engine_policy import GENERAL_FREE_FALLBACK as _GENERAL_FREE_COMBO # noqa: E402
|
|
217
|
+
except ImportError:
|
|
218
|
+
_GENERAL_FREE_COMBO = ["anysearch", "duckduckgo", "local_bing", "local_baidu", "wikipedia"]
|
|
219
|
+
|
|
220
|
+
# 空结果恢复只允许「通用检索 + 百科」;垂直污染源(pypi/npm/jin10 等)一律禁入。
|
|
221
|
+
# 例外:已尝试引擎同族可保留(如 tried 含 thesportsdb → 可补 sports 族)。
|
|
222
|
+
_RECOVERY_SAFE_FAMILIES = frozenset({"web_general", "knowledge"})
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _engine_family(name: str) -> str:
|
|
226
|
+
try:
|
|
227
|
+
from engine_families import family_of
|
|
228
|
+
try:
|
|
229
|
+
from config import get_engines
|
|
230
|
+
spec = (get_engines() or {}).get(name)
|
|
231
|
+
except Exception:
|
|
232
|
+
spec = None
|
|
233
|
+
# 必须把引擎声明传进去:config.yaml 的 family 字段才是来源。不传时
|
|
234
|
+
# family_of 会退回静态覆盖表、再退到默认值 web_general——而恢复链正好
|
|
235
|
+
# 把 web_general 列为「安全族」,于是那些只在 config 里声明了族的垂直源
|
|
236
|
+
# (如 doi=academic)会被误判成通用源放行进来,正是 _RECOVERY_SAFE_FAMILIES
|
|
237
|
+
# 想挡住的东西。
|
|
238
|
+
return family_of(name, spec)
|
|
239
|
+
except Exception:
|
|
240
|
+
return "web_general"
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _recovery_engine_allowed(eng: str, tried_families: set[str]) -> bool:
|
|
244
|
+
"""L3/L4 备选是否安全:通用/百科,或与已试引擎同族。"""
|
|
245
|
+
fam = _engine_family(eng)
|
|
246
|
+
return fam in _RECOVERY_SAFE_FAMILIES or fam in tried_families
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def pick_alternative_engines(tried: list[str], engines_fallback: list[str] | None,
|
|
250
|
+
enabled: set[str] | None = None,
|
|
251
|
+
max_n: int = 2, mode: str = "auto") -> list[str]:
|
|
252
|
+
"""L3:挑选未尝试过的备选引擎。
|
|
253
|
+
|
|
254
|
+
优先通用免费组合(anysearch/ddg/bing/wiki),再同族垂直,再路由 fallback。
|
|
255
|
+
禁止 code/行情/快讯等与查询域无关的垂直源污染结果。
|
|
256
|
+
"""
|
|
257
|
+
tried_set = set(tried)
|
|
258
|
+
tried_families = {_engine_family(e) for e in tried}
|
|
259
|
+
picks: list[str] = []
|
|
260
|
+
|
|
261
|
+
def _try_add(eng: str) -> bool:
|
|
262
|
+
if eng in tried_set or eng in picks:
|
|
263
|
+
return False
|
|
264
|
+
if enabled is not None and eng not in enabled:
|
|
265
|
+
return False
|
|
266
|
+
if not _recovery_engine_allowed(eng, tried_families):
|
|
267
|
+
return False
|
|
268
|
+
picks.append(eng)
|
|
269
|
+
return len(picks) >= max_n
|
|
270
|
+
|
|
271
|
+
# 1) 通用免费源优先(避免 fallback 全量 enabled 把 pypi/npm 排到前面)
|
|
272
|
+
for eng in _GENERAL_FREE_COMBO:
|
|
273
|
+
if _try_add(eng):
|
|
274
|
+
return picks
|
|
275
|
+
|
|
276
|
+
# 2) 同族垂直(如 knowledge 补 wikidata;sports 补其它体育源)。候选 = 域声明
|
|
277
|
+
# engines_fallback + family_candidates 按已试族从全注册表激活的启用源(含
|
|
278
|
+
# 「单跑可用但域 combo 选不中」的孤儿源)——family_candidates 正是设计为此的
|
|
279
|
+
# 「能力族×语言×模式」单一取源口,此处首次接线。仍过 _engine_family 同族门 +
|
|
280
|
+
# _try_add 安全门(_recovery_engine_allowed)。family_candidates 不可用时退回
|
|
281
|
+
# engines_fallback(失败安全,不影响原有恢复链)。
|
|
282
|
+
_same_fam: list[str] = list(engines_fallback or [])
|
|
283
|
+
try:
|
|
284
|
+
from engine_families import family_candidates
|
|
285
|
+
for _fam in sorted(tried_families): # set 迭代序不定,候选顺序必须确定
|
|
286
|
+
for _e in family_candidates(_fam, "*", enabled=enabled, mode=mode):
|
|
287
|
+
if _e not in _same_fam:
|
|
288
|
+
_same_fam.append(_e)
|
|
289
|
+
except Exception:
|
|
290
|
+
pass
|
|
291
|
+
for eng in _same_fam:
|
|
292
|
+
if _engine_family(eng) in tried_families and _try_add(eng):
|
|
293
|
+
return picks
|
|
294
|
+
|
|
295
|
+
# 3) 其余安全 fallback(仍过 family 检查)
|
|
296
|
+
for eng in engines_fallback or []:
|
|
297
|
+
if _try_add(eng):
|
|
298
|
+
return picks
|
|
299
|
+
return picks
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
# ── L4 跨语言回退(原中英互译启发式升级)───────────────────────────────────────
|
|
303
|
+
|
|
304
|
+
# 非拉丁 / 中文主语言零结果时,用通用源补搜(原文即可,无需真翻译)
|
|
305
|
+
_CROSS_LANG_ENGINES = ["anysearch", "local_bing", "wikipedia"]
|
|
306
|
+
# 中英基线反向:英文主查询零结果且用户偏好含 zh 时,补中文源
|
|
307
|
+
_CN_BASELINE_ENGINES = ["local_bing", "anysearch", "bocha"]
|
|
308
|
+
|
|
309
|
+
_CROSS_LANG_SIGNAL = re.compile(
|
|
310
|
+
r"[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af\u0400-\u04ff\u0e00-\u0e7f"
|
|
311
|
+
r"\u0600-\u06ff\u0590-\u05ff\u0370-\u03ff\u0900-\u097f]"
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _is_non_latin_script(query: str) -> bool:
|
|
316
|
+
"""是否含非拉丁书写系统字符(日韩/西里尔/泰/阿/希伯来/希腊/天城体等)。"""
|
|
317
|
+
return bool(_CROSS_LANG_SIGNAL.search(query))
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def translate_heuristic(query: str) -> Optional[str]:
|
|
321
|
+
"""L4:从中文/日韩查询抽英文 token 组新查询;或用同义表做中→英映射。
|
|
322
|
+
|
|
323
|
+
非真翻译:仅提取已有英文 token + 同义表英文对照,供英文源二次检索。
|
|
324
|
+
无可用英文信号时返回 None。
|
|
325
|
+
"""
|
|
326
|
+
en_tokens = [w for w in _EN_WORD.findall(query)]
|
|
327
|
+
mapped: list[str] = []
|
|
328
|
+
table = _load_synonyms()["synonyms"]
|
|
329
|
+
for term, syns in table.items():
|
|
330
|
+
if term in query:
|
|
331
|
+
for s in syns:
|
|
332
|
+
if s.isascii():
|
|
333
|
+
mapped.append(s)
|
|
334
|
+
break
|
|
335
|
+
combined = []
|
|
336
|
+
for t in en_tokens + mapped:
|
|
337
|
+
if t not in combined:
|
|
338
|
+
combined.append(t)
|
|
339
|
+
if not combined:
|
|
340
|
+
return None
|
|
341
|
+
return " ".join(combined)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def cross_lang_query(query: str, primary_lang: str = "") -> tuple[Optional[str], str]:
|
|
345
|
+
"""跨语言回退:构造补搜查询与引擎组合。
|
|
346
|
+
|
|
347
|
+
规则:
|
|
348
|
+
- 主语言为英文/拉丁且无非拉丁字符 → 无需跨语言回退,返回 (None, "")。
|
|
349
|
+
- 含非拉丁书写系统(或主语言为非拉丁)→ 返回 (query, 通用源组合),
|
|
350
|
+
用原文在 duckduckgo/anysearch/wikipedia 补搜(无需真翻译)。
|
|
351
|
+
|
|
352
|
+
返回 (补搜查询, 引擎列表逗号串);无需回退时返回 (None, "")。
|
|
353
|
+
"""
|
|
354
|
+
if not query:
|
|
355
|
+
return None, ""
|
|
356
|
+
non_latin = _is_non_latin_script(query)
|
|
357
|
+
if primary_lang:
|
|
358
|
+
# 显式主语言:拉丁系且无非拉丁字符才跳过
|
|
359
|
+
if primary_lang in ("en", "latin") and not non_latin:
|
|
360
|
+
return None, ""
|
|
361
|
+
# mixed 纯拉丁也跳过;mixed 含汉字/假名等仍回退
|
|
362
|
+
if primary_lang == "mixed" and not non_latin:
|
|
363
|
+
return None, ""
|
|
364
|
+
elif not non_latin:
|
|
365
|
+
return None, ""
|
|
366
|
+
return query, ",".join(_CROSS_LANG_ENGINES)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def baseline_cross_query(
|
|
370
|
+
query: str,
|
|
371
|
+
primary_lang: str = "",
|
|
372
|
+
prefer: list[str] | None = None,
|
|
373
|
+
) -> tuple[Optional[str], str]:
|
|
374
|
+
"""中英基线补搜:英文主查询零结果时,若偏好含 zh,用中文源再试原文。
|
|
375
|
+
|
|
376
|
+
对称于 cross_lang_query(非拉丁→通用英源)。仅在 prefer 含 zh 时触发,
|
|
377
|
+
避免纯英文市场用户被强塞中文源。
|
|
378
|
+
"""
|
|
379
|
+
if not query:
|
|
380
|
+
return None, ""
|
|
381
|
+
lang = primary_lang
|
|
382
|
+
if not lang:
|
|
383
|
+
try:
|
|
384
|
+
from lang_detect import detect_language
|
|
385
|
+
lang = detect_language(query)
|
|
386
|
+
except ImportError:
|
|
387
|
+
lang = "en"
|
|
388
|
+
if lang not in ("en", "latin"):
|
|
389
|
+
return None, ""
|
|
390
|
+
if _is_non_latin_script(query):
|
|
391
|
+
return None, ""
|
|
392
|
+
prefs = prefer
|
|
393
|
+
if prefs is None:
|
|
394
|
+
try:
|
|
395
|
+
from lang_pref import prefer_langs
|
|
396
|
+
prefs = prefer_langs(query_lang=lang)
|
|
397
|
+
except ImportError:
|
|
398
|
+
prefs = ["zh", "en"]
|
|
399
|
+
if "zh" not in (prefs or []):
|
|
400
|
+
return None, ""
|
|
401
|
+
return query, ",".join(_CN_BASELINE_ENGINES)
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
# ── 计划生成 ──────────────────────────────────────────────────────────────────
|
|
405
|
+
|
|
406
|
+
def build_recovery_plan(query: str, tried_engines: list[str],
|
|
407
|
+
engines_fallback: list[str] | None = None,
|
|
408
|
+
enabled: set[str] | None = None,
|
|
409
|
+
mode: str = "auto",
|
|
410
|
+
max_level: str | None = None) -> list[RecoveryStep]:
|
|
411
|
+
"""构造有序恢复步骤(L1→L5)。
|
|
412
|
+
|
|
413
|
+
Args:
|
|
414
|
+
query: 已经失败(零结果)的查询。
|
|
415
|
+
tried_engines: 已尝试的引擎列表。
|
|
416
|
+
engines_fallback: 路由给出的未使用引擎候选。
|
|
417
|
+
enabled: 当前可用引擎集合(用于过滤 L3 候选)。
|
|
418
|
+
mode: 预算模式;fast 仅允许 L1 + L3,禁止 L4。
|
|
419
|
+
|
|
420
|
+
Returns:
|
|
421
|
+
RecoveryStep 列表(可能为空)。
|
|
422
|
+
"""
|
|
423
|
+
steps: list[RecoveryStep] = []
|
|
424
|
+
base_engines = list(tried_engines) or ["anysearch"]
|
|
425
|
+
|
|
426
|
+
# 本轮主语言:供 L4 跨语 / 中英基线判断
|
|
427
|
+
primary_lang = ""
|
|
428
|
+
try:
|
|
429
|
+
from lang_detect import detect_language
|
|
430
|
+
primary_lang = detect_language(query)
|
|
431
|
+
except ImportError:
|
|
432
|
+
pass
|
|
433
|
+
|
|
434
|
+
prefer: list[str] = []
|
|
435
|
+
try:
|
|
436
|
+
from lang_pref import prefer_langs
|
|
437
|
+
prefer = prefer_langs(query_lang=primary_lang)
|
|
438
|
+
except ImportError:
|
|
439
|
+
prefer = ["zh", "en"]
|
|
440
|
+
|
|
441
|
+
# L1 放宽
|
|
442
|
+
# 止损:显式指定单一引擎且无备选时,L1 放宽仍在同一引擎上重试,白烧一轮 RTT。
|
|
443
|
+
# 只在有备选引擎(L3 可能换引擎)时才做 L1 同引擎重试。
|
|
444
|
+
_has_fallback = bool(engines_fallback) or len(tried_engines) > 1
|
|
445
|
+
relaxed = relax_query(query)
|
|
446
|
+
if relaxed and relaxed != query and _has_fallback:
|
|
447
|
+
steps.append(RecoveryStep(
|
|
448
|
+
level="L1", strategy="relax", query=relaxed, engines=base_engines,
|
|
449
|
+
reason="去停用词/引号/尾修饰后重试"))
|
|
450
|
+
|
|
451
|
+
# L1.5 结构化语法放宽(结果归零时逆序撤条件,每步只撤一类)
|
|
452
|
+
for _i, rq in enumerate(structured_relax_steps(query, max_steps=4)):
|
|
453
|
+
if rq == query:
|
|
454
|
+
continue
|
|
455
|
+
steps.append(RecoveryStep(
|
|
456
|
+
level="L1", strategy="drop_syntax", query=rq, engines=base_engines,
|
|
457
|
+
reason="结构化语法放宽(按「排除→热度→媒体→语言→日期→账号→短语」撤一类条件)"))
|
|
458
|
+
|
|
459
|
+
# L2 同义(fast 也允许,成本低)
|
|
460
|
+
syn = synonym_expand(query)
|
|
461
|
+
if syn and syn != query:
|
|
462
|
+
steps.append(RecoveryStep(
|
|
463
|
+
level="L2", strategy="synonym", query=syn, engines=base_engines,
|
|
464
|
+
reason="同义词替换重试"))
|
|
465
|
+
|
|
466
|
+
# L3 换引擎
|
|
467
|
+
alt = pick_alternative_engines(tried_engines, engines_fallback, enabled, mode=mode)
|
|
468
|
+
if alt:
|
|
469
|
+
steps.append(RecoveryStep(
|
|
470
|
+
level="L3", strategy="switch_engine", query=relaxed or query,
|
|
471
|
+
engines=alt, reason=f"切换未试引擎: {','.join(alt)}"))
|
|
472
|
+
|
|
473
|
+
# L4 跨语言回退(fast 禁止)
|
|
474
|
+
if mode != "fast":
|
|
475
|
+
# 非拉丁 / 中文 → 通用英源补搜
|
|
476
|
+
cross_q, cross_engines = cross_lang_query(query, primary_lang=primary_lang)
|
|
477
|
+
if cross_engines:
|
|
478
|
+
steps.append(RecoveryStep(
|
|
479
|
+
level="L4", strategy="cross_lang",
|
|
480
|
+
query=cross_q or query, engines=cross_engines.split(","),
|
|
481
|
+
reason="跨语言回退:非英文主语言用通用源补搜"))
|
|
482
|
+
# 中英基线反向:英文查询 + 偏好含 zh → 中文源补搜
|
|
483
|
+
base_q, base_engines_s = baseline_cross_query(
|
|
484
|
+
query, primary_lang=primary_lang, prefer=prefer)
|
|
485
|
+
if base_engines_s:
|
|
486
|
+
eng_list = [e for e in base_engines_s.split(",")
|
|
487
|
+
if enabled is None or e in enabled]
|
|
488
|
+
if not eng_list:
|
|
489
|
+
eng_list = base_engines_s.split(",")
|
|
490
|
+
steps.append(RecoveryStep(
|
|
491
|
+
level="L4", strategy="baseline_zh",
|
|
492
|
+
query=base_q or query, engines=eng_list,
|
|
493
|
+
reason="中英基线:英文主查询零结果,偏好含 zh 时补中文源"))
|
|
494
|
+
# 中英互译启发式(有英文 token / 同义表映射时)
|
|
495
|
+
trans = translate_heuristic(query)
|
|
496
|
+
if trans and trans.lower() != query.lower():
|
|
497
|
+
trans_engines = pick_alternative_engines(
|
|
498
|
+
[], engines_fallback,
|
|
499
|
+
enabled, max_n=2, mode=mode) or ["duckduckgo", "wikipedia"]
|
|
500
|
+
steps.append(RecoveryStep(
|
|
501
|
+
level="L4", strategy="translate", query=trans,
|
|
502
|
+
engines=trans_engines, reason="中英互译启发式重试"))
|
|
503
|
+
|
|
504
|
+
# fast 模式:仅保留 L1 + L3
|
|
505
|
+
if mode == "fast":
|
|
506
|
+
steps = [s for s in steps if s.level in ("L1", "L3")]
|
|
507
|
+
|
|
508
|
+
# 复杂度门控:low 查询只保留低成本放宽(L1/L2),禁高价多源/跨语言(L3/L4)
|
|
509
|
+
if max_level:
|
|
510
|
+
_TIER = {"L1": 0, "L2": 1, "L3": 2, "L4": 3}
|
|
511
|
+
cut = _TIER.get(max_level, 3)
|
|
512
|
+
steps = [s for s in steps if _TIER.get(s.level, 3) <= cut]
|
|
513
|
+
|
|
514
|
+
return steps
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
# ── 执行调度(供 search.py 调用)───────────────────────────────────────────────
|
|
518
|
+
|
|
519
|
+
# recovery 接受结果前的停用词(不含 WHO 等缩写;缩写靠 isupper 保留)
|
|
520
|
+
_REC_SIGNAL_STOP = {
|
|
521
|
+
"the", "a", "an", "of", "for", "to", "in", "on", "at", "is", "are", "was", "were",
|
|
522
|
+
"how", "what", "which", "when", "where", "whom", "whose", "why", "who",
|
|
523
|
+
"and", "or", "with", "from", "that", "this", "year", "years", "founded", "founding",
|
|
524
|
+
"headquarters", "please", "latest", "about",
|
|
525
|
+
"请问", "怎么", "如何", "关于", "有关", "年份", "时间", "成立", "创办", "创立",
|
|
526
|
+
"总部", "职能", "简介", "最新", "详细",
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def _result_has_query_signal(query: str, item: dict[str, Any]) -> bool:
|
|
531
|
+
"""恢复结果须与查询有实质 token 重叠,避免 Chegg/无关广告被当成恢复成功。"""
|
|
532
|
+
if not query or not isinstance(item, dict):
|
|
533
|
+
return False
|
|
534
|
+
blob = f"{item.get('title', '')} {item.get('snippet', '')} {item.get('url', '')}".lower()
|
|
535
|
+
if not blob.strip():
|
|
536
|
+
return False
|
|
537
|
+
# 中文走 2-gram 展开(cjk_tokens):旧写法 `[\u4e00-\u9fff]{2,}` 会把无空格
|
|
538
|
+
# 中文整句吃成一个 token,子串包含恒假 → 真实结果全被拒 → 中文查询的恢复
|
|
539
|
+
# 链路形同不存在(实测 6 个无空格中文查询:旧判据通过 0~2/5,展开后 5/5)。
|
|
540
|
+
keys = signal_tokens(query, stop=_REC_SIGNAL_STOP)
|
|
541
|
+
if not keys:
|
|
542
|
+
return True # 无可用信号时不拦
|
|
543
|
+
return any(k in blob for k in keys)
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def run_recovery(query: str, tried_engines: list[str],
|
|
547
|
+
executor: Callable[[str, list[str]], list[dict[str, Any]]],
|
|
548
|
+
engines_fallback: list[str] | None = None,
|
|
549
|
+
enabled: set[str] | None = None,
|
|
550
|
+
mode: str = "auto",
|
|
551
|
+
max_level: str | None = None) -> tuple[list[dict[str, Any]], RecoveryResult]:
|
|
552
|
+
"""执行恢复决策树,返回 (结果列表, 结构化 RecoveryResult)。
|
|
553
|
+
|
|
554
|
+
executor(query, engines) 必须返回结果 list(可空),不得抛异常。
|
|
555
|
+
|
|
556
|
+
Returns:
|
|
557
|
+
(results, recovery_result):results 为首个非空恢复结果(否则空列表)。
|
|
558
|
+
"""
|
|
559
|
+
result = RecoveryResult(triggered=True, final_query=query)
|
|
560
|
+
plan = build_recovery_plan(
|
|
561
|
+
query, tried_engines, engines_fallback, enabled, mode, max_level=max_level)
|
|
562
|
+
|
|
563
|
+
if not plan:
|
|
564
|
+
result.note = "无可用恢复策略"
|
|
565
|
+
return [], result
|
|
566
|
+
|
|
567
|
+
for step in plan:
|
|
568
|
+
try:
|
|
569
|
+
hits = executor(step.query, step.engines)
|
|
570
|
+
except Exception as e: # 显式记录执行异常,不静默
|
|
571
|
+
result.steps_tried.append({
|
|
572
|
+
"level": step.level, "strategy": step.strategy,
|
|
573
|
+
"query": step.query, "engines": step.engines,
|
|
574
|
+
"outcome": f"executor-error: {type(e).__name__}",
|
|
575
|
+
})
|
|
576
|
+
continue
|
|
577
|
+
goods = [r for r in (hits or [])
|
|
578
|
+
if isinstance(r, dict) and "error" not in r
|
|
579
|
+
and _result_has_query_signal(step.query, r)]
|
|
580
|
+
result.steps_tried.append({
|
|
581
|
+
"level": step.level, "strategy": step.strategy,
|
|
582
|
+
"query": step.query, "engines": step.engines,
|
|
583
|
+
"outcome": f"{len(goods)} results",
|
|
584
|
+
})
|
|
585
|
+
if goods:
|
|
586
|
+
result.recovered = True
|
|
587
|
+
result.level_used = step.level
|
|
588
|
+
result.strategy_used = step.strategy
|
|
589
|
+
result.final_query = step.query
|
|
590
|
+
result.note = f"经 {step.level}/{step.strategy} 恢复成功"
|
|
591
|
+
return goods, result
|
|
592
|
+
|
|
593
|
+
result.note = "所有恢复策略均无结果"
|
|
594
|
+
return [], result
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
# ── CLI 测试 ──────────────────────────────────────────────────────────────────
|
|
598
|
+
|
|
599
|
+
if __name__ == "__main__":
|
|
600
|
+
import sys
|
|
601
|
+
|
|
602
|
+
tests = sys.argv[1:] if len(sys.argv) > 1 else [
|
|
603
|
+
'"贵州茅台" 2025 最新财报 详细',
|
|
604
|
+
"英伟达 财报 请问怎么看",
|
|
605
|
+
"React 教程 入门",
|
|
606
|
+
]
|
|
607
|
+
for q in tests:
|
|
608
|
+
print(f"原始: {q}")
|
|
609
|
+
print(f" L1 放宽: {relax_query(q)}")
|
|
610
|
+
print(f" L2 同义: {synonym_expand(q)}")
|
|
611
|
+
print(f" L4 译词: {translate_heuristic(q)}")
|
|
612
|
+
plan = build_recovery_plan(q, ["anysearch"], ["duckduckgo", "wikipedia"],
|
|
613
|
+
mode="auto")
|
|
614
|
+
for s in plan:
|
|
615
|
+
print(f" [{s.level}] {s.strategy}: q='{s.query}' engines={s.engines}")
|
|
616
|
+
print("-" * 60)
|