argo-search 1.0.1 → 2.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -21
- package/README.en.md +822 -0
- package/README.es.md +864 -0
- package/README.ja.md +864 -0
- package/README.ko.md +864 -0
- package/README.md +867 -320
- package/SKILL.md +135 -0
- package/assets/readme/hero.svg +121 -0
- package/assets/readme/made-with-beautify.svg +10 -0
- package/assets/readme/proof-routes.svg +73 -0
- package/assets/readme/settings-keys.png +0 -0
- package/assets/readme/settings-tools.png +0 -0
- package/assets/readme/why-better.svg +104 -0
- package/assets/readme/workflow.svg +83 -0
- package/backends/domain_profiles.json +2308 -96
- package/backends/engine_registry.yaml +3314 -503
- package/backends/query_synonyms_cn.json +35 -0
- package/backends/quota_profiles.json +2409 -161
- package/backends/source_types_cn.json +83 -0
- package/backends/text_lexicon.json +44 -0
- package/bin/argo +703 -0
- package/bin/argo.js +71 -39
- package/config.yaml +4976 -363
- package/cordis.patch.yml +21 -0
- package/data/countries.json +40 -0
- package/data/lang_matrix/lang_capability.json +842 -0
- package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
- package/data/regions_cn.json +1 -0
- package/docs/ADDING_NEW_ENGINE.md +184 -0
- package/docs/ARGO_INTRO.md +124 -0
- package/docs/DATA_SOURCE_GAPS.md +81 -0
- package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
- package/docs/ENGINE_CATALOG.md +564 -0
- package/docs/MCP_SETUP.md +106 -0
- package/docs/PLATFORM_PATHS.md +111 -0
- package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
- package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
- package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
- package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
- package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
- package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
- package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
- package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
- package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
- package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
- package/docs/SEARCH_ARCHIVE.md +115 -0
- package/docs/assets/architecture.svg +104 -0
- package/docs/assets/hero.svg +98 -0
- package/docs/engines/bangumi.md +27 -0
- package/docs/engines/biorxiv.md +27 -0
- package/docs/engines/clawhub.md +27 -0
- package/docs/engines/crt_sh.md +27 -0
- package/docs/engines/datacite.md +27 -0
- package/docs/engines/deps_dev.md +27 -0
- package/docs/engines/douban_movie.md +27 -0
- package/docs/engines/endoflife.md +27 -0
- package/docs/engines/flk_law.md +27 -0
- package/docs/engines/fxtwitter.md +27 -0
- package/docs/engines/google_news.md +27 -0
- package/docs/engines/gov_regulations.md +27 -0
- package/docs/engines/guqian.md +27 -0
- package/docs/engines/hackernews.md +27 -0
- package/docs/engines/iplant.md +27 -0
- package/docs/engines/keenable.md +27 -0
- package/docs/engines/met_museum.md +27 -0
- package/docs/engines/nasa_images.md +27 -0
- package/docs/engines/netease_music.md +27 -0
- package/docs/engines/npm.md +27 -0
- package/docs/engines/nvd.md +27 -0
- package/docs/engines/openreview.md +27 -0
- package/docs/engines/openstd.md +27 -0
- package/docs/engines/parallel_free.md +27 -0
- package/docs/engines/people_daily.md +27 -0
- package/docs/engines/realtime_index.md +27 -0
- package/docs/engines/sec_edgar.md +27 -0
- package/docs/engines/seltz.md +27 -0
- package/docs/engines/skillsmp.md +27 -0
- package/docs/engines/sspai.md +27 -0
- package/docs/engines/std_samr.md +27 -0
- package/docs/engines/train.md +27 -0
- package/docs/engines/un_comtrade.md +27 -0
- package/docs/engines/wikisource.md +27 -0
- package/docs/engines/zdic.md +27 -0
- package/docs/engines/zenodo.md +27 -0
- package/docs/engines/zhihu_hot_app.md +27 -0
- package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
- package/engines/README.md +10 -0
- package/engines/_template_cli.yaml +39 -0
- package/engines/_template_http.yaml +37 -0
- package/engines/plugins/_template_plugin.py +50 -0
- package/engines/specs/_cisa_kev.yaml +54 -0
- package/engines/specs/_osv.yaml +63 -0
- package/engines/specs/aljazeera.yaml +43 -0
- package/engines/specs/antara.yaml +43 -0
- package/engines/specs/artic.yaml +20 -0
- package/engines/specs/aviation_weather.yaml +39 -0
- package/engines/specs/bangumi.yaml +21 -0
- package/engines/specs/biorxiv.yaml +24 -0
- package/engines/specs/carbon_intensity.yaml +32 -0
- package/engines/specs/cinii.yaml +44 -0
- package/engines/specs/clawhub.yaml +49 -0
- package/engines/specs/cleveland.yaml +19 -0
- package/engines/specs/cn_ai_news.yaml +43 -0
- package/engines/specs/core.yaml +35 -0
- package/engines/specs/crt_sh.yaml +38 -0
- package/engines/specs/ct24.yaml +43 -0
- package/engines/specs/datacite.yaml +27 -0
- package/engines/specs/datagov_uk.yaml +41 -0
- package/engines/specs/ddgs_images.yaml +45 -0
- package/engines/specs/deepseek_search.yaml +74 -0
- package/engines/specs/deezer.yaml +20 -0
- package/engines/specs/deps_dev.yaml +21 -0
- package/engines/specs/digup.yaml +36 -0
- package/engines/specs/doi.yaml +39 -0
- package/engines/specs/douban_movie.yaml +22 -0
- package/engines/specs/dw_news.yaml +47 -0
- package/engines/specs/egov_law.yaml +21 -0
- package/engines/specs/elpais.yaml +43 -0
- package/engines/specs/endoflife.yaml +21 -0
- package/engines/specs/energy_charts.yaml +21 -0
- package/engines/specs/factcheck_org.yaml +52 -0
- package/engines/specs/faz.yaml +43 -0
- package/engines/specs/federal_register.yaml +57 -0
- package/engines/specs/firecrawl.yaml +53 -0
- package/engines/specs/flk_law.yaml +24 -0
- package/engines/specs/folha.yaml +43 -0
- package/engines/specs/france24.yaml +44 -0
- package/engines/specs/full_fact.yaml +47 -0
- package/engines/specs/fxtwitter.yaml +31 -0
- package/engines/specs/gbfs_nyc.yaml +32 -0
- package/engines/specs/gdacs.yaml +21 -0
- package/engines/specs/gdelt.yaml +64 -0
- package/engines/specs/google_news.yaml +22 -0
- package/engines/specs/gov_regulations.yaml +38 -0
- package/engines/specs/guardian_rss.yaml +59 -0
- package/engines/specs/hurriyet.yaml +43 -0
- package/engines/specs/iplant.yaml +25 -0
- package/engines/specs/jikan.yaml +24 -0
- package/engines/specs/jolpica.yaml +20 -0
- package/engines/specs/k10plus.yaml +20 -0
- package/engines/specs/keenable.yaml +44 -0
- package/engines/specs/lefigaro.yaml +43 -0
- package/engines/specs/listenbrainz.yaml +19 -0
- package/engines/specs/matichon.yaml +43 -0
- package/engines/specs/met_museum.yaml +21 -0
- package/engines/specs/nasa_images.yaml +45 -0
- package/engines/specs/netease_music.yaml +38 -0
- package/engines/specs/nhk.yaml +43 -0
- package/engines/specs/nhtsa_vpic.yaml +20 -0
- package/engines/specs/noaa_swpc.yaml +20 -0
- package/engines/specs/nvd.yaml +38 -0
- package/engines/specs/obis.yaml +20 -0
- package/engines/specs/opencitations.yaml +52 -0
- package/engines/specs/openf1.yaml +21 -0
- package/engines/specs/openfoodfacts.yaml +45 -0
- package/engines/specs/openligadb.yaml +20 -0
- package/engines/specs/openreview.yaml +37 -0
- package/engines/specs/openstd.yaml +21 -0
- package/engines/specs/people_daily.yaml +22 -0
- package/engines/specs/realtime_index.yaml +34 -0
- package/engines/specs/ror.yaml +20 -0
- package/engines/specs/satnogs.yaml +20 -0
- package/engines/specs/sec_edgar.yaml +46 -0
- package/engines/specs/skillsmp.yaml +50 -0
- package/engines/specs/sogou_weixin_browser.yaml +30 -0
- package/engines/specs/soilgrids.yaml +25 -0
- package/engines/specs/sspai.yaml +36 -0
- package/engines/specs/stackexchange.yaml +46 -0
- package/engines/specs/std_samr.yaml +21 -0
- package/engines/specs/tass.yaml +43 -0
- package/engines/specs/tle_mirror.yaml +21 -0
- package/engines/specs/train.yaml +37 -0
- package/engines/specs/tvmaze.yaml +20 -0
- package/engines/specs/twitter_syndication.yaml +31 -0
- package/engines/specs/un_comtrade.yaml +23 -0
- package/engines/specs/unpaywall.yaml +54 -0
- package/engines/specs/vnpress.yaml +43 -0
- package/engines/specs/weather.yaml +32 -0
- package/engines/specs/who_don.yaml +21 -0
- package/engines/specs/who_gho.yaml +20 -0
- package/engines/specs/wikimedia_commons.yaml +34 -0
- package/engines/specs/wikisource.yaml +22 -0
- package/engines/specs/worms.yaml +19 -0
- package/engines/specs/x_trending.yaml +34 -0
- package/engines/specs/yna.yaml +43 -0
- package/engines/specs/ynet.yaml +43 -0
- package/engines/specs/zbmath.yaml +40 -0
- package/engines/specs/zdic.yaml +22 -0
- package/engines/specs/zenodo.yaml +27 -0
- package/engines/specs/zhihu_hot_app.yaml +35 -0
- package/mcp/clients.yaml +74 -0
- package/package.json +51 -12
- package/packages/dsh-plugin/README.md +134 -0
- package/packages/dsh-plugin/cordis.patch.yml +48 -0
- package/packages/dsh-plugin/dsh/client.js +1244 -0
- package/packages/dsh-plugin/dsh/index.js +2290 -0
- package/packages/dsh-plugin/dsh/native-tools.mjs +743 -0
- package/packages/dsh-plugin/package.json +38 -0
- package/packages/dsh-plugin/test/client.test.js +232 -0
- package/packages/dsh-plugin/test/config.test.js +30 -0
- package/packages/dsh-plugin/test/index.test.js +1338 -0
- package/references/query-templates.md +96 -0
- package/references/query-templates.yaml +118 -0
- package/scripts/ab_eval.py +219 -0
- package/scripts/ab_eval_p0p1.py +266 -0
- package/scripts/adaptive.py +271 -70
- package/scripts/answer.py +154 -0
- package/scripts/archive_run.py +591 -0
- package/scripts/argo_engine_registry.py +172 -0
- package/scripts/argo_paths.py +728 -0
- package/scripts/article.py +136 -0
- package/scripts/arxiv_source.py +201 -0
- package/scripts/badcase_ledger.py +252 -0
- package/scripts/batch_probe.py +241 -0
- package/scripts/benchmark.py +19 -0
- package/scripts/bounded_run.py +144 -0
- package/scripts/browser_auth.py +323 -0
- package/scripts/burstiness_detector.py +147 -0
- package/scripts/cache.py +1078 -67
- package/scripts/cache_guard.py +271 -0
- package/scripts/cache_key_vdom.py +56 -0
- package/scripts/cache_similarity.py +170 -0
- package/scripts/caliber_isolation.py +195 -0
- package/scripts/calibrate_lowquality.py +210 -0
- package/scripts/candidate_envelope.py +366 -0
- package/scripts/chrome_cdp.py +871 -0
- package/scripts/circuit_breaker.py +389 -0
- package/scripts/citations.py +334 -0
- package/scripts/cite_cli.py +59 -0
- package/scripts/cjk_tokens.py +95 -0
- package/scripts/clarify.py +47 -24
- package/scripts/cli_io.py +240 -0
- package/scripts/config.py +683 -45
- package/scripts/config_cache.py +402 -0
- package/scripts/conn_pool.py +220 -0
- package/scripts/content_security.py +685 -0
- package/scripts/content_signals.py +730 -0
- package/scripts/crawl.py +74 -35
- package/scripts/daemon.py +866 -0
- package/scripts/digup_bridge.py +214 -0
- package/scripts/engine_admission.py +366 -0
- package/scripts/engine_dispatch.py +1019 -0
- package/scripts/engine_env.py +534 -0
- package/scripts/engine_failure.py +411 -0
- package/scripts/engine_families.py +732 -0
- package/scripts/engine_policy.py +218 -0
- package/scripts/engine_requires.py +190 -0
- package/scripts/engine_status.py +322 -0
- package/scripts/engine_validate.py +638 -0
- package/scripts/engines.py +778 -294
- package/scripts/engines_base.py +1521 -0
- package/scripts/engines_builders.py +298 -0
- package/scripts/engines_builders_batch10.py +159 -0
- package/scripts/engines_builders_batch11.py +174 -0
- package/scripts/engines_builders_batch12.py +103 -0
- package/scripts/engines_builders_batch13.py +340 -0
- package/scripts/engines_builders_batch9.py +1468 -0
- package/scripts/engines_builders_cn.py +2283 -0
- package/scripts/engines_builders_data.py +2923 -0
- package/scripts/engines_builders_data_macro.py +775 -0
- package/scripts/engines_builders_feeds.py +255 -0
- package/scripts/engines_builders_intl.py +1239 -0
- package/scripts/engines_builders_search.py +440 -0
- package/scripts/engines_builders_tech.py +1360 -0
- package/scripts/evidence.py +588 -162
- package/scripts/evidence_loop.py +592 -0
- package/scripts/evidence_tier.py +298 -0
- package/scripts/except_sets.py +25 -0
- package/scripts/extract.py +55 -16
- package/scripts/fact_align.py +207 -0
- package/scripts/fact_crosscheck.py +230 -0
- package/scripts/fetch.py +25 -12
- package/scripts/fetch_browser.py +87 -0
- package/scripts/fetch_jsrun.py +122 -0
- package/scripts/fetch_quality.py +225 -0
- package/scripts/fetch_render_tinyfish.py +117 -0
- package/scripts/fetch_v3.py +1779 -0
- package/scripts/focus_extract.py +350 -0
- package/scripts/fulltext_store.py +169 -0
- package/scripts/fxtwitter_api.py +325 -0
- package/scripts/gen_engine_catalog.py +364 -0
- package/scripts/gen_native_tools.py +91 -0
- package/scripts/gen_routing_snapshot.py +184 -0
- package/scripts/health_check.py +140 -402
- package/scripts/health_probe.py +23 -8
- package/scripts/hot_state.py +134 -0
- package/scripts/http_client.py +1053 -0
- package/scripts/image_ops.py +430 -0
- package/scripts/image_vision/vision_probe +0 -0
- package/scripts/image_vision/vision_probe.swift +126 -0
- package/scripts/install.ps1 +148 -0
- package/scripts/install.sh +206 -0
- package/scripts/job.py +1186 -0
- package/scripts/jump_resolver.py +235 -0
- package/scripts/lang_capability.py +150 -0
- package/scripts/lang_detect.py +289 -0
- package/scripts/lang_matrix_probe.py +329 -0
- package/scripts/lang_pref.py +311 -0
- package/scripts/link_source.py +340 -0
- package/scripts/local_engines.py +321 -0
- package/scripts/local_image.py +970 -0
- package/scripts/local_seek.py +366 -0
- package/scripts/macro_countries.py +82 -0
- package/scripts/matrix_search_eval.py +1105 -0
- package/scripts/mcp_diag.py +8 -3
- package/scripts/mcp_handlers.py +1074 -0
- package/scripts/mcp_handlers_surface.py +157 -0
- package/scripts/mcp_launch.sh +122 -0
- package/scripts/mcp_server.py +118 -417
- package/scripts/mcp_setup.py +407 -0
- package/scripts/mcp_tools.py +330 -0
- package/scripts/mcp_transport.py +191 -0
- package/scripts/min_version_scan.py +335 -0
- package/scripts/net_proxy.py +258 -0
- package/scripts/network_aware.py +152 -0
- package/scripts/paper.py +274 -0
- package/scripts/pdf_extract.py +518 -0
- package/scripts/phrase_filter.py +66 -0
- package/scripts/plan.py +422 -0
- package/scripts/postprocess_signals.py +150 -0
- package/scripts/pre-commit +48 -0
- package/scripts/query_enhance.py +157 -0
- package/scripts/query_operators.py +196 -0
- package/scripts/query_rewriter.py +116 -7
- package/scripts/query_signals.py +300 -0
- package/scripts/query_templates.py +167 -0
- package/scripts/query_understanding.py +411 -0
- package/scripts/query_variants.py +298 -0
- package/scripts/quota.py +488 -76
- package/scripts/rank_signals.py +775 -0
- package/scripts/ranking_eval.py +319 -0
- package/scripts/readability_extract.py +433 -0
- package/scripts/recompute.py +415 -0
- package/scripts/recovery.py +616 -0
- package/scripts/recovery_l0.py +160 -0
- package/scripts/redskill/redskill_engine.py +332 -0
- package/scripts/regression_p0p1.py +318 -0
- package/scripts/replay_eval.py +640 -0
- package/scripts/rerank_tfidf.py +225 -0
- package/scripts/research.py +591 -403
- package/scripts/research_cli.py +361 -0
- package/scripts/research_dossier.py +492 -0
- package/scripts/research_expand.py +146 -0
- package/scripts/research_gates.py +272 -0
- package/scripts/research_report.py +209 -0
- package/scripts/research_strategy.py +37 -0
- package/scripts/research_work_packages.py +195 -0
- package/scripts/result_lang.py +311 -0
- package/scripts/robots_guard.py +259 -0
- package/scripts/route.py +921 -254
- package/scripts/route_bangs.py +40 -0
- package/scripts/route_cache.py +225 -0
- package/scripts/route_combo.py +910 -0
- package/scripts/route_domains.py +328 -0
- package/scripts/route_lang.py +605 -0
- package/scripts/route_log.py +63 -0
- package/scripts/route_policy.py +250 -0
- package/scripts/rrf_classic.py +66 -0
- package/scripts/screenshot.py +73 -0
- package/scripts/search.py +897 -405
- package/scripts/search_attribution.py +88 -0
- package/scripts/search_benchmark.py +407 -0
- package/scripts/search_cli.py +627 -0
- package/scripts/search_entry.py +389 -0
- package/scripts/search_output.py +606 -0
- package/scripts/search_pipeline.py +1077 -0
- package/scripts/search_rank.py +991 -0
- package/scripts/search_types.py +27 -12
- package/scripts/seek_locator.py +68 -0
- package/scripts/semantic_evidence.py +215 -0
- package/scripts/serp_guard.py +228 -0
- package/scripts/serp_kill_log.py +51 -0
- package/scripts/single_flight.py +120 -0
- package/scripts/social_engines/__init__.py +10 -0
- package/scripts/social_engines/bilibili_engine.py +131 -0
- package/scripts/social_engines/hackernews_engine.py +113 -0
- package/scripts/social_engines/reddit_engine.py +172 -0
- package/scripts/social_engines/twitter_engine.py +397 -0
- package/scripts/social_engines/v2ex_engine.py +101 -0
- package/scripts/social_engines/weibo_engine.py +128 -0
- package/scripts/social_engines/xiaohongshu_engine.py +137 -0
- package/scripts/social_engines/zhihu_engine.py +139 -0
- package/scripts/social_research.py +201 -0
- package/scripts/sogou_weixin_browser.py +318 -0
- package/scripts/source_ledger.py +249 -0
- package/scripts/stage_timing.py +58 -0
- package/scripts/stats_cli.py +95 -0
- package/scripts/straggler_policy.py +60 -0
- package/scripts/stylometry_detector.py +231 -0
- package/scripts/sync_backends.py +353 -0
- package/scripts/sync_installs.py +29 -0
- package/scripts/template_cli.py +127 -0
- package/scripts/text_lexicon.py +72 -0
- package/scripts/tfidf_router.py +109 -37
- package/scripts/time_utils.py +233 -0
- package/scripts/topic_research_profiles.py +643 -0
- package/scripts/train.py +737 -0
- package/scripts/tweet.py +596 -0
- package/scripts/url_canon.py +217 -0
- package/scripts/url_optimize.py +56 -0
- package/scripts/url_safety.py +236 -0
- package/scripts/usage_log.py +116 -0
- package/scripts/usage_log_clean.py +113 -0
- package/scripts/v2ex_nodes.py +311 -0
- package/scripts/variant_recall.py +188 -0
- package/scripts/watch.py +205 -0
- package/scripts/wechat_article.py +258 -0
- package/scripts/wide_research.py +402 -0
- package/scripts/wx.py +349 -0
- package/scripts/x_article_fetch.py +141 -0
- package/scripts/x_search.py +234 -0
- package/scripts/x_trending.py +189 -0
- package/scripts/yaml_load.py +67 -0
- package/sub-skills/ego-search/SKILL.md +253 -0
- package/sub-skills/ego-search/references/browser-runtime.md +43 -0
- package/sub-skills/ego-search/references/install.md +67 -0
- package/sub-skills/ego-search/references/original-ego-upgrade.md +51 -0
- package/sub-skills/ego-search/scripts/ego_search.py +800 -0
- package/sub-skills/ego-search/scripts/merge.py +226 -0
- package/sub-skills/ego-search/scripts/quality.py +64 -0
- package/sub-skills/ego-search/scripts/runtime.py +172 -0
- package/sub-skills/ego-search/scripts/safety.py +109 -0
- package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
- package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
- package/sub-skills/ego-search/tests/test_query_normalize.py +68 -0
- package/sub-skills/js-run/SKILL.md +67 -0
- package/sub-skills/js-run/scripts/jsrun.py +112 -0
- package/sub-skills/js-run/scripts/shim.js +188 -0
- package/sub-skills/js-run/tests/fixtures/challenge_like.js +27 -0
- package/sub-skills/js-run/tests/test_jsrun.py +165 -0
- package/sub-skills/local-search/SKILL.md +50 -53
- package/sub-skills/local-search/bm25.py +162 -0
- package/sub-skills/local-search/config.yaml +152 -55
- package/sub-skills/local-search/engine_registry.py +91 -26
- package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
- package/sub-skills/local-search/local_search_adapter.py +13 -2
- package/sub-skills/local-search/parse_maps.yaml +40 -42
- package/sub-skills/local-search/search_v3.py +1182 -65
- package/sub-skills/local-search/smart_router.py +47 -7
- package/sub-skills/local-seek/SKILL.md +110 -0
- package/sub-skills/local-seek/config/domains.yaml +20 -0
- package/sub-skills/local-seek/references/auto-behaviors.md +24 -0
- package/sub-skills/local-seek/references/fd-mdfind-recipes.md +62 -0
- package/sub-skills/local-seek/references/git-integration.md +54 -0
- package/sub-skills/local-seek/references/remote-handoff.md +46 -0
- package/sub-skills/local-seek/references/rg-recipes.md +96 -0
- package/sub-skills/local-seek/references/strategies.md +52 -0
- package/sub-skills/local-seek/references/structural-search.md +59 -0
- package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
- package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
- package/sub-skills/local-seek/scripts/seek.py +1465 -0
- package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
- package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
- package/sub-skills/local-seek/scripts/seek_rank.py +193 -0
- package/sub-skills/local-seek/tests/test_content_rank.py +222 -0
- package/sub-skills/local-seek/tests/test_determinism_20261006.py +145 -0
- package/sub-skills/local-seek/tests/test_noise_tier.py +101 -0
- package/sub-skills/local-seek/tests/test_pcre2_probe.py +136 -0
- package/sub-skills/local-seek/tests/test_rank_pinyin.py +77 -0
- package/sub-skills/local-seek/tests/test_regression_20261003.py +169 -0
- package/sub-skills/local-seek/tests/test_regression_20261006.py +146 -0
- package/sub-skills/local-seek/tests/test_seek_degradation.py +111 -0
- package/sub-skills/local-seek/tests/test_seek_fts.py +313 -0
package/scripts/engines.py
CHANGED
|
@@ -1,24 +1,23 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
-
"""engines.py — Unified Search v2
|
|
2
|
+
"""engines.py — Unified Search v2 引擎适配层(门面)
|
|
3
3
|
|
|
4
|
-
配置驱动 + 声明式 output_map 字段提取 + 通用 parser
|
|
5
|
-
|
|
4
|
+
配置驱动 + 声明式 output_map 字段提取 + 通用 parser 保底。
|
|
5
|
+
实现拆分:
|
|
6
|
+
- engines_base.py 公共工具 / cli / http / html / 通用解析
|
|
7
|
+
- engines_builders.py 专用引擎构建器
|
|
8
|
+
本文件仅负责 BUILDERS 注册表与 search() 入口。
|
|
6
9
|
"""
|
|
7
10
|
|
|
8
11
|
from __future__ import annotations
|
|
9
12
|
|
|
10
|
-
import
|
|
13
|
+
import contextlib
|
|
14
|
+
import inspect
|
|
11
15
|
import json
|
|
12
|
-
import logging
|
|
13
|
-
import os
|
|
14
16
|
import re
|
|
15
|
-
import subprocess
|
|
16
17
|
import sys
|
|
17
18
|
import time
|
|
18
|
-
import urllib.error
|
|
19
|
-
import urllib.request
|
|
20
19
|
from pathlib import Path
|
|
21
|
-
from typing import Any
|
|
20
|
+
from typing import Any
|
|
22
21
|
|
|
23
22
|
try:
|
|
24
23
|
from config import load_config, get_engines
|
|
@@ -26,344 +25,829 @@ except ImportError:
|
|
|
26
25
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
27
26
|
from config import load_config, get_engines
|
|
28
27
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
28
|
+
try:
|
|
29
|
+
from single_flight import engine_coalescer
|
|
30
|
+
except ImportError: # pragma: no cover
|
|
31
|
+
def engine_coalescer(): # type: ignore
|
|
32
|
+
class _Noop:
|
|
33
|
+
def run(self, key, fn, **kw):
|
|
34
|
+
return fn()
|
|
35
|
+
return _Noop()
|
|
36
|
+
|
|
37
|
+
from engines_base import (
|
|
38
|
+
safe_search,
|
|
39
|
+
_build_cli_engine,
|
|
40
|
+
_build_http_engine,
|
|
41
|
+
_build_html_engine,
|
|
42
|
+
_parse_generic,
|
|
43
|
+
_parse_text_output,
|
|
44
|
+
_parse_xml,
|
|
45
|
+
_parse_uapi,
|
|
46
|
+
_parse_semantic_scholar,
|
|
47
|
+
_ensure_engine_source,
|
|
48
|
+
_CUSTOM_JSON_PARSERS,
|
|
49
|
+
)
|
|
50
|
+
from recovery import strip_structured
|
|
51
|
+
from cli_io import dumps
|
|
52
|
+
|
|
53
|
+
# 对外/测试兼容:专用解析器与 source 纠正
|
|
54
|
+
__all__ = [
|
|
55
|
+
"search",
|
|
56
|
+
"available_engines",
|
|
57
|
+
"get_registry",
|
|
58
|
+
"safe_search",
|
|
59
|
+
"_parse_uapi",
|
|
60
|
+
"_parse_semantic_scholar",
|
|
61
|
+
"_ensure_engine_source",
|
|
62
|
+
]
|
|
63
|
+
from engines_builders import (
|
|
64
|
+
_build_who_don_engine,
|
|
65
|
+
_build_who_gho_engine,
|
|
66
|
+
_build_pubmed_engine,
|
|
67
|
+
_build_core_engine,
|
|
68
|
+
_build_gdacs_engine,
|
|
69
|
+
_build_obis_engine,
|
|
70
|
+
_build_worms_engine,
|
|
71
|
+
_build_energy_charts_engine,
|
|
72
|
+
_build_jolpica_engine,
|
|
73
|
+
_build_openf1_engine,
|
|
74
|
+
_build_openligadb_engine,
|
|
75
|
+
_build_artic_engine,
|
|
76
|
+
_build_cleveland_engine,
|
|
77
|
+
_build_tvmaze_engine,
|
|
78
|
+
_build_jikan_engine,
|
|
79
|
+
_build_deezer_engine,
|
|
80
|
+
_build_listenbrainz_engine,
|
|
81
|
+
_build_egov_law_engine,
|
|
82
|
+
_build_k10plus_engine,
|
|
83
|
+
_build_ror_engine,
|
|
84
|
+
_build_soilgrids_engine,
|
|
85
|
+
_build_noaa_swpc_engine,
|
|
86
|
+
_build_satnogs_engine,
|
|
87
|
+
_build_tle_mirror_engine,
|
|
88
|
+
_build_nhtsa_vpic_engine,
|
|
89
|
+
_build_exa_engine,
|
|
90
|
+
_build_anysearch_engine,
|
|
91
|
+
_build_parallel_engine,
|
|
92
|
+
_build_parallel_free_engine,
|
|
93
|
+
_build_seltz_engine,
|
|
94
|
+
_build_you_engine,
|
|
95
|
+
_build_em_miaoxiang_engine,
|
|
96
|
+
_build_cninfo_engine,
|
|
97
|
+
_build_sina_quote_engine,
|
|
98
|
+
_build_tencent_quote_engine,
|
|
99
|
+
_build_em_flow_engine,
|
|
100
|
+
_build_wechat_sogou_engine,
|
|
101
|
+
_build_hackernews_engine,
|
|
102
|
+
_build_stackoverflow_engine,
|
|
103
|
+
_build_google_scholar_engine,
|
|
104
|
+
_build_v2ex_engine,
|
|
105
|
+
_build_tineye_engine,
|
|
106
|
+
_build_bing_rss_engine,
|
|
107
|
+
_build_ths_hot_engine,
|
|
108
|
+
_build_cls_telegraph_engine,
|
|
109
|
+
_build_em_global_news_engine,
|
|
110
|
+
_build_eastmoney_engine,
|
|
111
|
+
_build_itotii_engine,
|
|
112
|
+
_build_baidu_hot_engine,
|
|
113
|
+
_build_toutiao_hot_engine,
|
|
114
|
+
_build_bilibili_hot_engine,
|
|
115
|
+
_build_zhihu_global_engine,
|
|
116
|
+
_build_zhihu_user_engine,
|
|
117
|
+
_build_bocha_engine,
|
|
118
|
+
_build_bocha_ai_engine,
|
|
119
|
+
_build_std_samr_engine,
|
|
120
|
+
_build_openstd_engine,
|
|
121
|
+
_build_bangumi_engine,
|
|
122
|
+
_build_so_engine,
|
|
123
|
+
_build_shenma_engine,
|
|
124
|
+
_build_qwant_engine,
|
|
125
|
+
_build_ecosia_engine,
|
|
126
|
+
_build_douban_movie_engine,
|
|
127
|
+
_build_zdic_engine,
|
|
128
|
+
_build_iplant_engine,
|
|
129
|
+
_build_people_daily_engine,
|
|
130
|
+
_build_flk_law_engine,
|
|
131
|
+
_build_wikisource_engine,
|
|
132
|
+
_build_weibo_hot_engine,
|
|
133
|
+
_build_douyin_hot_engine,
|
|
134
|
+
_build_csdn_engine,
|
|
135
|
+
_build_wallstreetcn_engine,
|
|
136
|
+
_build_weather_cn_engine,
|
|
137
|
+
_build_google_news_engine,
|
|
138
|
+
_build_met_museum_engine, _build_open_library_engine,
|
|
139
|
+
_build_weread_engine,
|
|
140
|
+
_build_douban_book_engine,
|
|
141
|
+
_build_fred_engine,
|
|
142
|
+
_build_fx_rate_engine,
|
|
143
|
+
_build_worldbank_engine,
|
|
144
|
+
_build_nbs_stats_engine,
|
|
145
|
+
_build_pubchem_engine,
|
|
146
|
+
_build_eurostat_engine,
|
|
147
|
+
_build_gbif_engine,
|
|
148
|
+
_build_rfc_editor_engine,
|
|
149
|
+
_build_twitter_syndication_engine,
|
|
150
|
+
_build_deps_dev_engine,
|
|
151
|
+
_build_endoflife_engine,
|
|
152
|
+
_build_biorxiv_engine,
|
|
153
|
+
_build_un_comtrade_engine,
|
|
154
|
+
_build_uniprot_engine,
|
|
155
|
+
_build_rcsb_pdb_engine,
|
|
156
|
+
_build_courtlistener_engine,
|
|
157
|
+
_build_gutenberg_engine,
|
|
158
|
+
_build_wayback_cdx_engine,
|
|
159
|
+
_build_usgs_engine,
|
|
160
|
+
_build_nasa_cmr_engine,
|
|
161
|
+
_build_free_dictionary_engine,
|
|
162
|
+
_build_baidu_baike_engine,
|
|
163
|
+
_build_pypi_engine,
|
|
164
|
+
_build_clinicaltrials_engine,
|
|
165
|
+
_build_openfda_engine,
|
|
166
|
+
_build_juejin_engine,
|
|
167
|
+
_build_models_dev_engine,
|
|
168
|
+
_build_finviz_engine,
|
|
169
|
+
_build_yahoo_finance_engine,
|
|
170
|
+
_build_sec_companyfacts_engine,
|
|
171
|
+
_build_frankfurter_engine,
|
|
172
|
+
_build_seeking_alpha_engine,
|
|
173
|
+
_build_qweather_engine,
|
|
174
|
+
_build_wenshu_engine,
|
|
175
|
+
_build_jin10_engine,
|
|
176
|
+
_build_octen_engine,
|
|
177
|
+
_build_imdb_engine,
|
|
178
|
+
_build_thesportsdb_engine,
|
|
179
|
+
_build_itunes_engine,
|
|
180
|
+
_build_opencorporates_engine,
|
|
181
|
+
_build_google_patents_engine,
|
|
182
|
+
_build_marginalia_engine,
|
|
183
|
+
_build_wiby_engine,
|
|
184
|
+
_build_cnii_engine,
|
|
185
|
+
_build_ndl_engine,
|
|
186
|
+
_build_kor_law_engine,
|
|
187
|
+
_build_hatena_bookmark_engine,
|
|
188
|
+
_build_dnb_engine,
|
|
189
|
+
_build_doaj_engine,
|
|
190
|
+
_build_europeana_engine,
|
|
191
|
+
_build_hal_engine,
|
|
192
|
+
_build_eu_opendata_engine,
|
|
193
|
+
_build_open_meteo_engine,
|
|
194
|
+
_build_searchmysite_engine,
|
|
195
|
+
_build_lieu_engine,
|
|
196
|
+
_build_opensky_engine,
|
|
197
|
+
_build_electricity_maps_engine,
|
|
198
|
+
_build_usda_engine,
|
|
199
|
+
_build_tatoeba_engine,
|
|
200
|
+
_build_figshare_engine,
|
|
201
|
+
_build_tencent_kline_engine,
|
|
202
|
+
_build_qq_music_engine,
|
|
203
|
+
_build_github_engine,
|
|
204
|
+
_build_rss_feed_engine,
|
|
205
|
+
)
|
|
206
|
+
# 批次十(engines_builders_batch10):engines_builders.py 聚合层暂未转出该模块,
|
|
207
|
+
# 先直连其源模块——engines_builders.py 补转出后可并入上方聚合 import。
|
|
208
|
+
# zhihu_global 用别名:与上方 engines_builders 转出的 cn 旧版同名,F811 门禁
|
|
209
|
+
# 禁止静默重定义;别名让「batch10 取代 cn 版」在注册表处显式可见。
|
|
210
|
+
from engines_builders_batch10 import ( # noqa: F401
|
|
211
|
+
_build_zhihu_global_engine as _build_zhihu_global_engine_v2,
|
|
212
|
+
)
|
|
213
|
+
# 批次十一(engines_builders_batch11):图源补强。Wikimedia Commons 的
|
|
214
|
+
# query.pages 是 pageid 字典,声明式 output_map 取不到字段,必须有 builder。
|
|
215
|
+
from engines_builders_batch11 import ( # noqa: F401
|
|
216
|
+
_build_wikimedia_commons_engine,
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
class _LazyLogger:
|
|
222
|
+
"""延迟创建 logger,避免 import logging 的重量级 import 链(traceback→dataclasses→inspect)。"""
|
|
223
|
+
_logger = None
|
|
224
|
+
|
|
225
|
+
def __getattr__(self, name: str):
|
|
226
|
+
if _LazyLogger._logger is None:
|
|
227
|
+
import logging
|
|
228
|
+
_LazyLogger._logger = logging.getLogger("unified_search.engines")
|
|
229
|
+
if not _LazyLogger._logger.handlers:
|
|
230
|
+
_LazyLogger._logger.setLevel(logging.WARNING)
|
|
231
|
+
_LazyLogger._logger.addHandler(logging.StreamHandler(sys.stderr))
|
|
232
|
+
return getattr(_LazyLogger._logger, name)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
logger = _LazyLogger()
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
# 「未知引擎」警告按名字每进程只报一次。
|
|
239
|
+
# 有界(2026-09-29):键可能来自用户/路由输入,长驻 MCP server 收到大量
|
|
240
|
+
# 非常规引擎名时,无界集合会随进程寿命线性增长。超限即整体清空——去重是
|
|
241
|
+
# best-effort,丢一轮去重只多几行重复警告,远比无界增长便宜。
|
|
242
|
+
_UNKNOWN_ENGINE_WARNED_MAX = 256
|
|
243
|
+
_unknown_engine_warned: set[str] = set()
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _warn_unknown_engine(engine: str) -> None:
|
|
247
|
+
"""未注册引擎的日志警告(同名每进程一次)。
|
|
248
|
+
|
|
249
|
+
why 去重:同一次调用里,主 combo、次域补充、恢复链会各试一遍同一个名字,
|
|
250
|
+
实测同一条警告重复 3–4 行。重复并不增加信息,只会把真信号淹掉。
|
|
251
|
+
机器可读的那一份不在这里——它走 outcome 的 `status=unknown-engine` + detail,
|
|
252
|
+
最终进响应的 `errors`(见 engine_dispatch 的前置拦截)。
|
|
253
|
+
"""
|
|
254
|
+
if engine in _unknown_engine_warned:
|
|
255
|
+
return
|
|
256
|
+
if len(_unknown_engine_warned) >= _UNKNOWN_ENGINE_WARNED_MAX:
|
|
257
|
+
_unknown_engine_warned.clear()
|
|
258
|
+
_unknown_engine_warned.add(engine)
|
|
259
|
+
logger.warning(
|
|
260
|
+
"未知引擎: %s(registry 查无此名:拼写错误,或该源未随本版发布)", engine)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def is_registered(engine: str) -> bool:
|
|
264
|
+
"""该名字在本版 registry 里是否存在。
|
|
265
|
+
|
|
266
|
+
唯一判据就是 `get_registry()`,与 `run_engine` 内部的 `registry.get()` 同源:
|
|
267
|
+
调用方(engine_dispatch 的前置拦截)必须与执行层看到同一个注册表,各写一份
|
|
268
|
+
`in` 判断迟早漂移。
|
|
269
|
+
"""
|
|
270
|
+
return engine in get_registry()
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def unknown_requested_engines(engine_override: str) -> list[str]:
|
|
274
|
+
"""显式点名的引擎里,本版 registry 查无的那些(按输入顺序,去重)。
|
|
275
|
+
|
|
276
|
+
why:`--engine nope` 此前产出
|
|
277
|
+
`status=completed / errors=[] / engines_used=['nope'] / count=0`——调用方
|
|
278
|
+
(Agent)据此判定「网上没有这个信息」并停止追问,而真相是这个名字在本版
|
|
279
|
+
registry 里根本不存在(拼写错、或该源未随本版发布)。它与「缺 API Key 的
|
|
280
|
+
静默 no-results」是同一根因,处置也应一致:变成可行动的 error。
|
|
281
|
+
|
|
282
|
+
**为什么判在请求层而不是执行层**:执行层收到的引擎名可以是测试替身
|
|
283
|
+
(test_serial_hedge 的 quick_primary、test_race_fast_budget 的 fast_garbage
|
|
284
|
+
都从 dispatch 入口走),在那里按 registry 判「未注册」会误伤替身——实测
|
|
285
|
+
11 个 dispatch 用例集体变红。而「用户点了个不存在的名字」是**请求级**的
|
|
286
|
+
事实,判在请求层才既不误伤、又落在正确的抽象层。
|
|
287
|
+
|
|
288
|
+
住在 engines.py 而不是 search.py:判据本体(is_registered)在这里,而
|
|
289
|
+
search.py 已压着 1000 行硬上限(本函数曾在那边,被体积门禁拦下)。这里只做
|
|
290
|
+
「请求串 → 查无的名字」这一层薄封装,不重复 `in` 判断。
|
|
291
|
+
"""
|
|
292
|
+
if not engine_override or engine_override == "auto":
|
|
48
293
|
return []
|
|
49
|
-
|
|
294
|
+
unknown: list[str] = []
|
|
295
|
+
for raw in str(engine_override).split(","):
|
|
296
|
+
name = raw.strip()
|
|
297
|
+
if name and name not in unknown and not is_registered(name):
|
|
298
|
+
unknown.append(name)
|
|
299
|
+
return unknown
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
@contextlib.contextmanager
|
|
303
|
+
def _sys_path_tmp(path: str):
|
|
304
|
+
"""临时把 path 加入 sys.path,退出时移除——只移除自己添加的那一次。
|
|
305
|
+
|
|
306
|
+
append 而非 insert(0):sub-skills 顶层有与 scripts 同名的模块
|
|
307
|
+
(health_check 等),插到前面会劫持 scripts 下同名模块的解析。
|
|
308
|
+
"""
|
|
309
|
+
added = path not in sys.path
|
|
310
|
+
if added:
|
|
311
|
+
sys.path.append(path)
|
|
312
|
+
try:
|
|
313
|
+
yield
|
|
314
|
+
finally:
|
|
315
|
+
if added:
|
|
316
|
+
sys.path.remove(path)
|
|
50
317
|
|
|
51
318
|
|
|
52
|
-
def
|
|
53
|
-
"""
|
|
54
|
-
try:
|
|
55
|
-
r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
|
56
|
-
if r.returncode == 0:
|
|
57
|
-
return r.stdout
|
|
58
|
-
tail = (r.stderr or "").strip()[:200]
|
|
59
|
-
logger.warning(f"引擎 {engine_name} 失败 (rc={r.returncode}): {tail}")
|
|
60
|
-
return r.stdout if r.stdout.strip() else ""
|
|
61
|
-
except subprocess.TimeoutExpired:
|
|
62
|
-
logger.warning(f"引擎 {engine_name} 超时 (>{timeout}s)")
|
|
63
|
-
except FileNotFoundError as e:
|
|
64
|
-
logger.error(f"引擎 {engine_name} CLI 缺失: {e}")
|
|
65
|
-
except Exception as e:
|
|
66
|
-
logger.error(f"引擎 {engine_name} 异常: {type(e).__name__}: {e}")
|
|
67
|
-
return ""
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
def _resolve(template: list[str] | str, query: str, n: int, **extra: Any) -> list[str] | str:
|
|
71
|
-
"""替换模板占位符。"""
|
|
72
|
-
if isinstance(template, list):
|
|
73
|
-
return [_resolve(item, query, n, **extra) for item in template]
|
|
74
|
-
s = template.replace("{query}", query).replace("{n}", str(n))
|
|
75
|
-
s = s.replace("{TIMESTAMP}", str(int(time.time())))
|
|
76
|
-
for key, val in extra.items():
|
|
77
|
-
s = s.replace(f"{{{key}}}", str(val))
|
|
78
|
-
if s.startswith("~"):
|
|
79
|
-
s = str(Path.home() / s[1:])
|
|
80
|
-
return re.sub(r"\{([A-Z_][A-Z0-9_]*)\}", lambda m: os.environ.get(m.group(1), m.group(0)), s)
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
def _extract_items(data: Any, path: str) -> list[dict]:
|
|
84
|
-
"""从嵌套 dict 按路径提取列表。"""
|
|
85
|
-
obj = data
|
|
86
|
-
for part in path.split("."):
|
|
87
|
-
if isinstance(obj, dict):
|
|
88
|
-
obj = obj.get(part, [])
|
|
89
|
-
else:
|
|
90
|
-
return []
|
|
91
|
-
return obj if isinstance(obj, list) else []
|
|
319
|
+
def _build_local_search_engine(spec: dict[str, Any]) -> Any:
|
|
320
|
+
"""进程内调用 local-search 子技能,避免 subprocess 冷启动(~300-500ms/次)。
|
|
92
321
|
|
|
322
|
+
直接 import search_v3.search_engines,复用其智能路由/健康过滤/批量并行,
|
|
323
|
+
输出与 unified-search 一致的 schema。安全降级:import 失败时回退
|
|
324
|
+
原 subprocess 调用,保证功能不丢。
|
|
325
|
+
"""
|
|
326
|
+
import os as _os
|
|
327
|
+
import subprocess as _subprocess
|
|
93
328
|
|
|
94
|
-
def _make_field_parser(path: str, fields: dict[str, str]) -> Callable:
|
|
95
|
-
"""构造声明式 parser。"""
|
|
96
|
-
def parser(data: Any) -> list[dict[str, Any]]:
|
|
97
|
-
items = _extract_items(data, path) if isinstance(data, dict) else []
|
|
98
|
-
results = []
|
|
99
|
-
for item in items:
|
|
100
|
-
if not isinstance(item, dict):
|
|
101
|
-
continue
|
|
102
|
-
r = {ok: iv[:300] if ok == "snippet" and isinstance(iv, str) else iv
|
|
103
|
-
for ok, ik in fields.items() if (iv := item.get(ik, ""))}
|
|
104
|
-
if r.get("title") or r.get("url"):
|
|
105
|
-
results.append(r)
|
|
106
|
-
return results[:10]
|
|
107
|
-
return parser
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
def _build_cli_engine(spec: dict[str, Any]) -> Any:
|
|
111
329
|
cmd_template = spec.get("cmd", [])
|
|
112
330
|
search_args = spec.get("search_args", [])
|
|
113
|
-
env_overrides = spec.get("env", {})
|
|
114
331
|
|
|
115
332
|
@safe_search
|
|
116
333
|
def _engine(query: str, n: int = 5, timeout: float = 8, mode: str = "fast", **kwargs) -> list[dict[str, Any]]:
|
|
117
|
-
|
|
118
|
-
|
|
334
|
+
# 回退路径无条件依赖 _resolve:必须放 try 外。放 try 首行时,import 失败
|
|
335
|
+
# 会被下面的 except Exception 吞掉,回退路径读 _resolve_tpl 直接
|
|
336
|
+
# UnboundLocalError——「进程内失败回退 subprocess」的语义被架空
|
|
337
|
+
# (ef4e509 同类:分支内 import 污染同函数后续读取,静态门禁
|
|
338
|
+
# test_local_import_shadows_global 依赖此处不报)。
|
|
339
|
+
from engines_base import _resolve as _resolve_tpl
|
|
340
|
+
# 进程内优先(省 subprocess 冷启动)
|
|
341
|
+
try:
|
|
342
|
+
sub_dir = Path(__file__).resolve().parent.parent / "sub-skills" / "local-search"
|
|
343
|
+
# 临时加路径 import,完事自动清理(助手语义见定义处)
|
|
344
|
+
with _sys_path_tmp(str(sub_dir)):
|
|
345
|
+
import search_v3
|
|
346
|
+
res = search_v3.search_engines(
|
|
347
|
+
query, engines=None, n=n, timeout=float(timeout),
|
|
348
|
+
# None = 读子技能 config 的 max_parallel_engines(此前硬编码 5,
|
|
349
|
+
# config 调并发对主链路径不生效);预算=单引擎超时,local-first
|
|
350
|
+
# 是「快」语义,墙钟不吃 30s 历史默认。
|
|
351
|
+
max_parallel=None, total_budget=float(timeout),
|
|
352
|
+
skip_cache=bool(kwargs.get("skip_cache", False)),
|
|
353
|
+
mode=mode,
|
|
354
|
+
since=kwargs.get("since"), until=kwargs.get("until"),
|
|
355
|
+
sort=kwargs.get("sort"),
|
|
356
|
+
)
|
|
357
|
+
results = res.get("results") or []
|
|
358
|
+
# 与子进程路径一致:每条带 _engine 标记,source 保持子引擎名
|
|
359
|
+
for r in results:
|
|
360
|
+
if isinstance(r, dict) and "error" not in r:
|
|
361
|
+
r.setdefault("_engine", r.get("source") or "local_search")
|
|
362
|
+
if results:
|
|
363
|
+
return results
|
|
364
|
+
except Exception:
|
|
365
|
+
pass # 进程内失败回退 subprocess
|
|
366
|
+
# 回退:subprocess 调用(原行为)
|
|
367
|
+
cmd = _resolve_tpl(cmd_template, query, n, mode=mode)
|
|
368
|
+
args = _resolve_tpl(search_args, query, n, mode=mode)
|
|
119
369
|
if not cmd:
|
|
120
370
|
return []
|
|
121
|
-
env =
|
|
122
|
-
env.update(
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
def _build_http_engine(spec: dict[str, Any]) -> Any:
|
|
129
|
-
"""统一 HTTP 引擎构造(GET/POST)。"""
|
|
130
|
-
url_template = spec.get("url", "")
|
|
131
|
-
headers = spec.get("headers", {"Content-Type": "application/json"})
|
|
132
|
-
query_param = spec.get("query_param", "q")
|
|
133
|
-
fmt = spec.get("format", "")
|
|
134
|
-
timeout = spec.get("timeout", 8)
|
|
135
|
-
extra_params = spec.get("extra_params", {})
|
|
136
|
-
output_map = spec.get("output_map", {})
|
|
137
|
-
is_get = spec.get("method", "GET") == "GET"
|
|
138
|
-
body_template = spec.get("body", {})
|
|
139
|
-
|
|
140
|
-
@safe_search
|
|
141
|
-
def _engine(query: str, n: int = 5, _timeout: float | None = None, depth: str = "fast", **kwargs) -> list[dict[str, Any]]:
|
|
142
|
-
to = _timeout or timeout
|
|
143
|
-
import urllib.parse as up
|
|
144
|
-
|
|
145
|
-
if is_get:
|
|
146
|
-
resolved_url = _resolve(url_template, query, n)
|
|
147
|
-
separator = "&" if "?" in resolved_url else "?"
|
|
148
|
-
full_url = f"{resolved_url}{separator}{query_param}={up.quote(query)}"
|
|
149
|
-
if fmt:
|
|
150
|
-
full_url += f"&format={fmt}"
|
|
151
|
-
for k, v in extra_params.items():
|
|
152
|
-
full_url += f"&{k}={up.quote(_resolve(str(v), query, n))}"
|
|
153
|
-
req = urllib.request.Request(full_url, headers={k: _resolve(v, query, n) for k, v in headers.items()})
|
|
154
|
-
else:
|
|
155
|
-
body: dict[str, Any] = {}
|
|
156
|
-
for k, v in body_template.items():
|
|
157
|
-
resolved = _resolve(str(v), query, n)
|
|
158
|
-
if k == "search_depth":
|
|
159
|
-
body[k] = depth
|
|
160
|
-
elif resolved.lower() == "true":
|
|
161
|
-
body[k] = True
|
|
162
|
-
elif resolved.lower() == "false":
|
|
163
|
-
body[k] = False
|
|
164
|
-
else:
|
|
165
|
-
try:
|
|
166
|
-
body[k] = int(resolved)
|
|
167
|
-
except ValueError:
|
|
168
|
-
try:
|
|
169
|
-
body[k] = float(resolved)
|
|
170
|
-
except ValueError:
|
|
171
|
-
body[k] = resolved
|
|
172
|
-
req = urllib.request.Request(url_template, data=json.dumps(body).encode("utf-8"),
|
|
173
|
-
headers={k: _resolve(v, query, n) for k, v in headers.items()})
|
|
174
|
-
|
|
371
|
+
env = _os.environ.copy()
|
|
372
|
+
env.update(spec.get("env", {}) or {})
|
|
373
|
+
proc = _subprocess.run(cmd + args, capture_output=True, text=True,
|
|
374
|
+
encoding="utf-8", errors="replace",
|
|
375
|
+
timeout=timeout, env=env)
|
|
376
|
+
if proc.returncode != 0:
|
|
377
|
+
return []
|
|
175
378
|
try:
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
if fmt == "xml":
|
|
179
|
-
return _parse_xml(raw, spec.get("_name", ""))
|
|
180
|
-
data = json.loads(raw)
|
|
181
|
-
if output_map:
|
|
182
|
-
return _make_field_parser(output_map.get("items", ""), {
|
|
183
|
-
"title": output_map.get("item_title", "title"),
|
|
184
|
-
"url": output_map.get("item_url", "url"),
|
|
185
|
-
"snippet": output_map.get("item_summary", "snippet"),
|
|
186
|
-
"source": output_map.get("item_source", "source"),
|
|
187
|
-
})(data)
|
|
188
|
-
return _parse_generic(data, spec.get("_name", ""))
|
|
189
|
-
except (urllib.error.URLError, urllib.error.HTTPError, Exception) as e:
|
|
190
|
-
logger.warning(f"HTTP 引擎失败: {e}")
|
|
379
|
+
data = json.loads(proc.stdout)
|
|
380
|
+
except (json.JSONDecodeError, ValueError):
|
|
191
381
|
return []
|
|
382
|
+
results = data.get("results") or []
|
|
383
|
+
for r in results:
|
|
384
|
+
if isinstance(r, dict) and "error" not in r:
|
|
385
|
+
r.setdefault("_engine", r.get("source") or "local_search")
|
|
386
|
+
return results
|
|
192
387
|
return _engine
|
|
193
388
|
|
|
194
389
|
|
|
195
|
-
_BUILDERS = {
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
""
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
""
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
""
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
390
|
+
_BUILDERS = {
|
|
391
|
+
"cli": _build_cli_engine,
|
|
392
|
+
"http": _build_http_engine,
|
|
393
|
+
"html": _build_html_engine,
|
|
394
|
+
"local_search": _build_local_search_engine,
|
|
395
|
+
"exa": _build_exa_engine,
|
|
396
|
+
"anysearch": _build_anysearch_engine,
|
|
397
|
+
"em_miaoxiang": _build_em_miaoxiang_engine,
|
|
398
|
+
"cninfo": _build_cninfo_engine,
|
|
399
|
+
"sina_quote": _build_sina_quote_engine,
|
|
400
|
+
"tencent_quote": _build_tencent_quote_engine,
|
|
401
|
+
"em_flow": _build_em_flow_engine,
|
|
402
|
+
"wechat_sogou": _build_wechat_sogou_engine,
|
|
403
|
+
"hackernews": _build_hackernews_engine,
|
|
404
|
+
"stackoverflow": _build_stackoverflow_engine,
|
|
405
|
+
"std_samr": _build_std_samr_engine,
|
|
406
|
+
"openstd": _build_openstd_engine,
|
|
407
|
+
"bangumi": _build_bangumi_engine,
|
|
408
|
+
"douban_movie": _build_douban_movie_engine,
|
|
409
|
+
"zdic": _build_zdic_engine,
|
|
410
|
+
"iplant": _build_iplant_engine,
|
|
411
|
+
"people_daily": _build_people_daily_engine,
|
|
412
|
+
"flk_law": _build_flk_law_engine,
|
|
413
|
+
"wikisource": _build_wikisource_engine,
|
|
414
|
+
"weibo_hot": _build_weibo_hot_engine,
|
|
415
|
+
"douyin_hot": _build_douyin_hot_engine,
|
|
416
|
+
"csdn": _build_csdn_engine,
|
|
417
|
+
"wallstreetcn": _build_wallstreetcn_engine,
|
|
418
|
+
"weather_cn": _build_weather_cn_engine,
|
|
419
|
+
"google_news": _build_google_news_engine,
|
|
420
|
+
"met_museum": _build_met_museum_engine,
|
|
421
|
+
"github": _build_github_engine,
|
|
422
|
+
"google_scholar": _build_google_scholar_engine,
|
|
423
|
+
"v2ex": _build_v2ex_engine,
|
|
424
|
+
"tineye": _build_tineye_engine,
|
|
425
|
+
"bing_rss": _build_bing_rss_engine,
|
|
426
|
+
"ths_hot": _build_ths_hot_engine,
|
|
427
|
+
"cls_telegraph": _build_cls_telegraph_engine,
|
|
428
|
+
"twitter_syndication": _build_twitter_syndication_engine,
|
|
429
|
+
"deps_dev": _build_deps_dev_engine,
|
|
430
|
+
"endoflife": _build_endoflife_engine,
|
|
431
|
+
"biorxiv": _build_biorxiv_engine,
|
|
432
|
+
"un_comtrade": _build_un_comtrade_engine,
|
|
433
|
+
"em_global_news": _build_em_global_news_engine,
|
|
434
|
+
"eastmoney": _build_eastmoney_engine,
|
|
435
|
+
"itotii": _build_itotii_engine,
|
|
436
|
+
"baidu_hot": _build_baidu_hot_engine,
|
|
437
|
+
"toutiao_hot": _build_toutiao_hot_engine,
|
|
438
|
+
"bilibili_hot": _build_bilibili_hot_engine,
|
|
439
|
+
"open_library": _build_open_library_engine,
|
|
440
|
+
"weread": _build_weread_engine,
|
|
441
|
+
"douban_book": _build_douban_book_engine,
|
|
442
|
+
"zhihu_global": _build_zhihu_global_engine_v2, # 补强版在 batch10(候选池取满 + 错误行动提示;cn 版被取代,cn 冻结在行数门禁上限)
|
|
443
|
+
"zhihu_user": _build_zhihu_user_engine,
|
|
444
|
+
"fred": _build_fred_engine,
|
|
445
|
+
"so": _build_so_engine,
|
|
446
|
+
"shenma": _build_shenma_engine,
|
|
447
|
+
"qwant": _build_qwant_engine,
|
|
448
|
+
"ecosia": _build_ecosia_engine,
|
|
449
|
+
"fx_rate": _build_fx_rate_engine,
|
|
450
|
+
"worldbank": _build_worldbank_engine,
|
|
451
|
+
"nbs_stats": _build_nbs_stats_engine,
|
|
452
|
+
"pubchem": _build_pubchem_engine,
|
|
453
|
+
"eurostat": _build_eurostat_engine,
|
|
454
|
+
"gbif": _build_gbif_engine,
|
|
455
|
+
"rfc_editor": _build_rfc_editor_engine,
|
|
456
|
+
"uniprot": _build_uniprot_engine,
|
|
457
|
+
"rcsb_pdb": _build_rcsb_pdb_engine,
|
|
458
|
+
"courtlistener": _build_courtlistener_engine,
|
|
459
|
+
"gutenberg": _build_gutenberg_engine,
|
|
460
|
+
"wayback_cdx": _build_wayback_cdx_engine,
|
|
461
|
+
"usgs": _build_usgs_engine,
|
|
462
|
+
"nasa_cmr": _build_nasa_cmr_engine,
|
|
463
|
+
"free_dictionary": _build_free_dictionary_engine,
|
|
464
|
+
"baidu_baike": _build_baidu_baike_engine,
|
|
465
|
+
"pypi": _build_pypi_engine,
|
|
466
|
+
"clinicaltrials": _build_clinicaltrials_engine,
|
|
467
|
+
"openfda": _build_openfda_engine,
|
|
468
|
+
"juejin": _build_juejin_engine,
|
|
469
|
+
"models_dev": _build_models_dev_engine,
|
|
470
|
+
"finviz": _build_finviz_engine,
|
|
471
|
+
"yahoo_finance": _build_yahoo_finance_engine,
|
|
472
|
+
"sec_companyfacts": _build_sec_companyfacts_engine,
|
|
473
|
+
"frankfurter": _build_frankfurter_engine,
|
|
474
|
+
"seeking_alpha": _build_seeking_alpha_engine,
|
|
475
|
+
"qweather": _build_qweather_engine,
|
|
476
|
+
"wenshu": _build_wenshu_engine,
|
|
477
|
+
"jin10": _build_jin10_engine,
|
|
478
|
+
"octen": _build_octen_engine,
|
|
479
|
+
"bocha": _build_bocha_engine,
|
|
480
|
+
"parallel": _build_parallel_engine,
|
|
481
|
+
"parallel_free": _build_parallel_free_engine,
|
|
482
|
+
"seltz": _build_seltz_engine,
|
|
483
|
+
"you": _build_you_engine,
|
|
484
|
+
"bocha_ai": _build_bocha_ai_engine,
|
|
485
|
+
"imdb": _build_imdb_engine,
|
|
486
|
+
"thesportsdb": _build_thesportsdb_engine,
|
|
487
|
+
"itunes": _build_itunes_engine,
|
|
488
|
+
"opencorporates": _build_opencorporates_engine,
|
|
489
|
+
"google_patents": _build_google_patents_engine,
|
|
490
|
+
"marginalia": _build_marginalia_engine,
|
|
491
|
+
"wiby": _build_wiby_engine,
|
|
492
|
+
"rss_feed": _build_rss_feed_engine,
|
|
493
|
+
"cnii": _build_cnii_engine,
|
|
494
|
+
"ndl": _build_ndl_engine,
|
|
495
|
+
"kor_law": _build_kor_law_engine,
|
|
496
|
+
"hatena_bookmark": _build_hatena_bookmark_engine,
|
|
497
|
+
"dnb": _build_dnb_engine,
|
|
498
|
+
"doaj": _build_doaj_engine,
|
|
499
|
+
"europeana": _build_europeana_engine,
|
|
500
|
+
"hal": _build_hal_engine,
|
|
501
|
+
"eu_opendata": _build_eu_opendata_engine,
|
|
502
|
+
"open_meteo": _build_open_meteo_engine,
|
|
503
|
+
"searchmysite": _build_searchmysite_engine,
|
|
504
|
+
"lieu": _build_lieu_engine,
|
|
505
|
+
"opensky": _build_opensky_engine,
|
|
506
|
+
"electricity_maps": _build_electricity_maps_engine,
|
|
507
|
+
"usda": _build_usda_engine,
|
|
508
|
+
"tatoeba": _build_tatoeba_engine,
|
|
509
|
+
"figshare": _build_figshare_engine,
|
|
510
|
+
"tencent_kline": _build_tencent_kline_engine,
|
|
511
|
+
"qq_music": _build_qq_music_engine,
|
|
512
|
+
"who_don": _build_who_don_engine,
|
|
513
|
+
"who_gho": _build_who_gho_engine,
|
|
514
|
+
"pubmed": _build_pubmed_engine,
|
|
515
|
+
"core": _build_core_engine,
|
|
516
|
+
"gdacs": _build_gdacs_engine,
|
|
517
|
+
"obis": _build_obis_engine,
|
|
518
|
+
"worms": _build_worms_engine,
|
|
519
|
+
"energy_charts": _build_energy_charts_engine,
|
|
520
|
+
"jolpica": _build_jolpica_engine,
|
|
521
|
+
"openf1": _build_openf1_engine,
|
|
522
|
+
"openligadb": _build_openligadb_engine,
|
|
523
|
+
"artic": _build_artic_engine,
|
|
524
|
+
"cleveland": _build_cleveland_engine,
|
|
525
|
+
"wikimedia_commons": _build_wikimedia_commons_engine,
|
|
526
|
+
"tvmaze": _build_tvmaze_engine,
|
|
527
|
+
"jikan": _build_jikan_engine,
|
|
528
|
+
"deezer": _build_deezer_engine,
|
|
529
|
+
"listenbrainz": _build_listenbrainz_engine,
|
|
530
|
+
"egov_law": _build_egov_law_engine,
|
|
531
|
+
"k10plus": _build_k10plus_engine,
|
|
532
|
+
"ror": _build_ror_engine,
|
|
533
|
+
"soilgrids": _build_soilgrids_engine,
|
|
534
|
+
"noaa_swpc": _build_noaa_swpc_engine,
|
|
535
|
+
"satnogs": _build_satnogs_engine,
|
|
536
|
+
"tle_mirror": _build_tle_mirror_engine,
|
|
537
|
+
"nhtsa_vpic": _build_nhtsa_vpic_engine,
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
# 语义型引擎:把 query 当自然语言语义检索,不识别平台原生结构化语法
|
|
541
|
+
# (from:/repo:/site:/until: 等)。对其剥掉字段只留核心词,避免污染相关性。
|
|
542
|
+
# 透传型/垂直源(local_*、social/academic/code、github 等)保持原 query,让平台语法生效。
|
|
543
|
+
# local_search 是聚合器(子引擎 local_bing 等透传平台语法),不在语义集内。
|
|
544
|
+
_SEMANTIC_ENGINES = frozenset({
|
|
545
|
+
"byted", "bocha", "anysearch", "tavily", "exa", "octen",
|
|
546
|
+
"uapi", "searxng", "parallel", "you", "bocha_ai",
|
|
547
|
+
})
|
|
548
|
+
|
|
549
|
+
# ── 实体型引擎的查询规范化(2026-09-06 live 金标教训)─────────────────────
|
|
550
|
+
# 自然语言句直达实体搜索接口会全军覆没:「NASA founding year」wikidata 0 条
|
|
551
|
+
# (裸「NASA」7 条)、「where is Eiffel Tower」同型且拖满 11s 超时、
|
|
552
|
+
# 「Cristiano Ronaldo club」thesportsdb 抖动放大。接口只吃实体名——分发层
|
|
553
|
+
# 统一剥疑问前缀与属性词,保留核心实体。只作用于实体型引擎;通用 web 引擎
|
|
554
|
+
# 不动(属性词对全文检索是有用信号)。
|
|
555
|
+
_ENTITY_QUERY_ENGINES = frozenset({"wikidata", "thesportsdb", "local_openstreetmap"})
|
|
556
|
+
_ENTITY_Q_PREFIX_RE = re.compile(
|
|
557
|
+
r"(?i)^\s*(where\s+(is|are|was|were)|where'?s|who\s+(is|are|was)|who'?s"
|
|
558
|
+
r"|what\s+(is|are)|when\s+(was|is|did)|which\s+\w+\s+(is|was)"
|
|
559
|
+
r"|哪里|在哪[里儿]?|是谁?|是什么|什么时候|哪一[个国年])\s*",
|
|
560
|
+
)
|
|
561
|
+
# 英文属性词带 \b(空格分词下词边界天然成立);中文属性词不进此表——
|
|
562
|
+
# CJK 字符全是 \w,「周杰伦专辑」「周杰伦的专辑」这类无空格连写永远撞不上
|
|
563
|
+
# 词边界(2026-09-06 审查实锤漏剥),改由下方尾部循环剥离。
|
|
564
|
+
_ENTITY_ATTR_RE = re.compile(
|
|
565
|
+
r"(?i)\b(founding year|founded|established|headquarters?|located( in)?"
|
|
566
|
+
r"|population|capital|address|club|team|stadium|league|album|discography"
|
|
567
|
+
r"|song|movie|film|director|cast)\b",
|
|
568
|
+
)
|
|
569
|
+
# 中文属性词(含繁体):只在查询尾部剥离——中文自然语序是实体名在前、
|
|
570
|
+
# 属性词在后;不碰串中,「电影频道」「歌曲排行榜」这类实体名内含属性词,
|
|
571
|
+
# 串中剥离会误伤。长词优先,避免「成立年份」被「成立」截断残留「年份」。
|
|
572
|
+
_ENTITY_ATTR_TAILS = (
|
|
573
|
+
"成立年份", "成立", "创立", "創立", "总部", "總部", "位置", "球队", "球隊",
|
|
574
|
+
"俱乐部", "俱樂部", "职能", "職能", "职责", "職責", "专辑", "專輯",
|
|
575
|
+
"歌曲", "电影", "電影", "导演", "導演", "主演",
|
|
576
|
+
)
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def _entity_core_query(query: str) -> str:
|
|
580
|
+
"""实体搜索接口的查询规范化:剥疑问前缀+属性词,保留核心实体名。"""
|
|
581
|
+
raw = (query or "").strip()
|
|
582
|
+
q = _ENTITY_Q_PREFIX_RE.sub(" ", raw)
|
|
583
|
+
# 中文疑问词多为尾部后置(「清华大学 在哪里」),与英文前缀形态分开剥
|
|
584
|
+
q = re.sub(r"(?i)(在哪里|在哪|哪儿|哪里|是什么|是谁|什么时候|有哪些)\s*[??]?\s*$",
|
|
585
|
+
" ", q)
|
|
586
|
+
q = _ENTITY_ATTR_RE.sub(" ", q)
|
|
587
|
+
# 空白归一化必须在尾部循环前:前缀/属性词剥除用空格替换,尾空格会让
|
|
588
|
+
# endswith 判空(「清华大学总部 」漏剥总部)
|
|
589
|
+
q = re.sub(r"\s+", " ", q).strip()
|
|
590
|
+
# 中文属性词尾部循环剥 + 「的」后缀收尾:「周杰伦的专辑」→「周杰伦的」
|
|
591
|
+
# →「周杰伦」。剩单字不剥(「美的」是实体,剥成「美」即误伤)
|
|
592
|
+
changed = True
|
|
593
|
+
while changed:
|
|
594
|
+
changed = False
|
|
595
|
+
for tail in _ENTITY_ATTR_TAILS:
|
|
596
|
+
if q.endswith(tail) and len(q) > len(tail) + 1:
|
|
597
|
+
q = q[:-len(tail)].strip()
|
|
598
|
+
changed = True
|
|
286
599
|
break
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
if not isinstance(i, dict):
|
|
294
|
-
continue
|
|
295
|
-
title = i.get("title", "")
|
|
296
|
-
if isinstance(title, list):
|
|
297
|
-
title = title[0] if title else ""
|
|
298
|
-
url = i.get("url", i.get("URL", i.get("html_url", "")))
|
|
299
|
-
snippet = (i.get("snippet", i.get("content", i.get("summary", i.get("description", "")))))[:300]
|
|
300
|
-
score = i.get("score", i.get("relevance_score", 0.5))
|
|
301
|
-
results.append({"title": str(title)[:200], "url": str(url),
|
|
302
|
-
"snippet": str(snippet), "score": score, "source": engine_name})
|
|
303
|
-
return results[:10]
|
|
600
|
+
if not changed and q.endswith("的") and len(q) > 3:
|
|
601
|
+
q = q[:-1].strip()
|
|
602
|
+
changed = True
|
|
603
|
+
q = re.sub(r"[??::,,]+", " ", q)
|
|
604
|
+
q = re.sub(r"\s+", " ", q).strip()
|
|
605
|
+
return q or raw
|
|
304
606
|
|
|
305
607
|
|
|
306
|
-
# ── 引擎注册表 ─────────────────────────────────────────────────────────────────
|
|
307
|
-
|
|
308
608
|
_engine_registry: dict[str, Any] = {}
|
|
609
|
+
_engine_specs: dict[str, dict[str, Any]] = {}
|
|
309
610
|
_engine_registry_loaded = False
|
|
611
|
+
_registry_stamp: float | None = None
|
|
310
612
|
|
|
311
613
|
|
|
312
614
|
def _load_registry():
|
|
313
|
-
global _engine_registry, _engine_registry_loaded
|
|
615
|
+
global _engine_registry, _engine_specs, _engine_registry_loaded, _registry_stamp
|
|
314
616
|
if _engine_registry_loaded:
|
|
315
617
|
return
|
|
316
618
|
cfg = load_config()
|
|
317
619
|
engines = get_engines(cfg)
|
|
620
|
+
# 本地个性化引擎插件(不随公开仓库提交)
|
|
621
|
+
try:
|
|
622
|
+
from local_engines import register as _register_local
|
|
623
|
+
_register_local(_BUILDERS, engines)
|
|
624
|
+
except ImportError:
|
|
625
|
+
pass
|
|
318
626
|
registry = {}
|
|
319
627
|
for name, spec in engines.items():
|
|
320
628
|
spec = dict(spec)
|
|
321
629
|
spec["_name"] = name
|
|
630
|
+
# local_search 走进程内 builder(config 里 type=cli,这里显式路由到专用实现)
|
|
631
|
+
if name == "local_search":
|
|
632
|
+
spec["type"] = "local_search"
|
|
633
|
+
# anysearch:type 已在 config.yaml 显式声明(引擎声明来源),
|
|
634
|
+
# 不再运行时硬覆盖;若旧配置缺 type 字段,保底路由到进程内 builder。
|
|
635
|
+
if name == "anysearch" and spec.get("type", "cli") not in _BUILDERS:
|
|
636
|
+
spec["type"] = "anysearch"
|
|
322
637
|
builder = _BUILDERS.get(spec.get("type", "cli"))
|
|
323
638
|
if builder:
|
|
324
639
|
registry[name] = builder(spec)
|
|
325
640
|
else:
|
|
326
641
|
logger.warning(f"未知引擎类型: {spec.get('type')} (引擎 {name})")
|
|
327
642
|
_engine_registry = registry
|
|
643
|
+
# spec 侧表:engine_env 缺 env 检测等需要原始声明(registry 值是闭包)
|
|
644
|
+
global _engine_specs
|
|
645
|
+
_engine_specs = engines
|
|
328
646
|
_engine_registry_loaded = True
|
|
647
|
+
try:
|
|
648
|
+
from config import config_stamp
|
|
649
|
+
_registry_stamp = config_stamp()
|
|
650
|
+
except ImportError:
|
|
651
|
+
_registry_stamp = None
|
|
329
652
|
|
|
330
653
|
|
|
331
654
|
def get_registry() -> dict[str, Any]:
|
|
655
|
+
"""引擎注册表。config 变更(综合 mtime 指纹)时自动重建——
|
|
656
|
+
增删引擎/改 qps/换声明等配置无需重启进程。"""
|
|
657
|
+
global _engine_registry, _engine_specs, _engine_registry_loaded, _registry_stamp
|
|
658
|
+
try:
|
|
659
|
+
from config import config_stamp
|
|
660
|
+
stamp = config_stamp()
|
|
661
|
+
except ImportError:
|
|
662
|
+
stamp = _registry_stamp
|
|
663
|
+
if _engine_registry_loaded and _registry_stamp is not None and stamp != _registry_stamp:
|
|
664
|
+
logger.info("config 变更 → 重建引擎注册表")
|
|
665
|
+
_engine_registry = {}
|
|
666
|
+
_engine_specs = {}
|
|
667
|
+
_engine_registry_loaded = False
|
|
332
668
|
_load_registry()
|
|
333
669
|
return _engine_registry
|
|
334
670
|
|
|
335
671
|
|
|
336
|
-
def
|
|
337
|
-
|
|
672
|
+
def get_engine_spec(name: str) -> dict[str, Any] | None:
|
|
673
|
+
"""返回引擎原始声明(spec),无此引擎返回 None。
|
|
338
674
|
|
|
675
|
+
registry 值是构建后的闭包,原始 spec(url/headers/required_env 等)
|
|
676
|
+
存侧表供 engine_env 缺 env 检测等调用方使用。
|
|
677
|
+
"""
|
|
678
|
+
get_registry() # 触发热重建检查,保持侧表与注册表同步
|
|
679
|
+
return _engine_specs.get(name)
|
|
339
680
|
|
|
340
|
-
|
|
341
|
-
|
|
681
|
+
|
|
682
|
+
def available_engines(routable_only: bool = False) -> list[str]:
|
|
683
|
+
"""可实例化的引擎名(有 builder)。
|
|
684
|
+
|
|
685
|
+
routable_only=True 时再按「真能用」过滤(密钥齐全、后端依赖就位、
|
|
686
|
+
未被熔断/禁用)——这是与 --list-engines --detail 的 routable 同一判定。
|
|
687
|
+
|
|
688
|
+
routable_only 此前没被实现(本函数不收参数),调用侧传进来会抛
|
|
689
|
+
TypeError 并被 except TypeError 吞掉回退到全量——`--list-engines
|
|
690
|
+
--routable-only` 因此静默返回全部引擎,用户以为筛过了。
|
|
691
|
+
"""
|
|
692
|
+
names = sorted(get_registry().keys())
|
|
693
|
+
if not routable_only:
|
|
694
|
+
return names
|
|
695
|
+
try:
|
|
696
|
+
from engine_status import list_routable_engine_ids
|
|
697
|
+
routable = set(list_routable_engine_ids())
|
|
698
|
+
return [n for n in names if n in routable]
|
|
699
|
+
except Exception:
|
|
700
|
+
return names
|
|
701
|
+
|
|
702
|
+
|
|
703
|
+
# ── 单飞合并 + 免费引擎结果数桶化 ──
|
|
704
|
+
# 场景:MCP 并发请求 / 多轨道同查询 → 同一引擎调用重复打上游。
|
|
705
|
+
# 桶化只对免费引擎(cost_factor >= 0.85,与 route 免费判定一致):n 向上
|
|
706
|
+
# snap 到 10/20/50/100,让同查询不同请求数共享一次执行与缓存;付费引擎
|
|
707
|
+
# 保持精确 n(按结果计费不得放大)。
|
|
708
|
+
_NUM_BUCKETS = (10, 20, 50, 100)
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
def _free_engine(engine: str) -> bool:
|
|
712
|
+
"""是否「免费档」——只有免费档才做 n 桶化。
|
|
713
|
+
|
|
714
|
+
此前判据是 cost_factor >= 0.85,而成本表没有 api 档的分支(fallthrough
|
|
715
|
+
到 1.0),于是 api 档(exa/octen/you/parallel/zhihu_global/tavily 这类
|
|
716
|
+
按量计费的源)被当成免费:要 5 条被放大到 10 条,按结果计费的接口直接
|
|
717
|
+
双倍计费。本函数注释一直写着「付费引擎保持精确 n(按结果计费不得放大)」,
|
|
718
|
+
实现却漏了这一档——现在按声明的档位判,不再依赖魔法阈值。
|
|
719
|
+
"""
|
|
720
|
+
try:
|
|
721
|
+
from config import cost_tier_of
|
|
722
|
+
return cost_tier_of(engine) == "free"
|
|
723
|
+
except Exception:
|
|
724
|
+
return False
|
|
725
|
+
|
|
726
|
+
|
|
727
|
+
def bucket_n(engine: str, n: int) -> int:
|
|
728
|
+
"""免费引擎:n 向上 snap 到缓存友好桶(≤100);付费引擎原样。"""
|
|
729
|
+
n = max(1, min(int(n), 100))
|
|
730
|
+
if not _free_engine(engine):
|
|
731
|
+
return n
|
|
732
|
+
for b in _NUM_BUCKETS:
|
|
733
|
+
if n <= b:
|
|
734
|
+
return b
|
|
735
|
+
return 100
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def _call_key(query: str, engine: str, n: int, kwargs: dict[str, Any],
|
|
739
|
+
*, timeout: float | None = None, depth: str = "fast",
|
|
740
|
+
mode: str = "fast") -> str:
|
|
741
|
+
"""单飞 key:同 query+engine+参数指纹才合并(时间窗/域等影响结果参数
|
|
742
|
+
必须进 key,否则不同检索被错误合并)。
|
|
743
|
+
|
|
744
|
+
2026-10-15 补齐:depth / mode / timeout 同样决定结果,此前只经闭包进
|
|
745
|
+
`_execute`、没进 key —— 跨档位并发被错误合并:deep 请求可能拿到 fast
|
|
746
|
+
的结果集(反向亦然,且 follower 还会继承 leader 的 timeout)。三者入 key
|
|
747
|
+
是治本,不是给合并器打补丁。
|
|
748
|
+
"""
|
|
749
|
+
import json as _json
|
|
750
|
+
try:
|
|
751
|
+
params = _json.dumps(kwargs, sort_keys=True, ensure_ascii=False,
|
|
752
|
+
default=str)
|
|
753
|
+
except Exception:
|
|
754
|
+
params = str(sorted(kwargs.items()))
|
|
755
|
+
# timeout 归一化:8 与 8.0 是同一次执行,不该分裂成两个 key
|
|
756
|
+
if timeout is None or isinstance(timeout, bool):
|
|
757
|
+
to_seg = str(timeout)
|
|
758
|
+
elif isinstance(timeout, (int, float)):
|
|
759
|
+
to_seg = f"{float(timeout):g}"
|
|
760
|
+
else:
|
|
761
|
+
to_seg = str(timeout)
|
|
762
|
+
return f"{engine}|{n}|{query}|{params}|{depth}|{mode}|{to_seg}"
|
|
763
|
+
|
|
764
|
+
|
|
765
|
+
def search(query: str, engine: str, n: int = 5, timeout: float = 8, depth: str = "fast", mode: str = "fast", **kwargs) -> list[dict[str, Any]]:
|
|
766
|
+
"""统一引擎调用入口;失败返回空 list,不抛异常。kwargs 透传到引擎 builder(如 since/until 时间窗)。
|
|
767
|
+
|
|
768
|
+
并发同调用(同 query+engine+参数)经进程内单飞合并为一次上游执行;
|
|
769
|
+
免费引擎 n 桶化(见 bucket_n),leader 按桶内最大 n 执行,调用方输出
|
|
770
|
+
层截断到请求数。
|
|
771
|
+
"""
|
|
342
772
|
registry = get_registry()
|
|
343
773
|
fn = registry.get(engine)
|
|
344
774
|
if not fn:
|
|
345
|
-
|
|
775
|
+
_warn_unknown_engine(engine)
|
|
346
776
|
return []
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
777
|
+
# 引擎可用性由熔断器统一裁决(dispatch 的 breaker.allow / get_negative)。
|
|
778
|
+
# 这里曾有一份 `neg:{engine}` 负缓存,但两处调用(读/写)与 SearchCache
|
|
779
|
+
# 的签名都不匹配,异常被 except 吞掉,从未生效过(2026-10-18 复核)。
|
|
780
|
+
# 不修它:单次失败就全局压制该引擎 45s,比熔断器「连续两次才打开」更激进,
|
|
781
|
+
# 会在一次网络抖动后误杀健康源。保持单一真源,避免第二套状态。
|
|
782
|
+
# 语义型引擎不识别平台结构化语法,剥掉字段只留核心词;透传型保持原 query。
|
|
783
|
+
if engine in _SEMANTIC_ENGINES:
|
|
784
|
+
query = strip_structured(query)
|
|
785
|
+
elif engine in _ENTITY_QUERY_ENGINES:
|
|
786
|
+
# 实体型引擎:自然语言句(疑问前缀/属性词)会让实体搜索接口空结果
|
|
787
|
+
query = _entity_core_query(query)
|
|
788
|
+
eff_n = bucket_n(engine, n)
|
|
789
|
+
|
|
790
|
+
# 引擎自声明 timeout 是该源的硬上限(此前被调用方 timeout 位置传参整体
|
|
791
|
+
# 覆盖成死配置——OSM timeout:6 实测跑出 10s+5s curl 守卫 11.3s)。
|
|
792
|
+
# deep 模式保持原语义:研究场景宁可等,不受 spec 短超时约束。
|
|
793
|
+
spec_to = (_engine_specs or {}).get(engine) or {}
|
|
794
|
+
st = spec_to.get("timeout") if isinstance(spec_to, dict) else None
|
|
795
|
+
call_to = timeout
|
|
796
|
+
if mode != "deep" and isinstance(st, (int, float)) and 0 < st < timeout:
|
|
797
|
+
call_to = float(st)
|
|
798
|
+
|
|
799
|
+
# key 用 call_to(实际执行时的超时)而非调用方声明的 timeout:被 spec
|
|
800
|
+
# 夹到同一上限的两次调用是同一执行,应共享单飞而不是白拆两份。
|
|
801
|
+
key = _call_key(query, engine, eff_n, kwargs,
|
|
802
|
+
timeout=call_to, depth=depth, mode=mode)
|
|
803
|
+
|
|
804
|
+
def _execute() -> list[dict[str, Any]]:
|
|
805
|
+
t0 = time.time()
|
|
351
806
|
try:
|
|
352
|
-
results = fn(query,
|
|
807
|
+
results = fn(query, eff_n, call_to, depth=depth, mode=mode, **kwargs)
|
|
353
808
|
except Exception as e:
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
809
|
+
# TypeError 可能来自引擎内部逻辑错误而非签名不匹配。
|
|
810
|
+
# 用 inspect.signature 确认引擎是否接受 depth/mode 参数,
|
|
811
|
+
# 减少误判:只有引擎函数签名明确不接受这些参数时才回退。
|
|
812
|
+
_retry_simple = False
|
|
813
|
+
if isinstance(e, TypeError):
|
|
814
|
+
try:
|
|
815
|
+
sig = inspect.signature(fn)
|
|
816
|
+
params = list(sig.parameters.keys())
|
|
817
|
+
_retry_simple = not (
|
|
818
|
+
"depth" in params or "mode" in params or "kwargs" in params
|
|
819
|
+
or any(
|
|
820
|
+
p.kind == inspect.Parameter.VAR_KEYWORD
|
|
821
|
+
for _, p in sig.parameters.items()
|
|
822
|
+
)
|
|
823
|
+
)
|
|
824
|
+
except (ValueError, TypeError):
|
|
825
|
+
_retry_simple = False
|
|
826
|
+
if _retry_simple:
|
|
827
|
+
try:
|
|
828
|
+
results = fn(query, eff_n, timeout)
|
|
829
|
+
except Exception as e2:
|
|
830
|
+
logger.error(f"引擎 {engine} 回退失败: {type(e2).__name__}: {e2}")
|
|
831
|
+
results = []
|
|
832
|
+
else:
|
|
833
|
+
logger.error(f"引擎 {engine} 异常: {type(e).__name__}: {e}")
|
|
834
|
+
results = []
|
|
835
|
+
elapsed = time.time() - t0
|
|
836
|
+
if results and isinstance(results, list):
|
|
837
|
+
for r in results:
|
|
838
|
+
if isinstance(r, dict) and "error" not in r:
|
|
839
|
+
r["_engine"] = engine
|
|
840
|
+
r["_elapsed"] = round(elapsed, 3)
|
|
841
|
+
return results if isinstance(results, list) else []
|
|
366
842
|
|
|
843
|
+
try:
|
|
844
|
+
results = engine_coalescer().run(key, _execute)
|
|
845
|
+
except Exception:
|
|
846
|
+
results = []
|
|
847
|
+
results = results or []
|
|
848
|
+
# 桶化后可能多于请求数:调用方按请求 n 截断语义由 execute_search 融合层负责;
|
|
849
|
+
# 此处保留全量(缓存键按 eff_n 存,命中时同样可截)。
|
|
850
|
+
return results
|
|
367
851
|
|
|
368
852
|
def _cli():
|
|
369
853
|
import argparse
|
|
@@ -375,11 +859,11 @@ def _cli():
|
|
|
375
859
|
parser.add_argument("--list", action="store_true")
|
|
376
860
|
args = parser.parse_args()
|
|
377
861
|
if args.list:
|
|
378
|
-
print(
|
|
862
|
+
print(dumps(available_engines()))
|
|
379
863
|
return
|
|
380
864
|
if not args.query:
|
|
381
865
|
parser.error("必须提供 query")
|
|
382
|
-
print(
|
|
866
|
+
print(dumps(search(args.query, args.engine, args.n, args.timeout)))
|
|
383
867
|
|
|
384
868
|
|
|
385
869
|
if __name__ == "__main__":
|