argo-search 1.0.1 → 2.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -21
- package/README.en.md +822 -0
- package/README.es.md +864 -0
- package/README.ja.md +864 -0
- package/README.ko.md +864 -0
- package/README.md +867 -320
- package/SKILL.md +135 -0
- package/assets/readme/hero.svg +121 -0
- package/assets/readme/made-with-beautify.svg +10 -0
- package/assets/readme/proof-routes.svg +73 -0
- package/assets/readme/settings-keys.png +0 -0
- package/assets/readme/settings-tools.png +0 -0
- package/assets/readme/why-better.svg +104 -0
- package/assets/readme/workflow.svg +83 -0
- package/backends/domain_profiles.json +2308 -96
- package/backends/engine_registry.yaml +3314 -503
- package/backends/query_synonyms_cn.json +35 -0
- package/backends/quota_profiles.json +2409 -161
- package/backends/source_types_cn.json +83 -0
- package/backends/text_lexicon.json +44 -0
- package/bin/argo +703 -0
- package/bin/argo.js +71 -39
- package/config.yaml +4976 -363
- package/cordis.patch.yml +21 -0
- package/data/countries.json +40 -0
- package/data/lang_matrix/lang_capability.json +842 -0
- package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
- package/data/regions_cn.json +1 -0
- package/docs/ADDING_NEW_ENGINE.md +184 -0
- package/docs/ARGO_INTRO.md +124 -0
- package/docs/DATA_SOURCE_GAPS.md +81 -0
- package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
- package/docs/ENGINE_CATALOG.md +564 -0
- package/docs/MCP_SETUP.md +106 -0
- package/docs/PLATFORM_PATHS.md +111 -0
- package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
- package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
- package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
- package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
- package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
- package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
- package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
- package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
- package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
- package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
- package/docs/SEARCH_ARCHIVE.md +115 -0
- package/docs/assets/architecture.svg +104 -0
- package/docs/assets/hero.svg +98 -0
- package/docs/engines/bangumi.md +27 -0
- package/docs/engines/biorxiv.md +27 -0
- package/docs/engines/clawhub.md +27 -0
- package/docs/engines/crt_sh.md +27 -0
- package/docs/engines/datacite.md +27 -0
- package/docs/engines/deps_dev.md +27 -0
- package/docs/engines/douban_movie.md +27 -0
- package/docs/engines/endoflife.md +27 -0
- package/docs/engines/flk_law.md +27 -0
- package/docs/engines/fxtwitter.md +27 -0
- package/docs/engines/google_news.md +27 -0
- package/docs/engines/gov_regulations.md +27 -0
- package/docs/engines/guqian.md +27 -0
- package/docs/engines/hackernews.md +27 -0
- package/docs/engines/iplant.md +27 -0
- package/docs/engines/keenable.md +27 -0
- package/docs/engines/met_museum.md +27 -0
- package/docs/engines/nasa_images.md +27 -0
- package/docs/engines/netease_music.md +27 -0
- package/docs/engines/npm.md +27 -0
- package/docs/engines/nvd.md +27 -0
- package/docs/engines/openreview.md +27 -0
- package/docs/engines/openstd.md +27 -0
- package/docs/engines/parallel_free.md +27 -0
- package/docs/engines/people_daily.md +27 -0
- package/docs/engines/realtime_index.md +27 -0
- package/docs/engines/sec_edgar.md +27 -0
- package/docs/engines/seltz.md +27 -0
- package/docs/engines/skillsmp.md +27 -0
- package/docs/engines/sspai.md +27 -0
- package/docs/engines/std_samr.md +27 -0
- package/docs/engines/train.md +27 -0
- package/docs/engines/un_comtrade.md +27 -0
- package/docs/engines/wikisource.md +27 -0
- package/docs/engines/zdic.md +27 -0
- package/docs/engines/zenodo.md +27 -0
- package/docs/engines/zhihu_hot_app.md +27 -0
- package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
- package/engines/README.md +10 -0
- package/engines/_template_cli.yaml +39 -0
- package/engines/_template_http.yaml +37 -0
- package/engines/plugins/_template_plugin.py +50 -0
- package/engines/specs/_cisa_kev.yaml +54 -0
- package/engines/specs/_osv.yaml +63 -0
- package/engines/specs/aljazeera.yaml +43 -0
- package/engines/specs/antara.yaml +43 -0
- package/engines/specs/artic.yaml +20 -0
- package/engines/specs/aviation_weather.yaml +39 -0
- package/engines/specs/bangumi.yaml +21 -0
- package/engines/specs/biorxiv.yaml +24 -0
- package/engines/specs/carbon_intensity.yaml +32 -0
- package/engines/specs/cinii.yaml +44 -0
- package/engines/specs/clawhub.yaml +49 -0
- package/engines/specs/cleveland.yaml +19 -0
- package/engines/specs/cn_ai_news.yaml +43 -0
- package/engines/specs/core.yaml +35 -0
- package/engines/specs/crt_sh.yaml +38 -0
- package/engines/specs/ct24.yaml +43 -0
- package/engines/specs/datacite.yaml +27 -0
- package/engines/specs/datagov_uk.yaml +41 -0
- package/engines/specs/ddgs_images.yaml +45 -0
- package/engines/specs/deepseek_search.yaml +74 -0
- package/engines/specs/deezer.yaml +20 -0
- package/engines/specs/deps_dev.yaml +21 -0
- package/engines/specs/digup.yaml +36 -0
- package/engines/specs/doi.yaml +39 -0
- package/engines/specs/douban_movie.yaml +22 -0
- package/engines/specs/dw_news.yaml +47 -0
- package/engines/specs/egov_law.yaml +21 -0
- package/engines/specs/elpais.yaml +43 -0
- package/engines/specs/endoflife.yaml +21 -0
- package/engines/specs/energy_charts.yaml +21 -0
- package/engines/specs/factcheck_org.yaml +52 -0
- package/engines/specs/faz.yaml +43 -0
- package/engines/specs/federal_register.yaml +57 -0
- package/engines/specs/firecrawl.yaml +53 -0
- package/engines/specs/flk_law.yaml +24 -0
- package/engines/specs/folha.yaml +43 -0
- package/engines/specs/france24.yaml +44 -0
- package/engines/specs/full_fact.yaml +47 -0
- package/engines/specs/fxtwitter.yaml +31 -0
- package/engines/specs/gbfs_nyc.yaml +32 -0
- package/engines/specs/gdacs.yaml +21 -0
- package/engines/specs/gdelt.yaml +64 -0
- package/engines/specs/google_news.yaml +22 -0
- package/engines/specs/gov_regulations.yaml +38 -0
- package/engines/specs/guardian_rss.yaml +59 -0
- package/engines/specs/hurriyet.yaml +43 -0
- package/engines/specs/iplant.yaml +25 -0
- package/engines/specs/jikan.yaml +24 -0
- package/engines/specs/jolpica.yaml +20 -0
- package/engines/specs/k10plus.yaml +20 -0
- package/engines/specs/keenable.yaml +44 -0
- package/engines/specs/lefigaro.yaml +43 -0
- package/engines/specs/listenbrainz.yaml +19 -0
- package/engines/specs/matichon.yaml +43 -0
- package/engines/specs/met_museum.yaml +21 -0
- package/engines/specs/nasa_images.yaml +45 -0
- package/engines/specs/netease_music.yaml +38 -0
- package/engines/specs/nhk.yaml +43 -0
- package/engines/specs/nhtsa_vpic.yaml +20 -0
- package/engines/specs/noaa_swpc.yaml +20 -0
- package/engines/specs/nvd.yaml +38 -0
- package/engines/specs/obis.yaml +20 -0
- package/engines/specs/opencitations.yaml +52 -0
- package/engines/specs/openf1.yaml +21 -0
- package/engines/specs/openfoodfacts.yaml +45 -0
- package/engines/specs/openligadb.yaml +20 -0
- package/engines/specs/openreview.yaml +37 -0
- package/engines/specs/openstd.yaml +21 -0
- package/engines/specs/people_daily.yaml +22 -0
- package/engines/specs/realtime_index.yaml +34 -0
- package/engines/specs/ror.yaml +20 -0
- package/engines/specs/satnogs.yaml +20 -0
- package/engines/specs/sec_edgar.yaml +46 -0
- package/engines/specs/skillsmp.yaml +50 -0
- package/engines/specs/sogou_weixin_browser.yaml +30 -0
- package/engines/specs/soilgrids.yaml +25 -0
- package/engines/specs/sspai.yaml +36 -0
- package/engines/specs/stackexchange.yaml +46 -0
- package/engines/specs/std_samr.yaml +21 -0
- package/engines/specs/tass.yaml +43 -0
- package/engines/specs/tle_mirror.yaml +21 -0
- package/engines/specs/train.yaml +37 -0
- package/engines/specs/tvmaze.yaml +20 -0
- package/engines/specs/twitter_syndication.yaml +31 -0
- package/engines/specs/un_comtrade.yaml +23 -0
- package/engines/specs/unpaywall.yaml +54 -0
- package/engines/specs/vnpress.yaml +43 -0
- package/engines/specs/weather.yaml +32 -0
- package/engines/specs/who_don.yaml +21 -0
- package/engines/specs/who_gho.yaml +20 -0
- package/engines/specs/wikimedia_commons.yaml +34 -0
- package/engines/specs/wikisource.yaml +22 -0
- package/engines/specs/worms.yaml +19 -0
- package/engines/specs/x_trending.yaml +34 -0
- package/engines/specs/yna.yaml +43 -0
- package/engines/specs/ynet.yaml +43 -0
- package/engines/specs/zbmath.yaml +40 -0
- package/engines/specs/zdic.yaml +22 -0
- package/engines/specs/zenodo.yaml +27 -0
- package/engines/specs/zhihu_hot_app.yaml +35 -0
- package/mcp/clients.yaml +74 -0
- package/package.json +51 -12
- package/packages/dsh-plugin/README.md +134 -0
- package/packages/dsh-plugin/cordis.patch.yml +48 -0
- package/packages/dsh-plugin/dsh/client.js +1244 -0
- package/packages/dsh-plugin/dsh/index.js +2290 -0
- package/packages/dsh-plugin/dsh/native-tools.mjs +743 -0
- package/packages/dsh-plugin/package.json +38 -0
- package/packages/dsh-plugin/test/client.test.js +232 -0
- package/packages/dsh-plugin/test/config.test.js +30 -0
- package/packages/dsh-plugin/test/index.test.js +1338 -0
- package/references/query-templates.md +96 -0
- package/references/query-templates.yaml +118 -0
- package/scripts/ab_eval.py +219 -0
- package/scripts/ab_eval_p0p1.py +266 -0
- package/scripts/adaptive.py +271 -70
- package/scripts/answer.py +154 -0
- package/scripts/archive_run.py +591 -0
- package/scripts/argo_engine_registry.py +172 -0
- package/scripts/argo_paths.py +728 -0
- package/scripts/article.py +136 -0
- package/scripts/arxiv_source.py +201 -0
- package/scripts/badcase_ledger.py +252 -0
- package/scripts/batch_probe.py +241 -0
- package/scripts/benchmark.py +19 -0
- package/scripts/bounded_run.py +144 -0
- package/scripts/browser_auth.py +323 -0
- package/scripts/burstiness_detector.py +147 -0
- package/scripts/cache.py +1078 -67
- package/scripts/cache_guard.py +271 -0
- package/scripts/cache_key_vdom.py +56 -0
- package/scripts/cache_similarity.py +170 -0
- package/scripts/caliber_isolation.py +195 -0
- package/scripts/calibrate_lowquality.py +210 -0
- package/scripts/candidate_envelope.py +366 -0
- package/scripts/chrome_cdp.py +871 -0
- package/scripts/circuit_breaker.py +389 -0
- package/scripts/citations.py +334 -0
- package/scripts/cite_cli.py +59 -0
- package/scripts/cjk_tokens.py +95 -0
- package/scripts/clarify.py +47 -24
- package/scripts/cli_io.py +240 -0
- package/scripts/config.py +683 -45
- package/scripts/config_cache.py +402 -0
- package/scripts/conn_pool.py +220 -0
- package/scripts/content_security.py +685 -0
- package/scripts/content_signals.py +730 -0
- package/scripts/crawl.py +74 -35
- package/scripts/daemon.py +866 -0
- package/scripts/digup_bridge.py +214 -0
- package/scripts/engine_admission.py +366 -0
- package/scripts/engine_dispatch.py +1019 -0
- package/scripts/engine_env.py +534 -0
- package/scripts/engine_failure.py +411 -0
- package/scripts/engine_families.py +732 -0
- package/scripts/engine_policy.py +218 -0
- package/scripts/engine_requires.py +190 -0
- package/scripts/engine_status.py +322 -0
- package/scripts/engine_validate.py +638 -0
- package/scripts/engines.py +778 -294
- package/scripts/engines_base.py +1521 -0
- package/scripts/engines_builders.py +298 -0
- package/scripts/engines_builders_batch10.py +159 -0
- package/scripts/engines_builders_batch11.py +174 -0
- package/scripts/engines_builders_batch12.py +103 -0
- package/scripts/engines_builders_batch13.py +340 -0
- package/scripts/engines_builders_batch9.py +1468 -0
- package/scripts/engines_builders_cn.py +2283 -0
- package/scripts/engines_builders_data.py +2923 -0
- package/scripts/engines_builders_data_macro.py +775 -0
- package/scripts/engines_builders_feeds.py +255 -0
- package/scripts/engines_builders_intl.py +1239 -0
- package/scripts/engines_builders_search.py +440 -0
- package/scripts/engines_builders_tech.py +1360 -0
- package/scripts/evidence.py +588 -162
- package/scripts/evidence_loop.py +592 -0
- package/scripts/evidence_tier.py +298 -0
- package/scripts/except_sets.py +25 -0
- package/scripts/extract.py +55 -16
- package/scripts/fact_align.py +207 -0
- package/scripts/fact_crosscheck.py +230 -0
- package/scripts/fetch.py +25 -12
- package/scripts/fetch_browser.py +87 -0
- package/scripts/fetch_jsrun.py +122 -0
- package/scripts/fetch_quality.py +225 -0
- package/scripts/fetch_render_tinyfish.py +117 -0
- package/scripts/fetch_v3.py +1779 -0
- package/scripts/focus_extract.py +350 -0
- package/scripts/fulltext_store.py +169 -0
- package/scripts/fxtwitter_api.py +325 -0
- package/scripts/gen_engine_catalog.py +364 -0
- package/scripts/gen_native_tools.py +91 -0
- package/scripts/gen_routing_snapshot.py +184 -0
- package/scripts/health_check.py +140 -402
- package/scripts/health_probe.py +23 -8
- package/scripts/hot_state.py +134 -0
- package/scripts/http_client.py +1053 -0
- package/scripts/image_ops.py +430 -0
- package/scripts/image_vision/vision_probe +0 -0
- package/scripts/image_vision/vision_probe.swift +126 -0
- package/scripts/install.ps1 +148 -0
- package/scripts/install.sh +206 -0
- package/scripts/job.py +1186 -0
- package/scripts/jump_resolver.py +235 -0
- package/scripts/lang_capability.py +150 -0
- package/scripts/lang_detect.py +289 -0
- package/scripts/lang_matrix_probe.py +329 -0
- package/scripts/lang_pref.py +311 -0
- package/scripts/link_source.py +340 -0
- package/scripts/local_engines.py +321 -0
- package/scripts/local_image.py +970 -0
- package/scripts/local_seek.py +366 -0
- package/scripts/macro_countries.py +82 -0
- package/scripts/matrix_search_eval.py +1105 -0
- package/scripts/mcp_diag.py +8 -3
- package/scripts/mcp_handlers.py +1074 -0
- package/scripts/mcp_handlers_surface.py +157 -0
- package/scripts/mcp_launch.sh +122 -0
- package/scripts/mcp_server.py +118 -417
- package/scripts/mcp_setup.py +407 -0
- package/scripts/mcp_tools.py +330 -0
- package/scripts/mcp_transport.py +191 -0
- package/scripts/min_version_scan.py +335 -0
- package/scripts/net_proxy.py +258 -0
- package/scripts/network_aware.py +152 -0
- package/scripts/paper.py +274 -0
- package/scripts/pdf_extract.py +518 -0
- package/scripts/phrase_filter.py +66 -0
- package/scripts/plan.py +422 -0
- package/scripts/postprocess_signals.py +150 -0
- package/scripts/pre-commit +48 -0
- package/scripts/query_enhance.py +157 -0
- package/scripts/query_operators.py +196 -0
- package/scripts/query_rewriter.py +116 -7
- package/scripts/query_signals.py +300 -0
- package/scripts/query_templates.py +167 -0
- package/scripts/query_understanding.py +411 -0
- package/scripts/query_variants.py +298 -0
- package/scripts/quota.py +488 -76
- package/scripts/rank_signals.py +775 -0
- package/scripts/ranking_eval.py +319 -0
- package/scripts/readability_extract.py +433 -0
- package/scripts/recompute.py +415 -0
- package/scripts/recovery.py +616 -0
- package/scripts/recovery_l0.py +160 -0
- package/scripts/redskill/redskill_engine.py +332 -0
- package/scripts/regression_p0p1.py +318 -0
- package/scripts/replay_eval.py +640 -0
- package/scripts/rerank_tfidf.py +225 -0
- package/scripts/research.py +591 -403
- package/scripts/research_cli.py +361 -0
- package/scripts/research_dossier.py +492 -0
- package/scripts/research_expand.py +146 -0
- package/scripts/research_gates.py +272 -0
- package/scripts/research_report.py +209 -0
- package/scripts/research_strategy.py +37 -0
- package/scripts/research_work_packages.py +195 -0
- package/scripts/result_lang.py +311 -0
- package/scripts/robots_guard.py +259 -0
- package/scripts/route.py +921 -254
- package/scripts/route_bangs.py +40 -0
- package/scripts/route_cache.py +225 -0
- package/scripts/route_combo.py +910 -0
- package/scripts/route_domains.py +328 -0
- package/scripts/route_lang.py +605 -0
- package/scripts/route_log.py +63 -0
- package/scripts/route_policy.py +250 -0
- package/scripts/rrf_classic.py +66 -0
- package/scripts/screenshot.py +73 -0
- package/scripts/search.py +897 -405
- package/scripts/search_attribution.py +88 -0
- package/scripts/search_benchmark.py +407 -0
- package/scripts/search_cli.py +627 -0
- package/scripts/search_entry.py +389 -0
- package/scripts/search_output.py +606 -0
- package/scripts/search_pipeline.py +1077 -0
- package/scripts/search_rank.py +991 -0
- package/scripts/search_types.py +27 -12
- package/scripts/seek_locator.py +68 -0
- package/scripts/semantic_evidence.py +215 -0
- package/scripts/serp_guard.py +228 -0
- package/scripts/serp_kill_log.py +51 -0
- package/scripts/single_flight.py +120 -0
- package/scripts/social_engines/__init__.py +10 -0
- package/scripts/social_engines/bilibili_engine.py +131 -0
- package/scripts/social_engines/hackernews_engine.py +113 -0
- package/scripts/social_engines/reddit_engine.py +172 -0
- package/scripts/social_engines/twitter_engine.py +397 -0
- package/scripts/social_engines/v2ex_engine.py +101 -0
- package/scripts/social_engines/weibo_engine.py +128 -0
- package/scripts/social_engines/xiaohongshu_engine.py +137 -0
- package/scripts/social_engines/zhihu_engine.py +139 -0
- package/scripts/social_research.py +201 -0
- package/scripts/sogou_weixin_browser.py +318 -0
- package/scripts/source_ledger.py +249 -0
- package/scripts/stage_timing.py +58 -0
- package/scripts/stats_cli.py +95 -0
- package/scripts/straggler_policy.py +60 -0
- package/scripts/stylometry_detector.py +231 -0
- package/scripts/sync_backends.py +353 -0
- package/scripts/sync_installs.py +29 -0
- package/scripts/template_cli.py +127 -0
- package/scripts/text_lexicon.py +72 -0
- package/scripts/tfidf_router.py +109 -37
- package/scripts/time_utils.py +233 -0
- package/scripts/topic_research_profiles.py +643 -0
- package/scripts/train.py +737 -0
- package/scripts/tweet.py +596 -0
- package/scripts/url_canon.py +217 -0
- package/scripts/url_optimize.py +56 -0
- package/scripts/url_safety.py +236 -0
- package/scripts/usage_log.py +116 -0
- package/scripts/usage_log_clean.py +113 -0
- package/scripts/v2ex_nodes.py +311 -0
- package/scripts/variant_recall.py +188 -0
- package/scripts/watch.py +205 -0
- package/scripts/wechat_article.py +258 -0
- package/scripts/wide_research.py +402 -0
- package/scripts/wx.py +349 -0
- package/scripts/x_article_fetch.py +141 -0
- package/scripts/x_search.py +234 -0
- package/scripts/x_trending.py +189 -0
- package/scripts/yaml_load.py +67 -0
- package/sub-skills/ego-search/SKILL.md +253 -0
- package/sub-skills/ego-search/references/browser-runtime.md +43 -0
- package/sub-skills/ego-search/references/install.md +67 -0
- package/sub-skills/ego-search/references/original-ego-upgrade.md +51 -0
- package/sub-skills/ego-search/scripts/ego_search.py +800 -0
- package/sub-skills/ego-search/scripts/merge.py +226 -0
- package/sub-skills/ego-search/scripts/quality.py +64 -0
- package/sub-skills/ego-search/scripts/runtime.py +172 -0
- package/sub-skills/ego-search/scripts/safety.py +109 -0
- package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
- package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
- package/sub-skills/ego-search/tests/test_query_normalize.py +68 -0
- package/sub-skills/js-run/SKILL.md +67 -0
- package/sub-skills/js-run/scripts/jsrun.py +112 -0
- package/sub-skills/js-run/scripts/shim.js +188 -0
- package/sub-skills/js-run/tests/fixtures/challenge_like.js +27 -0
- package/sub-skills/js-run/tests/test_jsrun.py +165 -0
- package/sub-skills/local-search/SKILL.md +50 -53
- package/sub-skills/local-search/bm25.py +162 -0
- package/sub-skills/local-search/config.yaml +152 -55
- package/sub-skills/local-search/engine_registry.py +91 -26
- package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
- package/sub-skills/local-search/local_search_adapter.py +13 -2
- package/sub-skills/local-search/parse_maps.yaml +40 -42
- package/sub-skills/local-search/search_v3.py +1182 -65
- package/sub-skills/local-search/smart_router.py +47 -7
- package/sub-skills/local-seek/SKILL.md +110 -0
- package/sub-skills/local-seek/config/domains.yaml +20 -0
- package/sub-skills/local-seek/references/auto-behaviors.md +24 -0
- package/sub-skills/local-seek/references/fd-mdfind-recipes.md +62 -0
- package/sub-skills/local-seek/references/git-integration.md +54 -0
- package/sub-skills/local-seek/references/remote-handoff.md +46 -0
- package/sub-skills/local-seek/references/rg-recipes.md +96 -0
- package/sub-skills/local-seek/references/strategies.md +52 -0
- package/sub-skills/local-seek/references/structural-search.md +59 -0
- package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
- package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
- package/sub-skills/local-seek/scripts/seek.py +1465 -0
- package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
- package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
- package/sub-skills/local-seek/scripts/seek_rank.py +193 -0
- package/sub-skills/local-seek/tests/test_content_rank.py +222 -0
- package/sub-skills/local-seek/tests/test_determinism_20261006.py +145 -0
- package/sub-skills/local-seek/tests/test_noise_tier.py +101 -0
- package/sub-skills/local-seek/tests/test_pcre2_probe.py +136 -0
- package/sub-skills/local-seek/tests/test_rank_pinyin.py +77 -0
- package/sub-skills/local-seek/tests/test_regression_20261003.py +169 -0
- package/sub-skills/local-seek/tests/test_regression_20261006.py +146 -0
- package/sub-skills/local-seek/tests/test_seek_degradation.py +111 -0
- package/sub-skills/local-seek/tests/test_seek_fts.py +313 -0
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""engines_builders — 聚合专用构建器导出(兼容 from engines_builders import *)。"""
|
|
3
|
+
|
|
4
|
+
from engines_builders_batch9 import (
|
|
5
|
+
_build_who_don_engine,
|
|
6
|
+
_build_who_gho_engine,
|
|
7
|
+
_build_pubmed_engine,
|
|
8
|
+
_build_core_engine,
|
|
9
|
+
_build_gdacs_engine,
|
|
10
|
+
_build_obis_engine,
|
|
11
|
+
_build_worms_engine,
|
|
12
|
+
_build_energy_charts_engine,
|
|
13
|
+
_build_jolpica_engine,
|
|
14
|
+
_build_openf1_engine,
|
|
15
|
+
_build_openligadb_engine,
|
|
16
|
+
_build_artic_engine,
|
|
17
|
+
_build_cleveland_engine,
|
|
18
|
+
_build_tvmaze_engine,
|
|
19
|
+
_build_jikan_engine,
|
|
20
|
+
_build_deezer_engine,
|
|
21
|
+
_build_listenbrainz_engine,
|
|
22
|
+
_build_egov_law_engine,
|
|
23
|
+
_build_k10plus_engine,
|
|
24
|
+
_build_ror_engine,
|
|
25
|
+
_build_soilgrids_engine,
|
|
26
|
+
_build_noaa_swpc_engine,
|
|
27
|
+
_build_satnogs_engine,
|
|
28
|
+
_build_tle_mirror_engine,
|
|
29
|
+
_build_nhtsa_vpic_engine,
|
|
30
|
+
)
|
|
31
|
+
from engines_builders_tech import (
|
|
32
|
+
_build_exa_engine,
|
|
33
|
+
_build_anysearch_engine,
|
|
34
|
+
_build_wechat_sogou_engine,
|
|
35
|
+
_build_hackernews_engine,
|
|
36
|
+
_build_stackoverflow_engine,
|
|
37
|
+
_build_google_scholar_engine,
|
|
38
|
+
_build_deps_dev_engine,
|
|
39
|
+
_build_endoflife_engine,
|
|
40
|
+
_build_v2ex_engine,
|
|
41
|
+
_build_github_engine,
|
|
42
|
+
_build_tineye_engine,
|
|
43
|
+
_build_bing_rss_engine,
|
|
44
|
+
)
|
|
45
|
+
from engines_builders_cn import (
|
|
46
|
+
_build_em_miaoxiang_engine,
|
|
47
|
+
_build_cninfo_engine,
|
|
48
|
+
_build_sina_quote_engine,
|
|
49
|
+
_build_tencent_quote_engine,
|
|
50
|
+
_build_em_flow_engine,
|
|
51
|
+
_build_ths_hot_engine,
|
|
52
|
+
_build_cls_telegraph_engine,
|
|
53
|
+
_build_em_global_news_engine,
|
|
54
|
+
_build_eastmoney_engine,
|
|
55
|
+
_build_itotii_engine,
|
|
56
|
+
_build_baidu_hot_engine,
|
|
57
|
+
_build_toutiao_hot_engine,
|
|
58
|
+
_build_bilibili_hot_engine,
|
|
59
|
+
_build_zhihu_global_engine,
|
|
60
|
+
_build_zhihu_user_engine,
|
|
61
|
+
_build_bocha_engine,
|
|
62
|
+
_build_bocha_ai_engine,
|
|
63
|
+
_build_std_samr_engine,
|
|
64
|
+
_build_openstd_engine,
|
|
65
|
+
_build_bangumi_engine,
|
|
66
|
+
_build_douban_movie_engine,
|
|
67
|
+
_build_zdic_engine,
|
|
68
|
+
_build_iplant_engine,
|
|
69
|
+
_build_people_daily_engine,
|
|
70
|
+
_build_flk_law_engine,
|
|
71
|
+
_build_wikisource_engine,
|
|
72
|
+
_build_weibo_hot_engine,
|
|
73
|
+
_build_douyin_hot_engine,
|
|
74
|
+
_build_csdn_engine,
|
|
75
|
+
_build_wallstreetcn_engine,
|
|
76
|
+
_build_weather_cn_engine,
|
|
77
|
+
)
|
|
78
|
+
from engines_builders_batch12 import (
|
|
79
|
+
_build_so_engine,
|
|
80
|
+
_build_shenma_engine,
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
from engines_builders_batch13 import (
|
|
84
|
+
_build_yahoo_finance_engine,
|
|
85
|
+
_build_sec_companyfacts_engine,
|
|
86
|
+
_build_frankfurter_engine,
|
|
87
|
+
)
|
|
88
|
+
from engines_builders_data import (
|
|
89
|
+
_build_open_library_engine,
|
|
90
|
+
_build_weread_engine,
|
|
91
|
+
_build_douban_book_engine,
|
|
92
|
+
_build_pubchem_engine,
|
|
93
|
+
_build_gbif_engine,
|
|
94
|
+
_build_rfc_editor_engine,
|
|
95
|
+
_build_uniprot_engine,
|
|
96
|
+
_build_biorxiv_engine,
|
|
97
|
+
_build_un_comtrade_engine,
|
|
98
|
+
_build_rcsb_pdb_engine,
|
|
99
|
+
_build_courtlistener_engine,
|
|
100
|
+
_build_gutenberg_engine,
|
|
101
|
+
_build_wayback_cdx_engine,
|
|
102
|
+
_build_usgs_engine,
|
|
103
|
+
_build_nasa_cmr_engine,
|
|
104
|
+
_build_free_dictionary_engine,
|
|
105
|
+
_build_baidu_baike_engine,
|
|
106
|
+
_build_pypi_engine,
|
|
107
|
+
_build_clinicaltrials_engine,
|
|
108
|
+
_build_openfda_engine,
|
|
109
|
+
_build_juejin_engine,
|
|
110
|
+
_build_models_dev_engine,
|
|
111
|
+
_build_finviz_engine,
|
|
112
|
+
_build_seeking_alpha_engine,
|
|
113
|
+
_build_qweather_engine,
|
|
114
|
+
_build_wenshu_engine,
|
|
115
|
+
_build_jin10_engine,
|
|
116
|
+
_build_octen_engine,
|
|
117
|
+
_build_imdb_engine,
|
|
118
|
+
_build_thesportsdb_engine,
|
|
119
|
+
_build_itunes_engine,
|
|
120
|
+
_build_opencorporates_engine,
|
|
121
|
+
_build_google_patents_engine,
|
|
122
|
+
_build_marginalia_engine,
|
|
123
|
+
_build_wiby_engine,
|
|
124
|
+
)
|
|
125
|
+
from engines_builders_data_macro import (
|
|
126
|
+
_build_fred_engine,
|
|
127
|
+
_build_fx_rate_engine,
|
|
128
|
+
_build_worldbank_engine,
|
|
129
|
+
_build_nbs_stats_engine,
|
|
130
|
+
_build_eurostat_engine,
|
|
131
|
+
)
|
|
132
|
+
from engines_builders_search import (
|
|
133
|
+
_build_parallel_engine,
|
|
134
|
+
_build_parallel_free_engine,
|
|
135
|
+
_build_seltz_engine,
|
|
136
|
+
_build_you_engine,
|
|
137
|
+
_build_qwant_engine,
|
|
138
|
+
_build_ecosia_engine,
|
|
139
|
+
)
|
|
140
|
+
from engines_builders_intl import (
|
|
141
|
+
_build_cnii_engine,
|
|
142
|
+
_build_ndl_engine,
|
|
143
|
+
_build_kor_law_engine,
|
|
144
|
+
_build_hatena_bookmark_engine,
|
|
145
|
+
_build_dnb_engine,
|
|
146
|
+
_build_doaj_engine,
|
|
147
|
+
_build_europeana_engine,
|
|
148
|
+
_build_hal_engine,
|
|
149
|
+
_build_eu_opendata_engine,
|
|
150
|
+
_build_open_meteo_engine,
|
|
151
|
+
_build_twitter_syndication_engine,
|
|
152
|
+
_build_searchmysite_engine,
|
|
153
|
+
_build_lieu_engine,
|
|
154
|
+
_build_opensky_engine,
|
|
155
|
+
_build_electricity_maps_engine,
|
|
156
|
+
_build_usda_engine,
|
|
157
|
+
_build_tatoeba_engine,
|
|
158
|
+
_build_figshare_engine,
|
|
159
|
+
_build_tencent_kline_engine,
|
|
160
|
+
_build_qq_music_engine,
|
|
161
|
+
_build_google_news_engine,
|
|
162
|
+
_build_met_museum_engine,
|
|
163
|
+
)
|
|
164
|
+
from engines_builders_feeds import (
|
|
165
|
+
_build_rss_feed_engine,
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
__all__ = [
|
|
169
|
+
"_build_em_miaoxiang_engine",
|
|
170
|
+
"_build_cninfo_engine",
|
|
171
|
+
"_build_sina_quote_engine",
|
|
172
|
+
"_build_tencent_quote_engine",
|
|
173
|
+
"_build_em_flow_engine",
|
|
174
|
+
"_build_exa_engine",
|
|
175
|
+
"_build_wechat_sogou_engine",
|
|
176
|
+
"_build_hackernews_engine",
|
|
177
|
+
"_build_stackoverflow_engine",
|
|
178
|
+
"_build_std_samr_engine",
|
|
179
|
+
"_build_openstd_engine",
|
|
180
|
+
"_build_bangumi_engine",
|
|
181
|
+
"_build_douban_movie_engine",
|
|
182
|
+
"_build_zdic_engine",
|
|
183
|
+
"_build_iplant_engine",
|
|
184
|
+
"_build_people_daily_engine",
|
|
185
|
+
"_build_flk_law_engine",
|
|
186
|
+
"_build_wikisource_engine",
|
|
187
|
+
"_build_weibo_hot_engine",
|
|
188
|
+
"_build_douyin_hot_engine",
|
|
189
|
+
"_build_csdn_engine",
|
|
190
|
+
"_build_wallstreetcn_engine",
|
|
191
|
+
"_build_weather_cn_engine",
|
|
192
|
+
"_build_google_news_engine",
|
|
193
|
+
"_build_met_museum_engine",
|
|
194
|
+
"_build_google_scholar_engine",
|
|
195
|
+
"_build_v2ex_engine",
|
|
196
|
+
"_build_github_engine",
|
|
197
|
+
"_build_ths_hot_engine",
|
|
198
|
+
"_build_cls_telegraph_engine",
|
|
199
|
+
"_build_em_global_news_engine",
|
|
200
|
+
"_build_eastmoney_engine",
|
|
201
|
+
"_build_itotii_engine",
|
|
202
|
+
"_build_baidu_hot_engine",
|
|
203
|
+
"_build_toutiao_hot_engine",
|
|
204
|
+
"_build_bilibili_hot_engine",
|
|
205
|
+
"_build_open_library_engine",
|
|
206
|
+
"_build_weread_engine",
|
|
207
|
+
"_build_douban_book_engine",
|
|
208
|
+
"_build_fred_engine",
|
|
209
|
+
"_build_fx_rate_engine",
|
|
210
|
+
"_build_worldbank_engine",
|
|
211
|
+
"_build_nbs_stats_engine",
|
|
212
|
+
"_build_pubchem_engine",
|
|
213
|
+
"_build_eurostat_engine",
|
|
214
|
+
"_build_gbif_engine",
|
|
215
|
+
"_build_rfc_editor_engine",
|
|
216
|
+
"_build_uniprot_engine",
|
|
217
|
+
"_build_rcsb_pdb_engine",
|
|
218
|
+
"_build_courtlistener_engine",
|
|
219
|
+
"_build_gutenberg_engine",
|
|
220
|
+
"_build_wayback_cdx_engine",
|
|
221
|
+
"_build_usgs_engine",
|
|
222
|
+
"_build_nasa_cmr_engine",
|
|
223
|
+
"_build_free_dictionary_engine",
|
|
224
|
+
"_build_baidu_baike_engine",
|
|
225
|
+
"_build_pypi_engine",
|
|
226
|
+
"_build_clinicaltrials_engine",
|
|
227
|
+
"_build_openfda_engine",
|
|
228
|
+
"_build_juejin_engine",
|
|
229
|
+
"_build_models_dev_engine",
|
|
230
|
+
"_build_finviz_engine",
|
|
231
|
+
"_build_yahoo_finance_engine",
|
|
232
|
+
"_build_sec_companyfacts_engine",
|
|
233
|
+
"_build_frankfurter_engine",
|
|
234
|
+
"_build_seeking_alpha_engine",
|
|
235
|
+
"_build_qweather_engine",
|
|
236
|
+
"_build_wenshu_engine",
|
|
237
|
+
"_build_jin10_engine",
|
|
238
|
+
"_build_octen_engine",
|
|
239
|
+
"_build_imdb_engine",
|
|
240
|
+
"_build_thesportsdb_engine",
|
|
241
|
+
"_build_itunes_engine",
|
|
242
|
+
"_build_opencorporates_engine",
|
|
243
|
+
"_build_google_patents_engine",
|
|
244
|
+
"_build_marginalia_engine",
|
|
245
|
+
"_build_wiby_engine",
|
|
246
|
+
"_build_cnii_engine",
|
|
247
|
+
"_build_ndl_engine",
|
|
248
|
+
"_build_kor_law_engine",
|
|
249
|
+
"_build_hatena_bookmark_engine",
|
|
250
|
+
"_build_dnb_engine",
|
|
251
|
+
"_build_doaj_engine",
|
|
252
|
+
"_build_europeana_engine",
|
|
253
|
+
"_build_hal_engine",
|
|
254
|
+
"_build_eu_opendata_engine",
|
|
255
|
+
"_build_open_meteo_engine",
|
|
256
|
+
"_build_twitter_syndication_engine",
|
|
257
|
+
"_build_searchmysite_engine",
|
|
258
|
+
"_build_searchmysite_engine",
|
|
259
|
+
"_build_lieu_engine",
|
|
260
|
+
"_build_opensky_engine",
|
|
261
|
+
"_build_electricity_maps_engine",
|
|
262
|
+
"_build_usda_engine",
|
|
263
|
+
"_build_tatoeba_engine",
|
|
264
|
+
"_build_figshare_engine",
|
|
265
|
+
"_build_tencent_kline_engine",
|
|
266
|
+
"_build_qq_music_engine",
|
|
267
|
+
"_build_parallel_engine",
|
|
268
|
+
"_build_parallel_free_engine",
|
|
269
|
+
"_build_seltz_engine",
|
|
270
|
+
"_build_you_engine",
|
|
271
|
+
"_build_who_don_engine",
|
|
272
|
+
"_build_who_gho_engine",
|
|
273
|
+
"_build_pubmed_engine",
|
|
274
|
+
"_build_core_engine",
|
|
275
|
+
"_build_gdacs_engine",
|
|
276
|
+
"_build_obis_engine",
|
|
277
|
+
"_build_worms_engine",
|
|
278
|
+
"_build_energy_charts_engine",
|
|
279
|
+
"_build_jolpica_engine",
|
|
280
|
+
"_build_openf1_engine",
|
|
281
|
+
"_build_openligadb_engine",
|
|
282
|
+
"_build_artic_engine",
|
|
283
|
+
"_build_cleveland_engine",
|
|
284
|
+
"_build_tvmaze_engine",
|
|
285
|
+
"_build_jikan_engine",
|
|
286
|
+
"_build_deezer_engine",
|
|
287
|
+
"_build_listenbrainz_engine",
|
|
288
|
+
"_build_egov_law_engine",
|
|
289
|
+
"_build_k10plus_engine",
|
|
290
|
+
"_build_ror_engine",
|
|
291
|
+
"_build_soilgrids_engine",
|
|
292
|
+
"_build_noaa_swpc_engine",
|
|
293
|
+
"_build_satnogs_engine",
|
|
294
|
+
"_build_tle_mirror_engine",
|
|
295
|
+
"_build_nhtsa_vpic_engine",
|
|
296
|
+
"_build_rss_feed_engine",
|
|
297
|
+
]
|
|
298
|
+
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""批次十构建器:zhihu_global 补强。
|
|
3
|
+
|
|
4
|
+
为什么不在 engines_builders_cn.py 里加(任务原指定位置):该文件被
|
|
5
|
+
test_module_size_gate.py 祖父清单冻结在 2319 行(当前恰好 2319,只能减不能增),
|
|
6
|
+
且 tests/ 门禁只放行新建测试文件——门禁自身文档写明「想加功能,正解是拆模块,
|
|
7
|
+
不是把数字往上调」,故按 batch9 命名先例落新文件(新增文件天然合规)。
|
|
8
|
+
|
|
9
|
+
zhihu_global 说明:cn 里的旧实现与本文件同名实现并存,engines.py 注册表已
|
|
10
|
+
指向本文件版本(候选池取满 + 错误文案行动提示两处补强);cn 旧版仅在
|
|
11
|
+
engines_builders 聚合层保留转出,属被取代代码。
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import re
|
|
18
|
+
import sys
|
|
19
|
+
import time
|
|
20
|
+
import urllib.error
|
|
21
|
+
import urllib.parse
|
|
22
|
+
import urllib.request
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
from engine_env import get_env
|
|
26
|
+
|
|
27
|
+
from engines_base import (
|
|
28
|
+
http_open,
|
|
29
|
+
rank_score,
|
|
30
|
+
safe_search,
|
|
31
|
+
)
|
|
32
|
+
from engines_builders_cn import _parse_site_filter
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class _LazyLogger:
|
|
36
|
+
"""延迟创建 logger,避免 import logging 的重量级 import 链(traceback→dataclasses→inspect)。"""
|
|
37
|
+
_logger = None
|
|
38
|
+
|
|
39
|
+
def __getattr__(self, name: str):
|
|
40
|
+
if _LazyLogger._logger is None:
|
|
41
|
+
import logging
|
|
42
|
+
_LazyLogger._logger = logging.getLogger("unified_search.engines")
|
|
43
|
+
if not _LazyLogger._logger.handlers:
|
|
44
|
+
_LazyLogger._logger.setLevel(logging.WARNING)
|
|
45
|
+
_LazyLogger._logger.addHandler(logging.StreamHandler(sys.stderr))
|
|
46
|
+
return getattr(_LazyLogger._logger, name)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
logger = _LazyLogger()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# ── zhihu_global 补强版(取代 engines_builders_cn 同名实现)───────────────────
|
|
53
|
+
|
|
54
|
+
# 端点 Count 上限:官方 global_search 上限 20(cn 旧版 min(n, 20) 同源口径)
|
|
55
|
+
_ZHIHU_COUNT_MAX = 20
|
|
56
|
+
|
|
57
|
+
# 非成功响应 → 可行动提示。时钟偏差类按 Message 文案识别(错误码表未公开,
|
|
58
|
+
# 不猜码值):签名带 X-Request-Timestamp,本机时钟漂移会让服务端判签名过期,
|
|
59
|
+
# 现象是「密钥没错却全挂」——提示必须指向校时,防止用户误判 key 失效反复换 key
|
|
60
|
+
_ZHIHU_CLOCK_HINT_RE = re.compile(r"timestamp|时间戳|签名|时钟|clock", re.I)
|
|
61
|
+
_ZHIHU_CLOCK_HINT = "本机时钟可能偏差导致签名过期:先校准系统时间再重试,不必反复换密钥"
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _zhihu_error_item(code: Any, message: Any) -> dict[str, Any]:
|
|
65
|
+
"""非成功响应的 error item:保留原始 code/message,时钟类追加行动提示。"""
|
|
66
|
+
text = f"zhihu_global Code={code} {str(message or '')[:100]}"
|
|
67
|
+
if _ZHIHU_CLOCK_HINT_RE.search(str(message or "")):
|
|
68
|
+
text = f"{text};hint: {_ZHIHU_CLOCK_HINT}"
|
|
69
|
+
return {"error": text, "source": "zhihu_global"}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _build_zhihu_global_engine(spec: dict[str, Any]) -> Any:
|
|
73
|
+
"""知乎开放平台全网搜索(developer.zhihu.com global_search)。
|
|
74
|
+
|
|
75
|
+
在 cn 旧版语义之上的两处微机制补强(2026-09-26):
|
|
76
|
+
1. 候选池取满再截断:Filter host== 是服务端在候选池上后筛,只取 n 条
|
|
77
|
+
会把「过滤后恰好命中」的候选一并筛掉,空结果会被模型读成「知乎没有
|
|
78
|
+
相关内容」;带站点限定或时间下限(since)时取满端点 Count 上限,
|
|
79
|
+
客户端再截断到 n。since 的时间 Filter 语法官方未文档化,先只承担
|
|
80
|
+
「取满候选池」语义。
|
|
81
|
+
2. 错误码 → 行动提示:时钟偏差类错误给「校时」提示而非裸错误。
|
|
82
|
+
查询语法与 cn 版一致:site:/host: → Filter host==,普通查询 SearchDB=all。
|
|
83
|
+
"""
|
|
84
|
+
timeout = spec.get("timeout", 10)
|
|
85
|
+
|
|
86
|
+
@safe_search
|
|
87
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
88
|
+
to = _timeout or timeout
|
|
89
|
+
secret = get_env(["ARGO_ZHIHU_ACCESS_SECRET", "ZHIHU_ACCESS_SECRET"])
|
|
90
|
+
if not secret:
|
|
91
|
+
return []
|
|
92
|
+
|
|
93
|
+
# 解析 site:/host: 站点限定语法 → Filter: host=="..."(实现单点在 cn)
|
|
94
|
+
filter_expr, search_query = _parse_site_filter(query)
|
|
95
|
+
|
|
96
|
+
filtered = bool(filter_expr or kwargs.get("since"))
|
|
97
|
+
params: dict[str, Any] = {
|
|
98
|
+
"Query": search_query or query,
|
|
99
|
+
"Count": str(_ZHIHU_COUNT_MAX if filtered else min(int(n or 5), _ZHIHU_COUNT_MAX)),
|
|
100
|
+
"SearchDB": "all",
|
|
101
|
+
}
|
|
102
|
+
if filter_expr:
|
|
103
|
+
params["Filter"] = filter_expr
|
|
104
|
+
url = "https://developer.zhihu.com/api/v1/content/global_search?" + urllib.parse.urlencode(params)
|
|
105
|
+
headers = {
|
|
106
|
+
"Authorization": f"Bearer {secret}",
|
|
107
|
+
"X-Request-Timestamp": str(int(time.time())),
|
|
108
|
+
"Content-Type": "application/json",
|
|
109
|
+
"User-Agent": "argo-search/2.6 (unified-search@local)",
|
|
110
|
+
}
|
|
111
|
+
try:
|
|
112
|
+
with http_open(urllib.request.Request(url, headers=headers), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
113
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
114
|
+
except urllib.error.HTTPError as e:
|
|
115
|
+
# 401/403 等必须暴露为 error item 而非静默空——调用侧把
|
|
116
|
+
# 「没配置」「鉴权失败」「没结果」区分开才可行动
|
|
117
|
+
return [{"error": f"zhihu_global API HTTP {e.code}", "source": "zhihu_global"}]
|
|
118
|
+
except Exception as e:
|
|
119
|
+
logger.warning(f"zhihu_global 失败: {e}")
|
|
120
|
+
return [{"error": f"zhihu_global {type(e).__name__}: {e}", "source": "zhihu_global"}]
|
|
121
|
+
if data.get("Code") not in (0, None):
|
|
122
|
+
# 30001=频率限制 30002=配额限制,显式暴露供配额状态机归类
|
|
123
|
+
logger.warning(f"zhihu_global 返回码异常: {data.get('Code')} {data.get('Message')}")
|
|
124
|
+
return [_zhihu_error_item(data.get("Code"), data.get("Message"))]
|
|
125
|
+
items = (data.get("Data") or {}).get("Items") or []
|
|
126
|
+
results = []
|
|
127
|
+
for _rk3, item in enumerate(items[:n]):
|
|
128
|
+
if not isinstance(item, dict):
|
|
129
|
+
continue
|
|
130
|
+
title = (item.get("Title") or "").strip()
|
|
131
|
+
# API 标题统一带「 - 知乎」尾巴:截断前剥掉(先剥再切,尾巴不占正文)
|
|
132
|
+
if title.endswith(" - 知乎"):
|
|
133
|
+
title = title[: -len(" - 知乎")].rstrip()
|
|
134
|
+
url_ = item.get("Url") or ""
|
|
135
|
+
snippet = item.get("ContentText") or ""
|
|
136
|
+
# 去 <em> 高亮标签
|
|
137
|
+
snippet = re.sub(r"<[^>]+>", "", snippet).strip()
|
|
138
|
+
if not title and not url_:
|
|
139
|
+
continue
|
|
140
|
+
# 结构化信号:权威等级 / 互动 / 时效
|
|
141
|
+
social_meta = {
|
|
142
|
+
"author": item.get("AuthorName") or "",
|
|
143
|
+
"content_type": item.get("ContentType") or "",
|
|
144
|
+
"vote_up": item.get("VoteUpCount") or 0,
|
|
145
|
+
"comment_count": item.get("CommentCount") or 0,
|
|
146
|
+
"authority_level": item.get("AuthorityLevel") or "",
|
|
147
|
+
"edit_time": item.get("EditTime") or 0,
|
|
148
|
+
}
|
|
149
|
+
results.append({
|
|
150
|
+
"title": title[:200],
|
|
151
|
+
"url": url_,
|
|
152
|
+
"snippet": snippet[:300],
|
|
153
|
+
"source": "zhihu_global",
|
|
154
|
+
"score": rank_score(0.7, _rk3),
|
|
155
|
+
"authority_level": social_meta["authority_level"],
|
|
156
|
+
"social_meta": social_meta,
|
|
157
|
+
})
|
|
158
|
+
return results
|
|
159
|
+
return _engine
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""批次十一构建器:图片检索能力的两个补强源(2026-09-26)。
|
|
3
|
+
|
|
4
|
+
本批只收图源,目标是修「搜到图源却拿不到图」这一类缺陷的通用面:
|
|
5
|
+
|
|
6
|
+
wikimedia_commons Wikimedia Commons(免 key,通用图库,带 CC 许可与宽高)
|
|
7
|
+
|
|
8
|
+
为什么单独建一个 builder 而不是声明式 spec:Commons 的 `query.pages` 是
|
|
9
|
+
**以 pageid 为键的字典**,不是数组——`_extract_items` 遇到 dict 会把它当成
|
|
10
|
+
「单条结果」包成 `[{pageid: {...}}]`,声明式 output_map 取不到 title/url
|
|
11
|
+
(实测返回空列表)。这属于查询形状不兼容,只能用代码展开。
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import sys
|
|
18
|
+
import urllib.error
|
|
19
|
+
import urllib.parse
|
|
20
|
+
import urllib.request
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from engines_base import (
|
|
24
|
+
finalize_image_fields,
|
|
25
|
+
rank_score,
|
|
26
|
+
safe_search,
|
|
27
|
+
http_open,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class _LazyLogger:
|
|
32
|
+
"""延迟创建 logger,避免 import logging 的重量级 import 链(traceback→dataclasses→inspect)。"""
|
|
33
|
+
_logger = None
|
|
34
|
+
|
|
35
|
+
def __getattr__(self, name: str):
|
|
36
|
+
if _LazyLogger._logger is None:
|
|
37
|
+
import logging
|
|
38
|
+
_LazyLogger._logger = logging.getLogger("unified_search.engines")
|
|
39
|
+
if not _LazyLogger._logger.handlers:
|
|
40
|
+
_LazyLogger._logger.setLevel(logging.WARNING)
|
|
41
|
+
_LazyLogger._logger.addHandler(logging.StreamHandler(sys.stderr))
|
|
42
|
+
return getattr(_LazyLogger._logger, name)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
logger = _LazyLogger()
|
|
46
|
+
|
|
47
|
+
_UA = "argo-search/2.9 (+batch11; wikimedia_commons)"
|
|
48
|
+
_API = "https://commons.wikimedia.org/w/api.php"
|
|
49
|
+
|
|
50
|
+
# Commons 的 extmetadata 是一层「字段 → {value, source, hidden}」的包装,
|
|
51
|
+
# 取值要过两道:raw["LicenseShortName"]["value"]。漏掉里层取键会拿到 dict
|
|
52
|
+
# 本身,_coerce_field 会把它丢成空串——表现为「许可字段时有时无」。
|
|
53
|
+
#
|
|
54
|
+
# 许可取两个字段:`LicenseShortName` 是给人看的(`CC BY-SA 4.0`),
|
|
55
|
+
# `License` 是机器码(`cc-by-sa-4.0`)。两者都要——实测有约 8% 的文件
|
|
56
|
+
# shortname 只写「Attribution」(Commons 在无法确定具体协议时的泛称),
|
|
57
|
+
# 而机器码为空;反过来也有机器码齐全而 shortname 缺失的。任一可用即可判定。
|
|
58
|
+
_META_FIELDS = ("LicenseShortName", "License", "Artist", "ImageDescription",
|
|
59
|
+
"LicenseUrl")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _meta(ext: dict[str, Any], key: str) -> str:
|
|
63
|
+
"""取 extmetadata 里的纯文本值(内层是 {value, source, hidden})。"""
|
|
64
|
+
node = ext.get(key)
|
|
65
|
+
if isinstance(node, dict):
|
|
66
|
+
node = node.get("value")
|
|
67
|
+
if node is None:
|
|
68
|
+
return ""
|
|
69
|
+
text = str(node)
|
|
70
|
+
# 上游描述字段带 HTML(<a href=...>、<span>),剥标签留文本——
|
|
71
|
+
# 交付出去的 snippet 里带标签既占字数又不可读。
|
|
72
|
+
if "<" in text and ">" in text:
|
|
73
|
+
import re
|
|
74
|
+
text = re.sub(r"<[^>]+>", " ", text)
|
|
75
|
+
text = re.sub(r"\s+", " ", text)
|
|
76
|
+
return text.strip()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _build_wikimedia_commons_engine(spec: dict[str, Any]) -> Any:
|
|
80
|
+
"""Wikimedia Commons 通用图库(免 key、CC 素材、字段齐全)。"""
|
|
81
|
+
timeout = spec.get("timeout", 20)
|
|
82
|
+
eng = spec.get("_name", "wikimedia_commons")
|
|
83
|
+
|
|
84
|
+
@safe_search
|
|
85
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
86
|
+
to = _timeout or timeout
|
|
87
|
+
q = (query or "").strip()
|
|
88
|
+
if not q:
|
|
89
|
+
return []
|
|
90
|
+
# filetype:bitmap 排除 PDF/DjVu/SVG 等非位图文件——不筛的话「猫」这类
|
|
91
|
+
# 查询会混进大量古籍扫描 PDF,它们没有可直接展示的位图。
|
|
92
|
+
params = {
|
|
93
|
+
"action": "query",
|
|
94
|
+
"generator": "search",
|
|
95
|
+
"gsrsearch": f"filetype:bitmap {q}",
|
|
96
|
+
"gsrnamespace": "6", # namespace 6 = File
|
|
97
|
+
"gsrlimit": str(max(1, min(int(n), 50))),
|
|
98
|
+
"prop": "imageinfo",
|
|
99
|
+
"iiprop": "url|size|extmetadata",
|
|
100
|
+
"iiurlwidth": "480", # 缩略图宽度,直接可嵌入
|
|
101
|
+
"format": "json",
|
|
102
|
+
"formatversion": "2",
|
|
103
|
+
}
|
|
104
|
+
url = f"{_API}?{urllib.parse.urlencode(params)}"
|
|
105
|
+
req = urllib.request.Request(url, headers={"User-Agent": _UA, "Accept": "application/json"})
|
|
106
|
+
try:
|
|
107
|
+
with http_open(req, timeout=to, engine=eng) as resp:
|
|
108
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
109
|
+
except (urllib.error.URLError, OSError, ValueError) as e:
|
|
110
|
+
logger.warning(f"Wikimedia Commons 失败: {e}")
|
|
111
|
+
return []
|
|
112
|
+
|
|
113
|
+
pages: Any = ((data or {}).get("query") or {}).get("pages")
|
|
114
|
+
# formatversion=2 返回 list,=1 返回 dict。两种都收——上游改默认值
|
|
115
|
+
# 是常事,只认一种会让引擎在版本切换时静默归零。
|
|
116
|
+
if isinstance(pages, dict):
|
|
117
|
+
pages = list(pages.values())
|
|
118
|
+
if not isinstance(pages, list):
|
|
119
|
+
return []
|
|
120
|
+
|
|
121
|
+
results: list[dict[str, Any]] = []
|
|
122
|
+
for rk, page in enumerate(pages[: max(1, int(n))]):
|
|
123
|
+
if not isinstance(page, dict):
|
|
124
|
+
continue
|
|
125
|
+
info = page.get("imageinfo")
|
|
126
|
+
if isinstance(info, list):
|
|
127
|
+
info = info[0] if info else None
|
|
128
|
+
if not isinstance(info, dict):
|
|
129
|
+
continue
|
|
130
|
+
title = str(page.get("title") or "").strip()
|
|
131
|
+
# File: 前缀是 MediaWiki 的命名空间标记,不是文件名的一部分
|
|
132
|
+
title = title[5:].strip() if title.startswith("File:") else title
|
|
133
|
+
img = str(info.get("url") or "").strip()
|
|
134
|
+
if not title or not img:
|
|
135
|
+
continue
|
|
136
|
+
|
|
137
|
+
ext = info.get("extmetadata")
|
|
138
|
+
ext = ext if isinstance(ext, dict) else {}
|
|
139
|
+
license_label = _meta(ext, "LicenseShortName")
|
|
140
|
+
# shortname 是泛称(Commons 对无法确定具体协议的件写「Attribution」)
|
|
141
|
+
# 或缺失时,退到机器码 `cc-by-sa-4.0`——它虽然不好读,但能被
|
|
142
|
+
# finalize_image_fields 归一成标准标注,比一个认不出的泛称有用。
|
|
143
|
+
if not license_label or license_label.strip().lower() in (
|
|
144
|
+
"attribution", "copyrighted", "unknown"):
|
|
145
|
+
license_label = _meta(ext, "License") or license_label
|
|
146
|
+
artist = _meta(ext, "Artist")
|
|
147
|
+
desc = _meta(ext, "ImageDescription")
|
|
148
|
+
page_url = str(page.get("descriptionurl") or "").strip()
|
|
149
|
+
if not page_url:
|
|
150
|
+
page_url = f"https://commons.wikimedia.org/wiki/File:{urllib.parse.quote(title.replace(' ', '_'))}"
|
|
151
|
+
|
|
152
|
+
snippet_bits = [b for b in (artist and f"作者 {artist}", desc) if b]
|
|
153
|
+
snippet = " · ".join(snippet_bits) or "Wikimedia Commons 开放素材"
|
|
154
|
+
|
|
155
|
+
row: dict[str, Any] = {
|
|
156
|
+
"title": title[:200],
|
|
157
|
+
"url": page_url,
|
|
158
|
+
"snippet": snippet[:300],
|
|
159
|
+
"source": eng,
|
|
160
|
+
"score": rank_score(0.85, rk),
|
|
161
|
+
# 图片直链用 imageinfo.url(原图),thumburl 是缩略图——
|
|
162
|
+
# 交付时给原图,需要小图的一方自己缩(反过来则拿不到原图)。
|
|
163
|
+
"image_url": img,
|
|
164
|
+
"image_license": license_label,
|
|
165
|
+
"image_license_url": _meta(ext, "LicenseUrl"),
|
|
166
|
+
"image_width": info.get("width"),
|
|
167
|
+
"image_height": info.get("height"),
|
|
168
|
+
}
|
|
169
|
+
results.append(row)
|
|
170
|
+
# 尺寸转 int / 许可归一 / 算商用判据——与声明式路径同一个收尾函数,
|
|
171
|
+
# 保证两类引擎交付的字段契约完全一致。
|
|
172
|
+
return finalize_image_fields(results, spec.get("image_license") or "")
|
|
173
|
+
|
|
174
|
+
return _engine
|