argo-search 1.0.1 → 2.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -21
- package/README.en.md +822 -0
- package/README.es.md +864 -0
- package/README.ja.md +864 -0
- package/README.ko.md +864 -0
- package/README.md +867 -320
- package/SKILL.md +135 -0
- package/assets/readme/hero.svg +121 -0
- package/assets/readme/made-with-beautify.svg +10 -0
- package/assets/readme/proof-routes.svg +73 -0
- package/assets/readme/settings-keys.png +0 -0
- package/assets/readme/settings-tools.png +0 -0
- package/assets/readme/why-better.svg +104 -0
- package/assets/readme/workflow.svg +83 -0
- package/backends/domain_profiles.json +2308 -96
- package/backends/engine_registry.yaml +3314 -503
- package/backends/query_synonyms_cn.json +35 -0
- package/backends/quota_profiles.json +2409 -161
- package/backends/source_types_cn.json +83 -0
- package/backends/text_lexicon.json +44 -0
- package/bin/argo +703 -0
- package/bin/argo.js +71 -39
- package/config.yaml +4976 -363
- package/cordis.patch.yml +21 -0
- package/data/countries.json +40 -0
- package/data/lang_matrix/lang_capability.json +842 -0
- package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
- package/data/regions_cn.json +1 -0
- package/docs/ADDING_NEW_ENGINE.md +184 -0
- package/docs/ARGO_INTRO.md +124 -0
- package/docs/DATA_SOURCE_GAPS.md +81 -0
- package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
- package/docs/ENGINE_CATALOG.md +564 -0
- package/docs/MCP_SETUP.md +106 -0
- package/docs/PLATFORM_PATHS.md +111 -0
- package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
- package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
- package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
- package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
- package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
- package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
- package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
- package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
- package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
- package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
- package/docs/SEARCH_ARCHIVE.md +115 -0
- package/docs/assets/architecture.svg +104 -0
- package/docs/assets/hero.svg +98 -0
- package/docs/engines/bangumi.md +27 -0
- package/docs/engines/biorxiv.md +27 -0
- package/docs/engines/clawhub.md +27 -0
- package/docs/engines/crt_sh.md +27 -0
- package/docs/engines/datacite.md +27 -0
- package/docs/engines/deps_dev.md +27 -0
- package/docs/engines/douban_movie.md +27 -0
- package/docs/engines/endoflife.md +27 -0
- package/docs/engines/flk_law.md +27 -0
- package/docs/engines/fxtwitter.md +27 -0
- package/docs/engines/google_news.md +27 -0
- package/docs/engines/gov_regulations.md +27 -0
- package/docs/engines/guqian.md +27 -0
- package/docs/engines/hackernews.md +27 -0
- package/docs/engines/iplant.md +27 -0
- package/docs/engines/keenable.md +27 -0
- package/docs/engines/met_museum.md +27 -0
- package/docs/engines/nasa_images.md +27 -0
- package/docs/engines/netease_music.md +27 -0
- package/docs/engines/npm.md +27 -0
- package/docs/engines/nvd.md +27 -0
- package/docs/engines/openreview.md +27 -0
- package/docs/engines/openstd.md +27 -0
- package/docs/engines/parallel_free.md +27 -0
- package/docs/engines/people_daily.md +27 -0
- package/docs/engines/realtime_index.md +27 -0
- package/docs/engines/sec_edgar.md +27 -0
- package/docs/engines/seltz.md +27 -0
- package/docs/engines/skillsmp.md +27 -0
- package/docs/engines/sspai.md +27 -0
- package/docs/engines/std_samr.md +27 -0
- package/docs/engines/train.md +27 -0
- package/docs/engines/un_comtrade.md +27 -0
- package/docs/engines/wikisource.md +27 -0
- package/docs/engines/zdic.md +27 -0
- package/docs/engines/zenodo.md +27 -0
- package/docs/engines/zhihu_hot_app.md +27 -0
- package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
- package/engines/README.md +10 -0
- package/engines/_template_cli.yaml +39 -0
- package/engines/_template_http.yaml +37 -0
- package/engines/plugins/_template_plugin.py +50 -0
- package/engines/specs/_cisa_kev.yaml +54 -0
- package/engines/specs/_osv.yaml +63 -0
- package/engines/specs/aljazeera.yaml +43 -0
- package/engines/specs/antara.yaml +43 -0
- package/engines/specs/artic.yaml +20 -0
- package/engines/specs/aviation_weather.yaml +39 -0
- package/engines/specs/bangumi.yaml +21 -0
- package/engines/specs/biorxiv.yaml +24 -0
- package/engines/specs/carbon_intensity.yaml +32 -0
- package/engines/specs/cinii.yaml +44 -0
- package/engines/specs/clawhub.yaml +49 -0
- package/engines/specs/cleveland.yaml +19 -0
- package/engines/specs/cn_ai_news.yaml +43 -0
- package/engines/specs/core.yaml +35 -0
- package/engines/specs/crt_sh.yaml +38 -0
- package/engines/specs/ct24.yaml +43 -0
- package/engines/specs/datacite.yaml +27 -0
- package/engines/specs/datagov_uk.yaml +41 -0
- package/engines/specs/ddgs_images.yaml +45 -0
- package/engines/specs/deepseek_search.yaml +74 -0
- package/engines/specs/deezer.yaml +20 -0
- package/engines/specs/deps_dev.yaml +21 -0
- package/engines/specs/digup.yaml +36 -0
- package/engines/specs/doi.yaml +39 -0
- package/engines/specs/douban_movie.yaml +22 -0
- package/engines/specs/dw_news.yaml +47 -0
- package/engines/specs/egov_law.yaml +21 -0
- package/engines/specs/elpais.yaml +43 -0
- package/engines/specs/endoflife.yaml +21 -0
- package/engines/specs/energy_charts.yaml +21 -0
- package/engines/specs/factcheck_org.yaml +52 -0
- package/engines/specs/faz.yaml +43 -0
- package/engines/specs/federal_register.yaml +57 -0
- package/engines/specs/firecrawl.yaml +53 -0
- package/engines/specs/flk_law.yaml +24 -0
- package/engines/specs/folha.yaml +43 -0
- package/engines/specs/france24.yaml +44 -0
- package/engines/specs/full_fact.yaml +47 -0
- package/engines/specs/fxtwitter.yaml +31 -0
- package/engines/specs/gbfs_nyc.yaml +32 -0
- package/engines/specs/gdacs.yaml +21 -0
- package/engines/specs/gdelt.yaml +64 -0
- package/engines/specs/google_news.yaml +22 -0
- package/engines/specs/gov_regulations.yaml +38 -0
- package/engines/specs/guardian_rss.yaml +59 -0
- package/engines/specs/hurriyet.yaml +43 -0
- package/engines/specs/iplant.yaml +25 -0
- package/engines/specs/jikan.yaml +24 -0
- package/engines/specs/jolpica.yaml +20 -0
- package/engines/specs/k10plus.yaml +20 -0
- package/engines/specs/keenable.yaml +44 -0
- package/engines/specs/lefigaro.yaml +43 -0
- package/engines/specs/listenbrainz.yaml +19 -0
- package/engines/specs/matichon.yaml +43 -0
- package/engines/specs/met_museum.yaml +21 -0
- package/engines/specs/nasa_images.yaml +45 -0
- package/engines/specs/netease_music.yaml +38 -0
- package/engines/specs/nhk.yaml +43 -0
- package/engines/specs/nhtsa_vpic.yaml +20 -0
- package/engines/specs/noaa_swpc.yaml +20 -0
- package/engines/specs/nvd.yaml +38 -0
- package/engines/specs/obis.yaml +20 -0
- package/engines/specs/opencitations.yaml +52 -0
- package/engines/specs/openf1.yaml +21 -0
- package/engines/specs/openfoodfacts.yaml +45 -0
- package/engines/specs/openligadb.yaml +20 -0
- package/engines/specs/openreview.yaml +37 -0
- package/engines/specs/openstd.yaml +21 -0
- package/engines/specs/people_daily.yaml +22 -0
- package/engines/specs/realtime_index.yaml +34 -0
- package/engines/specs/ror.yaml +20 -0
- package/engines/specs/satnogs.yaml +20 -0
- package/engines/specs/sec_edgar.yaml +46 -0
- package/engines/specs/skillsmp.yaml +50 -0
- package/engines/specs/sogou_weixin_browser.yaml +30 -0
- package/engines/specs/soilgrids.yaml +25 -0
- package/engines/specs/sspai.yaml +36 -0
- package/engines/specs/stackexchange.yaml +46 -0
- package/engines/specs/std_samr.yaml +21 -0
- package/engines/specs/tass.yaml +43 -0
- package/engines/specs/tle_mirror.yaml +21 -0
- package/engines/specs/train.yaml +37 -0
- package/engines/specs/tvmaze.yaml +20 -0
- package/engines/specs/twitter_syndication.yaml +31 -0
- package/engines/specs/un_comtrade.yaml +23 -0
- package/engines/specs/unpaywall.yaml +54 -0
- package/engines/specs/vnpress.yaml +43 -0
- package/engines/specs/weather.yaml +32 -0
- package/engines/specs/who_don.yaml +21 -0
- package/engines/specs/who_gho.yaml +20 -0
- package/engines/specs/wikimedia_commons.yaml +34 -0
- package/engines/specs/wikisource.yaml +22 -0
- package/engines/specs/worms.yaml +19 -0
- package/engines/specs/x_trending.yaml +34 -0
- package/engines/specs/yna.yaml +43 -0
- package/engines/specs/ynet.yaml +43 -0
- package/engines/specs/zbmath.yaml +40 -0
- package/engines/specs/zdic.yaml +22 -0
- package/engines/specs/zenodo.yaml +27 -0
- package/engines/specs/zhihu_hot_app.yaml +35 -0
- package/mcp/clients.yaml +74 -0
- package/package.json +51 -12
- package/packages/dsh-plugin/README.md +134 -0
- package/packages/dsh-plugin/cordis.patch.yml +48 -0
- package/packages/dsh-plugin/dsh/client.js +1244 -0
- package/packages/dsh-plugin/dsh/index.js +2290 -0
- package/packages/dsh-plugin/dsh/native-tools.mjs +743 -0
- package/packages/dsh-plugin/package.json +38 -0
- package/packages/dsh-plugin/test/client.test.js +232 -0
- package/packages/dsh-plugin/test/config.test.js +30 -0
- package/packages/dsh-plugin/test/index.test.js +1338 -0
- package/references/query-templates.md +96 -0
- package/references/query-templates.yaml +118 -0
- package/scripts/ab_eval.py +219 -0
- package/scripts/ab_eval_p0p1.py +266 -0
- package/scripts/adaptive.py +271 -70
- package/scripts/answer.py +154 -0
- package/scripts/archive_run.py +591 -0
- package/scripts/argo_engine_registry.py +172 -0
- package/scripts/argo_paths.py +728 -0
- package/scripts/article.py +136 -0
- package/scripts/arxiv_source.py +201 -0
- package/scripts/badcase_ledger.py +252 -0
- package/scripts/batch_probe.py +241 -0
- package/scripts/benchmark.py +19 -0
- package/scripts/bounded_run.py +144 -0
- package/scripts/browser_auth.py +323 -0
- package/scripts/burstiness_detector.py +147 -0
- package/scripts/cache.py +1078 -67
- package/scripts/cache_guard.py +271 -0
- package/scripts/cache_key_vdom.py +56 -0
- package/scripts/cache_similarity.py +170 -0
- package/scripts/caliber_isolation.py +195 -0
- package/scripts/calibrate_lowquality.py +210 -0
- package/scripts/candidate_envelope.py +366 -0
- package/scripts/chrome_cdp.py +871 -0
- package/scripts/circuit_breaker.py +389 -0
- package/scripts/citations.py +334 -0
- package/scripts/cite_cli.py +59 -0
- package/scripts/cjk_tokens.py +95 -0
- package/scripts/clarify.py +47 -24
- package/scripts/cli_io.py +240 -0
- package/scripts/config.py +683 -45
- package/scripts/config_cache.py +402 -0
- package/scripts/conn_pool.py +220 -0
- package/scripts/content_security.py +685 -0
- package/scripts/content_signals.py +730 -0
- package/scripts/crawl.py +74 -35
- package/scripts/daemon.py +866 -0
- package/scripts/digup_bridge.py +214 -0
- package/scripts/engine_admission.py +366 -0
- package/scripts/engine_dispatch.py +1019 -0
- package/scripts/engine_env.py +534 -0
- package/scripts/engine_failure.py +411 -0
- package/scripts/engine_families.py +732 -0
- package/scripts/engine_policy.py +218 -0
- package/scripts/engine_requires.py +190 -0
- package/scripts/engine_status.py +322 -0
- package/scripts/engine_validate.py +638 -0
- package/scripts/engines.py +778 -294
- package/scripts/engines_base.py +1521 -0
- package/scripts/engines_builders.py +298 -0
- package/scripts/engines_builders_batch10.py +159 -0
- package/scripts/engines_builders_batch11.py +174 -0
- package/scripts/engines_builders_batch12.py +103 -0
- package/scripts/engines_builders_batch13.py +340 -0
- package/scripts/engines_builders_batch9.py +1468 -0
- package/scripts/engines_builders_cn.py +2283 -0
- package/scripts/engines_builders_data.py +2923 -0
- package/scripts/engines_builders_data_macro.py +775 -0
- package/scripts/engines_builders_feeds.py +255 -0
- package/scripts/engines_builders_intl.py +1239 -0
- package/scripts/engines_builders_search.py +440 -0
- package/scripts/engines_builders_tech.py +1360 -0
- package/scripts/evidence.py +588 -162
- package/scripts/evidence_loop.py +592 -0
- package/scripts/evidence_tier.py +298 -0
- package/scripts/except_sets.py +25 -0
- package/scripts/extract.py +55 -16
- package/scripts/fact_align.py +207 -0
- package/scripts/fact_crosscheck.py +230 -0
- package/scripts/fetch.py +25 -12
- package/scripts/fetch_browser.py +87 -0
- package/scripts/fetch_jsrun.py +122 -0
- package/scripts/fetch_quality.py +225 -0
- package/scripts/fetch_render_tinyfish.py +117 -0
- package/scripts/fetch_v3.py +1779 -0
- package/scripts/focus_extract.py +350 -0
- package/scripts/fulltext_store.py +169 -0
- package/scripts/fxtwitter_api.py +325 -0
- package/scripts/gen_engine_catalog.py +364 -0
- package/scripts/gen_native_tools.py +91 -0
- package/scripts/gen_routing_snapshot.py +184 -0
- package/scripts/health_check.py +140 -402
- package/scripts/health_probe.py +23 -8
- package/scripts/hot_state.py +134 -0
- package/scripts/http_client.py +1053 -0
- package/scripts/image_ops.py +430 -0
- package/scripts/image_vision/vision_probe +0 -0
- package/scripts/image_vision/vision_probe.swift +126 -0
- package/scripts/install.ps1 +148 -0
- package/scripts/install.sh +206 -0
- package/scripts/job.py +1186 -0
- package/scripts/jump_resolver.py +235 -0
- package/scripts/lang_capability.py +150 -0
- package/scripts/lang_detect.py +289 -0
- package/scripts/lang_matrix_probe.py +329 -0
- package/scripts/lang_pref.py +311 -0
- package/scripts/link_source.py +340 -0
- package/scripts/local_engines.py +321 -0
- package/scripts/local_image.py +970 -0
- package/scripts/local_seek.py +366 -0
- package/scripts/macro_countries.py +82 -0
- package/scripts/matrix_search_eval.py +1105 -0
- package/scripts/mcp_diag.py +8 -3
- package/scripts/mcp_handlers.py +1074 -0
- package/scripts/mcp_handlers_surface.py +157 -0
- package/scripts/mcp_launch.sh +122 -0
- package/scripts/mcp_server.py +118 -417
- package/scripts/mcp_setup.py +407 -0
- package/scripts/mcp_tools.py +330 -0
- package/scripts/mcp_transport.py +191 -0
- package/scripts/min_version_scan.py +335 -0
- package/scripts/net_proxy.py +258 -0
- package/scripts/network_aware.py +152 -0
- package/scripts/paper.py +274 -0
- package/scripts/pdf_extract.py +518 -0
- package/scripts/phrase_filter.py +66 -0
- package/scripts/plan.py +422 -0
- package/scripts/postprocess_signals.py +150 -0
- package/scripts/pre-commit +48 -0
- package/scripts/query_enhance.py +157 -0
- package/scripts/query_operators.py +196 -0
- package/scripts/query_rewriter.py +116 -7
- package/scripts/query_signals.py +300 -0
- package/scripts/query_templates.py +167 -0
- package/scripts/query_understanding.py +411 -0
- package/scripts/query_variants.py +298 -0
- package/scripts/quota.py +488 -76
- package/scripts/rank_signals.py +775 -0
- package/scripts/ranking_eval.py +319 -0
- package/scripts/readability_extract.py +433 -0
- package/scripts/recompute.py +415 -0
- package/scripts/recovery.py +616 -0
- package/scripts/recovery_l0.py +160 -0
- package/scripts/redskill/redskill_engine.py +332 -0
- package/scripts/regression_p0p1.py +318 -0
- package/scripts/replay_eval.py +640 -0
- package/scripts/rerank_tfidf.py +225 -0
- package/scripts/research.py +591 -403
- package/scripts/research_cli.py +361 -0
- package/scripts/research_dossier.py +492 -0
- package/scripts/research_expand.py +146 -0
- package/scripts/research_gates.py +272 -0
- package/scripts/research_report.py +209 -0
- package/scripts/research_strategy.py +37 -0
- package/scripts/research_work_packages.py +195 -0
- package/scripts/result_lang.py +311 -0
- package/scripts/robots_guard.py +259 -0
- package/scripts/route.py +921 -254
- package/scripts/route_bangs.py +40 -0
- package/scripts/route_cache.py +225 -0
- package/scripts/route_combo.py +910 -0
- package/scripts/route_domains.py +328 -0
- package/scripts/route_lang.py +605 -0
- package/scripts/route_log.py +63 -0
- package/scripts/route_policy.py +250 -0
- package/scripts/rrf_classic.py +66 -0
- package/scripts/screenshot.py +73 -0
- package/scripts/search.py +897 -405
- package/scripts/search_attribution.py +88 -0
- package/scripts/search_benchmark.py +407 -0
- package/scripts/search_cli.py +627 -0
- package/scripts/search_entry.py +389 -0
- package/scripts/search_output.py +606 -0
- package/scripts/search_pipeline.py +1077 -0
- package/scripts/search_rank.py +991 -0
- package/scripts/search_types.py +27 -12
- package/scripts/seek_locator.py +68 -0
- package/scripts/semantic_evidence.py +215 -0
- package/scripts/serp_guard.py +228 -0
- package/scripts/serp_kill_log.py +51 -0
- package/scripts/single_flight.py +120 -0
- package/scripts/social_engines/__init__.py +10 -0
- package/scripts/social_engines/bilibili_engine.py +131 -0
- package/scripts/social_engines/hackernews_engine.py +113 -0
- package/scripts/social_engines/reddit_engine.py +172 -0
- package/scripts/social_engines/twitter_engine.py +397 -0
- package/scripts/social_engines/v2ex_engine.py +101 -0
- package/scripts/social_engines/weibo_engine.py +128 -0
- package/scripts/social_engines/xiaohongshu_engine.py +137 -0
- package/scripts/social_engines/zhihu_engine.py +139 -0
- package/scripts/social_research.py +201 -0
- package/scripts/sogou_weixin_browser.py +318 -0
- package/scripts/source_ledger.py +249 -0
- package/scripts/stage_timing.py +58 -0
- package/scripts/stats_cli.py +95 -0
- package/scripts/straggler_policy.py +60 -0
- package/scripts/stylometry_detector.py +231 -0
- package/scripts/sync_backends.py +353 -0
- package/scripts/sync_installs.py +29 -0
- package/scripts/template_cli.py +127 -0
- package/scripts/text_lexicon.py +72 -0
- package/scripts/tfidf_router.py +109 -37
- package/scripts/time_utils.py +233 -0
- package/scripts/topic_research_profiles.py +643 -0
- package/scripts/train.py +737 -0
- package/scripts/tweet.py +596 -0
- package/scripts/url_canon.py +217 -0
- package/scripts/url_optimize.py +56 -0
- package/scripts/url_safety.py +236 -0
- package/scripts/usage_log.py +116 -0
- package/scripts/usage_log_clean.py +113 -0
- package/scripts/v2ex_nodes.py +311 -0
- package/scripts/variant_recall.py +188 -0
- package/scripts/watch.py +205 -0
- package/scripts/wechat_article.py +258 -0
- package/scripts/wide_research.py +402 -0
- package/scripts/wx.py +349 -0
- package/scripts/x_article_fetch.py +141 -0
- package/scripts/x_search.py +234 -0
- package/scripts/x_trending.py +189 -0
- package/scripts/yaml_load.py +67 -0
- package/sub-skills/ego-search/SKILL.md +253 -0
- package/sub-skills/ego-search/references/browser-runtime.md +43 -0
- package/sub-skills/ego-search/references/install.md +67 -0
- package/sub-skills/ego-search/references/original-ego-upgrade.md +51 -0
- package/sub-skills/ego-search/scripts/ego_search.py +800 -0
- package/sub-skills/ego-search/scripts/merge.py +226 -0
- package/sub-skills/ego-search/scripts/quality.py +64 -0
- package/sub-skills/ego-search/scripts/runtime.py +172 -0
- package/sub-skills/ego-search/scripts/safety.py +109 -0
- package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
- package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
- package/sub-skills/ego-search/tests/test_query_normalize.py +68 -0
- package/sub-skills/js-run/SKILL.md +67 -0
- package/sub-skills/js-run/scripts/jsrun.py +112 -0
- package/sub-skills/js-run/scripts/shim.js +188 -0
- package/sub-skills/js-run/tests/fixtures/challenge_like.js +27 -0
- package/sub-skills/js-run/tests/test_jsrun.py +165 -0
- package/sub-skills/local-search/SKILL.md +50 -53
- package/sub-skills/local-search/bm25.py +162 -0
- package/sub-skills/local-search/config.yaml +152 -55
- package/sub-skills/local-search/engine_registry.py +91 -26
- package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
- package/sub-skills/local-search/local_search_adapter.py +13 -2
- package/sub-skills/local-search/parse_maps.yaml +40 -42
- package/sub-skills/local-search/search_v3.py +1182 -65
- package/sub-skills/local-search/smart_router.py +47 -7
- package/sub-skills/local-seek/SKILL.md +110 -0
- package/sub-skills/local-seek/config/domains.yaml +20 -0
- package/sub-skills/local-seek/references/auto-behaviors.md +24 -0
- package/sub-skills/local-seek/references/fd-mdfind-recipes.md +62 -0
- package/sub-skills/local-seek/references/git-integration.md +54 -0
- package/sub-skills/local-seek/references/remote-handoff.md +46 -0
- package/sub-skills/local-seek/references/rg-recipes.md +96 -0
- package/sub-skills/local-seek/references/strategies.md +52 -0
- package/sub-skills/local-seek/references/structural-search.md +59 -0
- package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
- package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
- package/sub-skills/local-seek/scripts/seek.py +1465 -0
- package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
- package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
- package/sub-skills/local-seek/scripts/seek_rank.py +193 -0
- package/sub-skills/local-seek/tests/test_content_rank.py +222 -0
- package/sub-skills/local-seek/tests/test_determinism_20261006.py +145 -0
- package/sub-skills/local-seek/tests/test_noise_tier.py +101 -0
- package/sub-skills/local-seek/tests/test_pcre2_probe.py +136 -0
- package/sub-skills/local-seek/tests/test_rank_pinyin.py +77 -0
- package/sub-skills/local-seek/tests/test_regression_20261003.py +169 -0
- package/sub-skills/local-seek/tests/test_regression_20261006.py +146 -0
- package/sub-skills/local-seek/tests/test_seek_degradation.py +111 -0
- package/sub-skills/local-seek/tests/test_seek_fts.py +313 -0
|
@@ -0,0 +1,2923 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""专用构建器:开放数据 / 包索引 / 金融工具 / Octen"""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import logging
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import time
|
|
11
|
+
import urllib.error
|
|
12
|
+
import urllib.parse
|
|
13
|
+
import urllib.request
|
|
14
|
+
from datetime import datetime
|
|
15
|
+
from html import unescape as _unescape
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from cjk_tokens import cjk_term_grams, signal_tokens
|
|
19
|
+
from engine_env import get_env
|
|
20
|
+
from engines_base import (
|
|
21
|
+
rank_score,
|
|
22
|
+
safe_search, _run, _resolve, _get_path, _coerce_field, _detect_anti_bot,
|
|
23
|
+
http_open,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger("unified_search.engines")
|
|
27
|
+
|
|
28
|
+
# ── Open Library 图书引擎 ────────────────────────────────────────────────────
|
|
29
|
+
|
|
30
|
+
def _build_open_library_engine(spec: dict[str, Any]) -> Any:
|
|
31
|
+
"""Open Library 图书搜索(openlibrary.org/search.json,免认证)"""
|
|
32
|
+
timeout = spec.get("timeout", 10)
|
|
33
|
+
|
|
34
|
+
@safe_search
|
|
35
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
36
|
+
import urllib.parse as up
|
|
37
|
+
to = _timeout or timeout
|
|
38
|
+
# OpenLibrary 要求查询至少 3 个字符(如「三体」只有 2 字会 422)
|
|
39
|
+
q = query if len(query) >= 3 else query * 2
|
|
40
|
+
url = "https://openlibrary.org/search.json?" + up.urlencode({
|
|
41
|
+
"q": q, "limit": min(n, 20),
|
|
42
|
+
})
|
|
43
|
+
try:
|
|
44
|
+
with http_open(urllib.request.Request(
|
|
45
|
+
url, headers={"User-Agent": "argo-search/2.4 (unified-search@local)"}), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
46
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
47
|
+
except Exception as e:
|
|
48
|
+
logger.warning(f"Open Library 失败: {e}")
|
|
49
|
+
return []
|
|
50
|
+
results = []
|
|
51
|
+
for _rk, d in enumerate((data.get("docs") or [])[:n]):
|
|
52
|
+
title = d.get("title", "")
|
|
53
|
+
key = d.get("key", "")
|
|
54
|
+
if not title and not key:
|
|
55
|
+
continue
|
|
56
|
+
authors = ", ".join((d.get("author_name") or [])[:3])
|
|
57
|
+
year = d.get("first_publish_year") or ""
|
|
58
|
+
parts = [p for p in (authors, str(year)) if p]
|
|
59
|
+
results.append({
|
|
60
|
+
"title": title,
|
|
61
|
+
"url": f"https://openlibrary.org{key}" if key else "",
|
|
62
|
+
"snippet": " · ".join(parts)[:300],
|
|
63
|
+
"source": "open_library",
|
|
64
|
+
"score": rank_score(0.7, _rk),
|
|
65
|
+
})
|
|
66
|
+
return results
|
|
67
|
+
return _engine
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# ── 微信读书图书引擎 ─────────────────────────────────────────────────────────
|
|
71
|
+
|
|
72
|
+
def _build_weread_engine(spec: dict[str, Any]) -> Any:
|
|
73
|
+
"""微信读书图书搜索(Agent Gateway /store/search,需 WEREAD_API_KEY)。
|
|
74
|
+
|
|
75
|
+
返回中文书目为主,附评分/评分人数/在读人数,比 Open Library 更适合中文图书。
|
|
76
|
+
"""
|
|
77
|
+
timeout = spec.get("timeout", 10)
|
|
78
|
+
gateway = "https://i.weread.qq.com/api/agent/gateway"
|
|
79
|
+
|
|
80
|
+
@safe_search
|
|
81
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
82
|
+
import urllib.parse as up
|
|
83
|
+
to = _timeout or timeout
|
|
84
|
+
key = get_env(["ARGO_WEREAD_API_KEY", "WEREAD_API_KEY"])
|
|
85
|
+
if not key:
|
|
86
|
+
logger.warning("Weread 缺 WEREAD_API_KEY")
|
|
87
|
+
return []
|
|
88
|
+
body = json.dumps({
|
|
89
|
+
"api_name": "/store/search",
|
|
90
|
+
"keyword": query,
|
|
91
|
+
"count": min(n, 10),
|
|
92
|
+
"skill_version": "1.0.3",
|
|
93
|
+
}).encode("utf-8")
|
|
94
|
+
req = urllib.request.Request(
|
|
95
|
+
gateway, data=body, method="POST",
|
|
96
|
+
headers={
|
|
97
|
+
"Authorization": f"Bearer {key}",
|
|
98
|
+
"Content-Type": "application/json",
|
|
99
|
+
"User-Agent": "argo-search/2.4 (unified-search@local)",
|
|
100
|
+
},
|
|
101
|
+
)
|
|
102
|
+
try:
|
|
103
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
104
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
105
|
+
except Exception as e:
|
|
106
|
+
logger.warning(f"微信读书搜索失败: {e}")
|
|
107
|
+
return []
|
|
108
|
+
if isinstance(data, dict) and data.get("errcode"):
|
|
109
|
+
logger.warning(f"微信读书 API 错误: {data.get('errcode')} {data.get('errmsg') or ''}")
|
|
110
|
+
return []
|
|
111
|
+
results = []
|
|
112
|
+
seen: set[str] = set()
|
|
113
|
+
for group in (data.get("results") or []):
|
|
114
|
+
for item in (group.get("books") or [])[:n]:
|
|
115
|
+
bi = item.get("bookInfo") or {}
|
|
116
|
+
title = bi.get("title", "")
|
|
117
|
+
if not title or title in seen:
|
|
118
|
+
continue
|
|
119
|
+
seen.add(title)
|
|
120
|
+
author = bi.get("author", "")
|
|
121
|
+
rating = bi.get("newRating")
|
|
122
|
+
rating_txt = f"{rating / 10:.1f}分" if isinstance(rating, (int, float)) and rating > 0 else ""
|
|
123
|
+
rating_count = bi.get("newRatingCount")
|
|
124
|
+
reading_count = item.get("readingCount")
|
|
125
|
+
parts = [p for p in (
|
|
126
|
+
author,
|
|
127
|
+
rating_txt,
|
|
128
|
+
f"{rating_count}人评分" if rating_count else "",
|
|
129
|
+
f"{reading_count}人在读" if reading_count else "",
|
|
130
|
+
) if p]
|
|
131
|
+
results.append({
|
|
132
|
+
"title": title,
|
|
133
|
+
"url": bi.get("deepLink") or f"https://weread.qq.com/book-detail?type=1&bookId={bi.get('bookId', '')}",
|
|
134
|
+
"snippet": " · ".join(parts)[:300],
|
|
135
|
+
"source": "weread",
|
|
136
|
+
"score": rank_score(0.85, len(results)),
|
|
137
|
+
})
|
|
138
|
+
if len(results) >= n:
|
|
139
|
+
break
|
|
140
|
+
if len(results) >= n:
|
|
141
|
+
break
|
|
142
|
+
return results
|
|
143
|
+
return _engine
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
# ── 豆瓣读书引擎 ─────────────────────────────────────────────────────────────
|
|
147
|
+
|
|
148
|
+
def _build_douban_book_engine(spec: dict[str, Any]) -> Any:
|
|
149
|
+
"""豆瓣读书搜索(search.douban.com/book/subject_search,免认证)。
|
|
150
|
+
|
|
151
|
+
页面内嵌 window.__DATA__ JSON,含评分/评分人数/出版社/年份/价格,
|
|
152
|
+
与微信读书互补(出版社维度)。无官方 API,走网页内嵌数据。
|
|
153
|
+
"""
|
|
154
|
+
timeout = spec.get("timeout", 10)
|
|
155
|
+
_UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
|
156
|
+
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36")
|
|
157
|
+
|
|
158
|
+
@safe_search
|
|
159
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
160
|
+
import urllib.parse as up
|
|
161
|
+
to = _timeout or timeout
|
|
162
|
+
url = "https://search.douban.com/book/subject_search?" + up.urlencode({
|
|
163
|
+
"search_text": query, "cat": "1001",
|
|
164
|
+
})
|
|
165
|
+
try:
|
|
166
|
+
req = urllib.request.Request(url, headers={"User-Agent": _UA})
|
|
167
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
168
|
+
html = resp.read().decode("utf-8", "replace")
|
|
169
|
+
except Exception as e:
|
|
170
|
+
logger.warning(f"豆瓣读书搜索失败: {e}")
|
|
171
|
+
return []
|
|
172
|
+
marker = "window.__DATA__ = "
|
|
173
|
+
i = html.find(marker)
|
|
174
|
+
if i < 0:
|
|
175
|
+
return []
|
|
176
|
+
try:
|
|
177
|
+
dec = json.JSONDecoder()
|
|
178
|
+
data, _ = dec.raw_decode(html[i + len(marker):].lstrip())
|
|
179
|
+
except Exception as e:
|
|
180
|
+
logger.warning(f"豆瓣 __DATA__ 解析失败: {e}")
|
|
181
|
+
return []
|
|
182
|
+
results = []
|
|
183
|
+
seen: set[str] = set()
|
|
184
|
+
for _rk2, item in enumerate((data.get("items") or [])):
|
|
185
|
+
if item.get("tpl_name") != "search_subject":
|
|
186
|
+
continue
|
|
187
|
+
title = item.get("title", "")
|
|
188
|
+
if not title or title in seen:
|
|
189
|
+
continue
|
|
190
|
+
seen.add(title)
|
|
191
|
+
rating = item.get("rating") or {}
|
|
192
|
+
rv = rating.get("value")
|
|
193
|
+
rc = rating.get("count")
|
|
194
|
+
parts = [p for p in (
|
|
195
|
+
item.get("abstract") or "",
|
|
196
|
+
f"{rv}分" if rv else "",
|
|
197
|
+
f"{rc}人评分" if rc else "",
|
|
198
|
+
) if p]
|
|
199
|
+
results.append({
|
|
200
|
+
"title": title,
|
|
201
|
+
"url": item.get("url") or f"https://book.douban.com/subject/{item.get('id', '')}/",
|
|
202
|
+
"snippet": " · ".join(parts)[:300],
|
|
203
|
+
"source": "douban_book",
|
|
204
|
+
"score": rank_score(0.85, _rk2),
|
|
205
|
+
})
|
|
206
|
+
if len(results) >= n:
|
|
207
|
+
break
|
|
208
|
+
return results
|
|
209
|
+
return _engine
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _build_free_dictionary_engine(spec: dict[str, Any]) -> Any:
|
|
213
|
+
"""英英词典(dictionaryapi.dev,免认证,响应为数组需专门解析)"""
|
|
214
|
+
timeout = spec.get("timeout", 8)
|
|
215
|
+
|
|
216
|
+
@safe_search
|
|
217
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
218
|
+
import urllib.parse as up
|
|
219
|
+
to = _timeout or timeout
|
|
220
|
+
q = re.sub(r"(?i)^(what\s+(does|is|do|are)\s+|define\s+|definition\s+of\s+|meaning\s+of\s+|the\s+word\s+)", "", query.strip())
|
|
221
|
+
q = re.sub(r"(是什么意思|什么意思|怎么读|读音|英文|单词|的含义|的定义|咋读)$", "", q).strip()
|
|
222
|
+
word = re.split(r"\s+", q)[0] if q else ""
|
|
223
|
+
if not word:
|
|
224
|
+
return []
|
|
225
|
+
url = f"https://api.dictionaryapi.dev/api/v2/entries/en/{up.quote(word)}"
|
|
226
|
+
try:
|
|
227
|
+
with http_open(urllib.request.Request(
|
|
228
|
+
url, headers={"User-Agent": "argo-search/2.4"}), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
229
|
+
entries = json.loads(resp.read().decode("utf-8", "replace"))
|
|
230
|
+
except Exception:
|
|
231
|
+
return []
|
|
232
|
+
if not isinstance(entries, list):
|
|
233
|
+
return []
|
|
234
|
+
results = []
|
|
235
|
+
for _rk3, e in enumerate(entries[:n]):
|
|
236
|
+
w = e.get("word", "")
|
|
237
|
+
if not w:
|
|
238
|
+
continue
|
|
239
|
+
urls = e.get("sourceUrls") or []
|
|
240
|
+
eurl = urls[0] if urls else f"https://en.wiktionary.org/wiki/{w}"
|
|
241
|
+
defs = []
|
|
242
|
+
for m in (e.get("meanings") or [])[:3]:
|
|
243
|
+
for d in (m.get("definitions") or [])[:2]:
|
|
244
|
+
defs.append(d.get("definition", ""))
|
|
245
|
+
results.append({
|
|
246
|
+
"title": w,
|
|
247
|
+
"url": eurl,
|
|
248
|
+
"snippet": " / ".join(x for x in defs if x)[:300],
|
|
249
|
+
"source": "free_dictionary",
|
|
250
|
+
"score": rank_score(0.8, _rk3),
|
|
251
|
+
})
|
|
252
|
+
return results
|
|
253
|
+
return _engine
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
# ── 百度百科(suggest 接口,免认证)──────────────────────────────────────────
|
|
257
|
+
|
|
258
|
+
def _build_baidu_baike_engine(spec: dict[str, Any]) -> Any:
|
|
259
|
+
"""百度百科词条搜索:优先 searchui/suggest,OpenAPI 卡片作精确补充。"""
|
|
260
|
+
timeout = spec.get("timeout", 8)
|
|
261
|
+
|
|
262
|
+
@safe_search
|
|
263
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
264
|
+
import urllib.parse as up
|
|
265
|
+
to = _timeout or timeout
|
|
266
|
+
q = query.strip()
|
|
267
|
+
if not q:
|
|
268
|
+
return []
|
|
269
|
+
headers = {
|
|
270
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120.0 Safari/537.36",
|
|
271
|
+
"Referer": "https://baike.baidu.com/",
|
|
272
|
+
"Accept": "application/json,text/plain,*/*",
|
|
273
|
+
}
|
|
274
|
+
results: list[dict[str, Any]] = []
|
|
275
|
+
# 1) suggest 多候选
|
|
276
|
+
sug_url = "https://baike.baidu.com/api/searchui/suggest?" + up.urlencode({"enc": "utf8", "wd": q})
|
|
277
|
+
try:
|
|
278
|
+
with http_open(urllib.request.Request(sug_url, headers=headers), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
279
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
280
|
+
for _rk25, item in enumerate((data.get("list") or [])[:n]):
|
|
281
|
+
title = item.get("lemmaTitle") or ""
|
|
282
|
+
lid = item.get("lemmaId") or ""
|
|
283
|
+
desc = item.get("lemmaDesc") or ""
|
|
284
|
+
if not title:
|
|
285
|
+
continue
|
|
286
|
+
path = up.quote(title)
|
|
287
|
+
url = f"https://baike.baidu.com/item/{path}/{lid}" if lid else f"https://baike.baidu.com/item/{path}"
|
|
288
|
+
results.append({
|
|
289
|
+
"title": title,
|
|
290
|
+
"url": url,
|
|
291
|
+
"snippet": str(desc)[:300],
|
|
292
|
+
"source": "baidu_baike",
|
|
293
|
+
"score": rank_score(0.85, _rk25),
|
|
294
|
+
})
|
|
295
|
+
except Exception as e:
|
|
296
|
+
logger.warning(f"baidu_baike suggest 失败: {e}")
|
|
297
|
+
# 2) OpenAPI 卡片补充(精确词条摘要,可能 errno=2)
|
|
298
|
+
if len(results) < n:
|
|
299
|
+
card_url = "https://baike.baidu.com/api/openapi/BaikeLemmaCardApi?" + up.urlencode({
|
|
300
|
+
"scope": "103", "format": "json", "appid": "379020",
|
|
301
|
+
"bk_key": q, "bk_length": "600",
|
|
302
|
+
})
|
|
303
|
+
try:
|
|
304
|
+
with http_open(urllib.request.Request(card_url, headers=headers), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
305
|
+
card = json.loads(resp.read().decode("utf-8", "replace"))
|
|
306
|
+
if isinstance(card, dict) and card.get("errno") in (None, 0) and (card.get("title") or card.get("key")):
|
|
307
|
+
title = card.get("title") or card.get("key") or q
|
|
308
|
+
abstract = card.get("abstract") or card.get("desc") or ""
|
|
309
|
+
curl = card.get("url") or f"https://baike.baidu.com/item/{up.quote(str(title))}"
|
|
310
|
+
if not any(r.get("title") == title for r in results):
|
|
311
|
+
results.insert(0, {
|
|
312
|
+
"title": str(title),
|
|
313
|
+
"url": str(curl),
|
|
314
|
+
"snippet": str(abstract)[:300],
|
|
315
|
+
"source": "baidu_baike",
|
|
316
|
+
"score": 0.9,
|
|
317
|
+
})
|
|
318
|
+
except Exception:
|
|
319
|
+
pass
|
|
320
|
+
# 相关度检查:零重叠词条丢弃(触发 recovery)。中文走 2-gram,见 cjk_tokens:
|
|
321
|
+
# 旧写法对无空格中文把整句当一个 token,子串包含恒假,词条全被丢弃。
|
|
322
|
+
q_keys = set(signal_tokens(q, stop={
|
|
323
|
+
"the", "and", "for", "year", "founded", "founding", "headquarters",
|
|
324
|
+
"where", "what", "when", "which", "with", "from", "that", "this",
|
|
325
|
+
"年份", "时间", "成立", "创办", "创立", "总部", "职能", "简介",
|
|
326
|
+
}))
|
|
327
|
+
if q_keys:
|
|
328
|
+
filtered = []
|
|
329
|
+
for r in results:
|
|
330
|
+
blob = f"{r.get('title','')} {r.get('snippet','')}".lower()
|
|
331
|
+
if any(k in blob for k in q_keys):
|
|
332
|
+
filtered.append(r)
|
|
333
|
+
results = filtered
|
|
334
|
+
return results[:n]
|
|
335
|
+
return _engine
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
# ── PyPI(精确包名 JSON;搜索页有 JS challenge 故不做 HTML 搜)────────────────
|
|
339
|
+
|
|
340
|
+
def _build_pypi_engine(spec: dict[str, Any]) -> Any:
|
|
341
|
+
"""PyPI 包查询:从查询词抽取候选包名,逐个请求 /pypi/{name}/json。"""
|
|
342
|
+
timeout = spec.get("timeout", 10)
|
|
343
|
+
|
|
344
|
+
def _candidates(query: str) -> list[str]:
|
|
345
|
+
q = query.strip()
|
|
346
|
+
# 去掉常见修饰语
|
|
347
|
+
q2 = re.sub(
|
|
348
|
+
r"(?i)\b(pypi|python\s+package|python\s+lib(?:rary)?|pip\s+install|package|库|包|模块|依赖)\b",
|
|
349
|
+
" ", q,
|
|
350
|
+
)
|
|
351
|
+
q2 = re.sub(r"\s+", " ", q2).strip()
|
|
352
|
+
cands: list[str] = []
|
|
353
|
+
for raw in (q2, q, *re.findall(r"[A-Za-z0-9][A-Za-z0-9._-]{1,80}", q2)):
|
|
354
|
+
name = raw.strip().strip("\"'").lower().replace(" ", "-")
|
|
355
|
+
name = re.sub(r"[^a-z0-9._-]", "", name)
|
|
356
|
+
if len(name) >= 2 and name not in cands:
|
|
357
|
+
cands.append(name)
|
|
358
|
+
return cands[:8]
|
|
359
|
+
|
|
360
|
+
@safe_search
|
|
361
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
362
|
+
to = _timeout or timeout
|
|
363
|
+
results: list[dict[str, Any]] = []
|
|
364
|
+
seen: set[str] = set()
|
|
365
|
+
headers = {"User-Agent": "argo-search/2.5 (unified-search@local)", "Accept": "application/json"}
|
|
366
|
+
for _rk4, name in enumerate(_candidates(query)):
|
|
367
|
+
if len(results) >= n:
|
|
368
|
+
break
|
|
369
|
+
url = f"https://pypi.org/pypi/{name}/json"
|
|
370
|
+
try:
|
|
371
|
+
with http_open(urllib.request.Request(url, headers=headers), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
372
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
373
|
+
except Exception:
|
|
374
|
+
continue
|
|
375
|
+
info = data.get("info") if isinstance(data, dict) else None
|
|
376
|
+
if not isinstance(info, dict):
|
|
377
|
+
continue
|
|
378
|
+
pkg = (info.get("name") or name).strip()
|
|
379
|
+
key = pkg.lower()
|
|
380
|
+
if key in seen:
|
|
381
|
+
continue
|
|
382
|
+
seen.add(key)
|
|
383
|
+
summary = info.get("summary") or ""
|
|
384
|
+
ver = info.get("version") or ""
|
|
385
|
+
home = info.get("package_url") or info.get("project_url") or f"https://pypi.org/project/{pkg}/"
|
|
386
|
+
snippet = f"{summary} · v{ver}".strip(" ·") if ver else summary
|
|
387
|
+
results.append({
|
|
388
|
+
"title": pkg,
|
|
389
|
+
"url": home,
|
|
390
|
+
"snippet": snippet[:300],
|
|
391
|
+
"source": "pypi",
|
|
392
|
+
"score": rank_score(0.9, _rk4),
|
|
393
|
+
})
|
|
394
|
+
return results[:n]
|
|
395
|
+
return _engine
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
# ── ClinicalTrials.gov v2 ────────────────────────────────────────────────────
|
|
399
|
+
|
|
400
|
+
def _build_clinicaltrials_engine(spec: dict[str, Any]) -> Any:
|
|
401
|
+
"""ClinicalTrials.gov API v2 试验检索(免认证)。"""
|
|
402
|
+
timeout = spec.get("timeout", 12)
|
|
403
|
+
|
|
404
|
+
@safe_search
|
|
405
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
406
|
+
import urllib.parse as up
|
|
407
|
+
to = _timeout or timeout
|
|
408
|
+
url = "https://clinicaltrials.gov/api/v2/studies?" + up.urlencode({
|
|
409
|
+
"query.term": query,
|
|
410
|
+
"pageSize": min(n, 20),
|
|
411
|
+
"format": "json",
|
|
412
|
+
})
|
|
413
|
+
try:
|
|
414
|
+
with http_open(urllib.request.Request(
|
|
415
|
+
url, headers={"User-Agent": "argo-search/2.5", "Accept": "application/json"}), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
416
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
417
|
+
except Exception as e:
|
|
418
|
+
logger.warning(f"clinicaltrials 失败: {e}")
|
|
419
|
+
return []
|
|
420
|
+
results = []
|
|
421
|
+
for _rk5, st in enumerate((data.get("studies") or [])[:n]):
|
|
422
|
+
ps = st.get("protocolSection") or {}
|
|
423
|
+
ident = ps.get("identificationModule") or {}
|
|
424
|
+
desc = ps.get("descriptionModule") or {}
|
|
425
|
+
status = (ps.get("statusModule") or {}).get("overallStatus") or ""
|
|
426
|
+
nct = ident.get("nctId") or ""
|
|
427
|
+
title = ident.get("briefTitle") or ident.get("officialTitle") or nct
|
|
428
|
+
summary = desc.get("briefSummary") or status
|
|
429
|
+
if not title and not nct:
|
|
430
|
+
continue
|
|
431
|
+
results.append({
|
|
432
|
+
"title": str(title)[:200],
|
|
433
|
+
"url": f"https://clinicaltrials.gov/study/{nct}" if nct else "",
|
|
434
|
+
"snippet": str(summary).replace("\n", " ")[:300],
|
|
435
|
+
"source": "clinicaltrials",
|
|
436
|
+
"score": rank_score(0.85, _rk5),
|
|
437
|
+
})
|
|
438
|
+
return results
|
|
439
|
+
return _engine
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
# ── openFDA 药品标签 ─────────────────────────────────────────────────────────
|
|
443
|
+
|
|
444
|
+
def _build_openfda_engine(spec: dict[str, Any]) -> Any:
|
|
445
|
+
"""openFDA drug label 搜索(免认证)。"""
|
|
446
|
+
timeout = spec.get("timeout", 12)
|
|
447
|
+
|
|
448
|
+
@safe_search
|
|
449
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
450
|
+
import urllib.parse as up
|
|
451
|
+
to = _timeout or timeout
|
|
452
|
+
# 清洗查询,优先 brand/generic 字段检索
|
|
453
|
+
q = re.sub(r"(?i)\b(drug|medicine|药品|药物|说明书|openfda|fda)\b", " ", query).strip()
|
|
454
|
+
q = q or query.strip()
|
|
455
|
+
search = f'openfda.brand_name:"{q}" openfda.generic_name:"{q}"'
|
|
456
|
+
url = "https://api.fda.gov/drug/label.json?" + up.urlencode({
|
|
457
|
+
"search": search,
|
|
458
|
+
"limit": min(n, 20),
|
|
459
|
+
})
|
|
460
|
+
try:
|
|
461
|
+
with http_open(urllib.request.Request(
|
|
462
|
+
url, headers={"User-Agent": "argo-search/2.5", "Accept": "application/json"}), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
463
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
464
|
+
except Exception as e:
|
|
465
|
+
# 回退:全文检索
|
|
466
|
+
try:
|
|
467
|
+
url2 = "https://api.fda.gov/drug/label.json?" + up.urlencode({
|
|
468
|
+
"search": q, "limit": min(n, 20),
|
|
469
|
+
})
|
|
470
|
+
with http_open(urllib.request.Request(
|
|
471
|
+
url2, headers={"User-Agent": "argo-search/2.5"}), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
472
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
473
|
+
except Exception as e2:
|
|
474
|
+
logger.warning(f"openfda 失败: {e2}")
|
|
475
|
+
return []
|
|
476
|
+
results = []
|
|
477
|
+
for _rk6, item in enumerate((data.get("results") or [])[:n]):
|
|
478
|
+
of = item.get("openfda") or {}
|
|
479
|
+
brands = of.get("brand_name") or []
|
|
480
|
+
generics = of.get("generic_name") or []
|
|
481
|
+
title = (brands[0] if brands else "") or (generics[0] if generics else "") or item.get("id", "drug label")
|
|
482
|
+
purpose = item.get("purpose") or item.get("indications_and_usage") or []
|
|
483
|
+
if isinstance(purpose, list):
|
|
484
|
+
purpose = purpose[0] if purpose else ""
|
|
485
|
+
set_id = (of.get("spl_set_id") or [None])[0] if isinstance(of.get("spl_set_id"), list) else of.get("spl_set_id")
|
|
486
|
+
url_out = f"https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid={set_id}" if set_id else "https://open.fda.gov/apis/drug/label/"
|
|
487
|
+
results.append({
|
|
488
|
+
"title": str(title)[:200],
|
|
489
|
+
"url": url_out,
|
|
490
|
+
"snippet": str(purpose).replace("\n", " ")[:300],
|
|
491
|
+
"source": "openfda",
|
|
492
|
+
"score": rank_score(0.8, _rk6),
|
|
493
|
+
})
|
|
494
|
+
return results
|
|
495
|
+
return _engine
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
# ── 掘金搜索 ─────────────────────────────────────────────────────────────────
|
|
499
|
+
|
|
500
|
+
def _build_juejin_engine(spec: dict[str, Any]) -> Any:
|
|
501
|
+
"""掘金文章搜索(search_api,免认证)。"""
|
|
502
|
+
timeout = spec.get("timeout", 10)
|
|
503
|
+
|
|
504
|
+
@safe_search
|
|
505
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
506
|
+
import urllib.parse as up
|
|
507
|
+
to = _timeout or timeout
|
|
508
|
+
# 官方搜索接口(GET)
|
|
509
|
+
url = "https://api.juejin.cn/search_api/v1/search?" + up.urlencode({
|
|
510
|
+
"query": query,
|
|
511
|
+
"id_type": 0,
|
|
512
|
+
"cursor": "0",
|
|
513
|
+
"limit": min(n, 20),
|
|
514
|
+
"search_type": 0,
|
|
515
|
+
"sort_type": 0,
|
|
516
|
+
})
|
|
517
|
+
headers = {
|
|
518
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120.0 Safari/537.36",
|
|
519
|
+
"Accept": "application/json",
|
|
520
|
+
"Referer": "https://juejin.cn/",
|
|
521
|
+
}
|
|
522
|
+
try:
|
|
523
|
+
with http_open(urllib.request.Request(url, headers=headers), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
524
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
525
|
+
except Exception as e:
|
|
526
|
+
logger.warning(f"juejin 失败: {e}")
|
|
527
|
+
return []
|
|
528
|
+
results = []
|
|
529
|
+
for _rk7, item in enumerate((data.get("data") or [])[: n * 2]):
|
|
530
|
+
model = item.get("result_model") or {}
|
|
531
|
+
info = model.get("article_info") or model
|
|
532
|
+
title = info.get("title") or item.get("title_highlight") or ""
|
|
533
|
+
# 去 HTML 高亮
|
|
534
|
+
title = re.sub(r"<[^>]+>", "", str(title)).strip()
|
|
535
|
+
aid = info.get("article_id") or model.get("article_id") or ""
|
|
536
|
+
brief = info.get("brief_content") or info.get("content") or item.get("content_highlight") or ""
|
|
537
|
+
brief = re.sub(r"<[^>]+>", "", str(brief)).strip()
|
|
538
|
+
if not title:
|
|
539
|
+
continue
|
|
540
|
+
# 文章 / 专栏 / 其他
|
|
541
|
+
aurl = f"https://juejin.cn/post/{aid}" if aid else "https://juejin.cn/search?query=" + up.quote(query)
|
|
542
|
+
results.append({
|
|
543
|
+
"title": title[:200],
|
|
544
|
+
"url": aurl,
|
|
545
|
+
"snippet": brief[:300],
|
|
546
|
+
"source": "juejin",
|
|
547
|
+
"score": rank_score(0.75, _rk7),
|
|
548
|
+
})
|
|
549
|
+
if len(results) >= n:
|
|
550
|
+
break
|
|
551
|
+
return results
|
|
552
|
+
return _engine
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
# ── models.dev AI 模型信息引擎 ──────────────────────────────────────────────────
|
|
557
|
+
|
|
558
|
+
_models_dev_cache: tuple[float, Any] = (0.0, None) # (timestamp, data)
|
|
559
|
+
_MODELS_DEV_TTL = 86400 # 24h(evergreen,模型信息更新不频繁)
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def _build_models_dev_engine(spec: dict[str, Any]) -> Any:
|
|
563
|
+
"""models.dev AI 模型信息搜索(结构化数据,零认证,全量缓存)。
|
|
564
|
+
|
|
565
|
+
数据源:https://models.dev/api.json(176 provider × 5907+ 模型)
|
|
566
|
+
策略:全量拉取后本地过滤,缓存 24h。
|
|
567
|
+
过滤维度:模型名/family/description 关键词 + 能力维度(vision/tool/reasoning/context/免费/开源)。
|
|
568
|
+
"""
|
|
569
|
+
|
|
570
|
+
@safe_search
|
|
571
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
572
|
+
global _models_dev_cache
|
|
573
|
+
to = _timeout or spec.get("timeout", 15)
|
|
574
|
+
ql = query.lower().strip()
|
|
575
|
+
|
|
576
|
+
# 拉取全量数据(带缓存)
|
|
577
|
+
now = time.time()
|
|
578
|
+
ts, cached = _models_dev_cache
|
|
579
|
+
if cached is None or (now - ts) > _MODELS_DEV_TTL:
|
|
580
|
+
req = urllib.request.Request(
|
|
581
|
+
"https://models.dev/api.json",
|
|
582
|
+
headers={"User-Agent": "argo-search/1.0 (+models.dev)"},
|
|
583
|
+
)
|
|
584
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
585
|
+
cached = json.loads(resp.read().decode("utf-8"))
|
|
586
|
+
_models_dev_cache = (now, cached)
|
|
587
|
+
|
|
588
|
+
# 能力维度过滤
|
|
589
|
+
want_vision = any(k in ql for k in ("vision", "视觉", "图片", "多模态", "multimodal"))
|
|
590
|
+
want_tool = any(k in ql for k in ("tool", "工具调用", "function call"))
|
|
591
|
+
want_long = any(k in ql for k in ("1m", "100万", "长上下文", "long context", "百万"))
|
|
592
|
+
want_free = any(k in ql for k in ("免费", "free", "zero cost"))
|
|
593
|
+
want_open = any(k in ql for k in ("开源", "open weight", "open source"))
|
|
594
|
+
|
|
595
|
+
# 关键词提取(去掉能力词后剩下的当搜索词)
|
|
596
|
+
search_terms = ql
|
|
597
|
+
for drop in ("vision", "视觉", "图片", "多模态", "multimodal", "tool", "工具调用",
|
|
598
|
+
"function call", "1m", "100万", "长上下文", "long context", "百万",
|
|
599
|
+
"免费", "free", "zero cost", "开源", "open weight", "open source",
|
|
600
|
+
"模型", "model", "价格", "price", "pricing", "多少钱", "api"):
|
|
601
|
+
search_terms = search_terms.replace(drop, "")
|
|
602
|
+
search_terms = search_terms.strip()
|
|
603
|
+
|
|
604
|
+
results: list[dict[str, Any]] = []
|
|
605
|
+
for prov_id, prov in (cached or {}).items():
|
|
606
|
+
if not isinstance(prov, dict):
|
|
607
|
+
continue
|
|
608
|
+
prov_name = prov.get("name", prov_id)
|
|
609
|
+
prov_doc = prov.get("doc", "https://models.dev")
|
|
610
|
+
for mid, m in (prov.get("models") or {}).items():
|
|
611
|
+
if not isinstance(m, dict):
|
|
612
|
+
continue
|
|
613
|
+
# 能力过滤
|
|
614
|
+
if want_vision and not m.get("attachment"):
|
|
615
|
+
continue
|
|
616
|
+
if want_tool and not m.get("tool_call"):
|
|
617
|
+
continue
|
|
618
|
+
if want_open and not m.get("open_weights"):
|
|
619
|
+
continue
|
|
620
|
+
lim = m.get("limit") or {}
|
|
621
|
+
cost = m.get("cost") or {}
|
|
622
|
+
if want_long and (lim.get("context") or 0) < 500000:
|
|
623
|
+
continue
|
|
624
|
+
if want_free and (cost.get("input") or 0) > 0:
|
|
625
|
+
continue
|
|
626
|
+
|
|
627
|
+
# 关键词匹配
|
|
628
|
+
name = m.get("name", mid)
|
|
629
|
+
family = m.get("family", "")
|
|
630
|
+
desc = m.get("description", "")
|
|
631
|
+
haystack = f"{name} {family} {desc} {prov_name}".lower()
|
|
632
|
+
if search_terms:
|
|
633
|
+
terms = [t for t in search_terms.split() if len(t) >= 2]
|
|
634
|
+
if terms and not any(t in haystack for t in terms):
|
|
635
|
+
continue
|
|
636
|
+
elif not any(k in ql for k in ("模型", "model", "llm", "gpt", "claude",
|
|
637
|
+
"gemini", "glm", "llama", "mistral", "deepseek", "价格", "price")):
|
|
638
|
+
# 无明确搜索词也无能力词 → 诚实零结果(不保底热门模型,
|
|
639
|
+
# 避免产出「看起来像结果」的噪声稀释融合)
|
|
640
|
+
continue
|
|
641
|
+
|
|
642
|
+
# 构建 snippet
|
|
643
|
+
mods_in = m.get("modalities", {}).get("input", [])
|
|
644
|
+
snippet = (
|
|
645
|
+
f"ctx={lim.get('context', '?')} | out={lim.get('output', '?')} | "
|
|
646
|
+
f"in=${cost.get('input', '?')}/M out=${cost.get('output', '?')}/M | "
|
|
647
|
+
f"tools={m.get('tool_call')} | vision={m.get('attachment')} | "
|
|
648
|
+
f"reasoning={m.get('reasoning')} | input={','.join(mods_in)} | "
|
|
649
|
+
f"{desc[:80]}"
|
|
650
|
+
)
|
|
651
|
+
results.append({
|
|
652
|
+
"title": f"{name} ({prov_name})",
|
|
653
|
+
# 可定位到具体模型页(prov_doc 是 provider 首页,多个结果
|
|
654
|
+
# 共享同一 URL,导致下游去重/引用全部失效)
|
|
655
|
+
"url": f"https://models.dev/?model={mid}",
|
|
656
|
+
"snippet": snippet[:300],
|
|
657
|
+
"source": "models_dev",
|
|
658
|
+
# 0.8 固定基线分(结构化目录条目,非语义搜索,不宜 0.95 压过真实相关结果)
|
|
659
|
+
"score": rank_score(0.8, len(results)),
|
|
660
|
+
})
|
|
661
|
+
if len(results) >= n:
|
|
662
|
+
break
|
|
663
|
+
if len(results) >= n:
|
|
664
|
+
break
|
|
665
|
+
|
|
666
|
+
return results
|
|
667
|
+
return _engine
|
|
668
|
+
|
|
669
|
+
# ── Finviz 美股筛选引擎(HTML 抓取)─────────────────────────────────────────────
|
|
670
|
+
|
|
671
|
+
def _build_finviz_engine(spec: dict[str, Any]) -> Any:
|
|
672
|
+
"""Finviz 个股快照(finviz.com/quote.ashx?t=TICKER)
|
|
673
|
+
|
|
674
|
+
从 snapshot 表格提取:公司名、行业、市值、PE、价格、涨跌幅。
|
|
675
|
+
query 视为 ticker(大写);非纯字母时取首个字母 token 作 ticker。
|
|
676
|
+
"""
|
|
677
|
+
timeout = spec.get("timeout", 10)
|
|
678
|
+
|
|
679
|
+
@safe_search
|
|
680
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
681
|
+
to = _timeout or timeout
|
|
682
|
+
m = re.search(r"[A-Za-z]{1,6}", query or "")
|
|
683
|
+
if not m:
|
|
684
|
+
return []
|
|
685
|
+
ticker = m.group(0).upper()
|
|
686
|
+
url = f"https://finviz.com/quote.ashx?t={ticker}"
|
|
687
|
+
headers = {
|
|
688
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
|
689
|
+
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
690
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
691
|
+
}
|
|
692
|
+
req = urllib.request.Request(url, headers=headers)
|
|
693
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
694
|
+
html = resp.read().decode("utf-8", errors="replace")
|
|
695
|
+
if _detect_anti_bot(html):
|
|
696
|
+
return []
|
|
697
|
+
|
|
698
|
+
# snapshot 表格是 key-value 交替单元格
|
|
699
|
+
cells = re.findall(r'<td[^>]*class="[^"]*snapshot-td2[^"]*"[^>]*>(.*?)</td>', html, re.DOTALL)
|
|
700
|
+
cleaned = [re.sub(r"<[^>]+>", "", c).strip() for c in cells]
|
|
701
|
+
kv: dict[str, str] = {}
|
|
702
|
+
for i in range(0, len(cleaned) - 1, 2):
|
|
703
|
+
kv[cleaned[i]] = cleaned[i + 1]
|
|
704
|
+
|
|
705
|
+
# 公司名 / 行业
|
|
706
|
+
name_m = re.search(r'<title>([^<]+)</title>', html)
|
|
707
|
+
company = name_m.group(1).split("Stock")[0].strip() if name_m else ticker
|
|
708
|
+
|
|
709
|
+
if not kv and company == ticker:
|
|
710
|
+
return []
|
|
711
|
+
|
|
712
|
+
price = kv.get("Price", "")
|
|
713
|
+
change = kv.get("Change", "")
|
|
714
|
+
market_cap = kv.get("Market Cap", "")
|
|
715
|
+
pe = kv.get("P/E", "")
|
|
716
|
+
snippet = (f"行业信息见页面 | 市值 {market_cap} | PE {pe} | "
|
|
717
|
+
f"价格 {price} | 涨跌幅 {change}")
|
|
718
|
+
return [{
|
|
719
|
+
"title": f"{company} ({ticker}) 美股快照",
|
|
720
|
+
"url": url,
|
|
721
|
+
"snippet": snippet[:300],
|
|
722
|
+
"source": "finviz",
|
|
723
|
+
"score": 0.9,
|
|
724
|
+
"facts": {"ticker": ticker, "price": price, "change": change,
|
|
725
|
+
"market_cap": market_cap, "pe": pe},
|
|
726
|
+
}]
|
|
727
|
+
return _engine
|
|
728
|
+
|
|
729
|
+
# ── Seeking Alpha 美股分析引擎(HTML 抓取)───────────────────────────────────────
|
|
730
|
+
|
|
731
|
+
def _build_seeking_alpha_engine(spec: dict[str, Any]) -> Any:
|
|
732
|
+
"""Seeking Alpha 个股概览(seekingalpha.com/symbol/TICKER)
|
|
733
|
+
|
|
734
|
+
提取:评级 / 目标价 / 分析摘要(尽力而为,站点反爬较强,失败返回 [])。
|
|
735
|
+
"""
|
|
736
|
+
timeout = spec.get("timeout", 12)
|
|
737
|
+
|
|
738
|
+
@safe_search
|
|
739
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
740
|
+
to = _timeout or timeout
|
|
741
|
+
m = re.search(r"[A-Za-z]{1,6}", query or "")
|
|
742
|
+
if not m:
|
|
743
|
+
return []
|
|
744
|
+
ticker = m.group(0).upper()
|
|
745
|
+
url = f"https://seekingalpha.com/symbol/{ticker}"
|
|
746
|
+
headers = {
|
|
747
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
|
748
|
+
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
749
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
750
|
+
}
|
|
751
|
+
req = urllib.request.Request(url, headers=headers)
|
|
752
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
753
|
+
html = resp.read().decode("utf-8", errors="replace")
|
|
754
|
+
if _detect_anti_bot(html):
|
|
755
|
+
return []
|
|
756
|
+
|
|
757
|
+
desc_m = re.search(r'<meta[^>]+name="description"[^>]+content="([^"]+)"', html)
|
|
758
|
+
summary = desc_m.group(1).strip() if desc_m else ""
|
|
759
|
+
rating_m = re.search(r'(Strong Buy|Buy|Hold|Sell|Strong Sell)', html)
|
|
760
|
+
rating = rating_m.group(1) if rating_m else "N/A"
|
|
761
|
+
target_m = re.search(r'Price Target[^0-9]{0,20}(\d+\.\d+)', html)
|
|
762
|
+
target = target_m.group(1) if target_m else ""
|
|
763
|
+
|
|
764
|
+
if not summary and rating == "N/A":
|
|
765
|
+
return []
|
|
766
|
+
snippet = f"评级 {rating}" + (f" | 目标价 {target}" if target else "") + \
|
|
767
|
+
(f" | {summary}" if summary else "")
|
|
768
|
+
return [{
|
|
769
|
+
"title": f"{ticker} — Seeking Alpha 分析",
|
|
770
|
+
"url": url,
|
|
771
|
+
"snippet": snippet[:300],
|
|
772
|
+
"source": "seeking_alpha",
|
|
773
|
+
"score": 0.85,
|
|
774
|
+
"facts": {"ticker": ticker, "rating": rating, "target_price": target},
|
|
775
|
+
}]
|
|
776
|
+
return _engine
|
|
777
|
+
|
|
778
|
+
# ── 和风天气引擎(HTTP JSON API,需 QWEATHER_KEY)─────────────────────────────────
|
|
779
|
+
|
|
780
|
+
def _build_qweather_engine(spec: dict[str, Any]) -> Any:
|
|
781
|
+
"""和风天气实时天气(devapi.qweather.com)
|
|
782
|
+
|
|
783
|
+
需环境变量 QWEATHER_KEY。先用 GeoAPI 将城市名解析为 LocationID,
|
|
784
|
+
再查实时天气。提取:温度、天气描述、湿度、风力。
|
|
785
|
+
无 key 时显式返回错误项(不静默)。
|
|
786
|
+
"""
|
|
787
|
+
timeout = spec.get("timeout", 8)
|
|
788
|
+
|
|
789
|
+
@safe_search
|
|
790
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
791
|
+
import urllib.parse as up
|
|
792
|
+
to = _timeout or timeout
|
|
793
|
+
key = get_env(["ARGO_QWEATHER_KEY", "QWEATHER_KEY"])
|
|
794
|
+
if not key:
|
|
795
|
+
logger.warning("QWEATHER_KEY 未设置,跳过和风天气")
|
|
796
|
+
return [{"error": "QWEATHER_KEY 未设置", "source": "qweather"}]
|
|
797
|
+
|
|
798
|
+
# 从 query 抽取城市名(去掉天气/预报等词)
|
|
799
|
+
city = re.sub(r"(天气|预报|气温|温度|今天|明天|实时|怎么样|多少度)", "", query or "").strip()
|
|
800
|
+
city = city or query.strip()
|
|
801
|
+
|
|
802
|
+
# 1) GeoAPI: 城市 → LocationID
|
|
803
|
+
geo_url = f"https://geoapi.qweather.com/v2/city/lookup?location={up.quote(city)}&key={key}"
|
|
804
|
+
req = urllib.request.Request(geo_url, headers={"User-Agent": "argo-search/1.0"})
|
|
805
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
806
|
+
geo = json.loads(resp.read().decode("utf-8"))
|
|
807
|
+
locations = geo.get("location") or []
|
|
808
|
+
if not locations:
|
|
809
|
+
return []
|
|
810
|
+
loc = locations[0]
|
|
811
|
+
loc_id = loc.get("id", "")
|
|
812
|
+
loc_name = loc.get("name", city)
|
|
813
|
+
|
|
814
|
+
# 2) 实时天气
|
|
815
|
+
now_url = f"https://devapi.qweather.com/v7/weather/now?location={loc_id}&key={key}"
|
|
816
|
+
req2 = urllib.request.Request(now_url, headers={"User-Agent": "argo-search/1.0"})
|
|
817
|
+
with http_open(req2, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
818
|
+
data = json.loads(resp.read().decode("utf-8"))
|
|
819
|
+
now = data.get("now") or {}
|
|
820
|
+
if not now:
|
|
821
|
+
return []
|
|
822
|
+
temp = now.get("temp", "")
|
|
823
|
+
text = now.get("text", "")
|
|
824
|
+
humidity = now.get("humidity", "")
|
|
825
|
+
wind = f"{now.get('windDir', '')}{now.get('windScale', '')}级"
|
|
826
|
+
snippet = f"{loc_name}:{text} {temp}°C | 湿度 {humidity}% | 风力 {wind}"
|
|
827
|
+
return [{
|
|
828
|
+
"title": f"{loc_name}实时天气",
|
|
829
|
+
"url": now.get("fxLink", "https://www.qweather.com/"),
|
|
830
|
+
"snippet": snippet[:300],
|
|
831
|
+
"source": "qweather",
|
|
832
|
+
"score": 1.0,
|
|
833
|
+
"facts": {"temp": temp, "text": text, "humidity": humidity, "wind": wind},
|
|
834
|
+
}]
|
|
835
|
+
return _engine
|
|
836
|
+
|
|
837
|
+
# ── 中国裁判文书网引擎(HTML/JSON 抓取)──────────────────────────────────────────
|
|
838
|
+
|
|
839
|
+
def _build_wenshu_engine(spec: dict[str, Any]) -> Any:
|
|
840
|
+
"""中国裁判文书网(wenshu.court.gov.cn)法律文书检索。
|
|
841
|
+
|
|
842
|
+
该站有较强反爬(参数加密),此处走公开列表接口,尽力而为;
|
|
843
|
+
失败或被拦截时显式返回 [](不伪造结果)。
|
|
844
|
+
提取:案号、案由、裁判日期、法院。
|
|
845
|
+
"""
|
|
846
|
+
timeout = spec.get("timeout", 12)
|
|
847
|
+
|
|
848
|
+
@safe_search
|
|
849
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
850
|
+
import urllib.parse as up
|
|
851
|
+
to = _timeout or timeout
|
|
852
|
+
# 公开检索页(列表 HTML)——政府站结构可能变化,解析失败即空
|
|
853
|
+
url = f"https://wenshu.court.gov.cn/website/wenshu/181217BMTKHNT2W0/index.html?q={up.quote(query)}"
|
|
854
|
+
headers = {
|
|
855
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
856
|
+
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
857
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
858
|
+
"Referer": "https://wenshu.court.gov.cn/",
|
|
859
|
+
}
|
|
860
|
+
try:
|
|
861
|
+
req = urllib.request.Request(url, headers=headers)
|
|
862
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
863
|
+
html = resp.read().decode("utf-8", errors="replace")
|
|
864
|
+
except Exception as e:
|
|
865
|
+
logger.warning(f"裁判文书网访问失败: {e}")
|
|
866
|
+
return []
|
|
867
|
+
if _detect_anti_bot(html):
|
|
868
|
+
return []
|
|
869
|
+
|
|
870
|
+
results: list[dict[str, Any]] = []
|
|
871
|
+
# 尝试解析嵌入的 JSON 数据块(站点结构变化时降级为空)
|
|
872
|
+
blocks = re.findall(r'\{[^{}]*"caseName"[^{}]*\}', html)
|
|
873
|
+
for b in blocks[:n]:
|
|
874
|
+
try:
|
|
875
|
+
item = json.loads(b)
|
|
876
|
+
except Exception:
|
|
877
|
+
continue
|
|
878
|
+
results.append({
|
|
879
|
+
"title": item.get("caseName", "")[:100],
|
|
880
|
+
"url": "https://wenshu.court.gov.cn/",
|
|
881
|
+
"snippet": (f"案号 {item.get('caseNo', '')} | 案由 {item.get('caseType', '')} | "
|
|
882
|
+
f"法院 {item.get('court', '')} | 裁判日期 {item.get('judgeDate', '')}")[:300],
|
|
883
|
+
"source": "wenshu",
|
|
884
|
+
"facts": {"case_no": item.get("caseNo", ""), "court": item.get("court", ""),
|
|
885
|
+
"judge_date": item.get("judgeDate", "")},
|
|
886
|
+
})
|
|
887
|
+
return results
|
|
888
|
+
return _engine
|
|
889
|
+
|
|
890
|
+
# ── 金十数据引擎(HTTP JSON API 财经快讯)────────────────────────────────────────
|
|
891
|
+
|
|
892
|
+
def _build_jin10_engine(spec: dict[str, Any]) -> Any:
|
|
893
|
+
"""金十数据财经快讯(flash-api.jin10.com/get_flash_list)
|
|
894
|
+
|
|
895
|
+
提取:时间、标题、内容摘要;按关键词过滤(query 命中)。
|
|
896
|
+
"""
|
|
897
|
+
timeout = spec.get("timeout", 8)
|
|
898
|
+
|
|
899
|
+
@safe_search
|
|
900
|
+
def _engine(query: str, n: int = 10, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
901
|
+
to = _timeout or timeout
|
|
902
|
+
url = f"https://flash-api.jin10.com/get_flash_list?channel=-8200&vip=1&t={int(time.time() * 1000)}"
|
|
903
|
+
headers = {
|
|
904
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
|
905
|
+
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
906
|
+
"x-app-id": "bVBF4FyRTn5NJF5n",
|
|
907
|
+
"x-version": "1.0.0",
|
|
908
|
+
"Referer": "https://www.jin10.com/",
|
|
909
|
+
}
|
|
910
|
+
req = urllib.request.Request(url, headers=headers)
|
|
911
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
912
|
+
data = json.loads(resp.read().decode("utf-8"))
|
|
913
|
+
items = data.get("data") or []
|
|
914
|
+
keywords = [k for k in (query or "").strip().split() if k]
|
|
915
|
+
# 去掉路由触发词,避免「金十」「快讯」本身当过滤条件
|
|
916
|
+
stop = {"金十", "jin10", "财经快讯", "市场快讯", "7x24", "快讯", "电报"}
|
|
917
|
+
keywords = [k for k in keywords if k.lower() not in stop and k not in stop]
|
|
918
|
+
results: list[dict[str, Any]] = []
|
|
919
|
+
for it in items:
|
|
920
|
+
d = it.get("data") or {}
|
|
921
|
+
content = d.get("content", "") or d.get("pic", "") or ""
|
|
922
|
+
title = d.get("title", "") or content[:40]
|
|
923
|
+
text = f"{title} {content}"
|
|
924
|
+
if keywords and not any(k.lower() in text.lower() for k in keywords):
|
|
925
|
+
continue
|
|
926
|
+
results.append({
|
|
927
|
+
"title": title[:80] or "金十快讯",
|
|
928
|
+
"url": "https://www.jin10.com/",
|
|
929
|
+
"snippet": f"{it.get('time', '')} | {content[:150]}",
|
|
930
|
+
"source": "jin10",
|
|
931
|
+
})
|
|
932
|
+
if len(results) >= n:
|
|
933
|
+
break
|
|
934
|
+
# 关键词未命中当前快讯池时,回退最新 N 条(金十是滚动电报,非全库检索)
|
|
935
|
+
if not results and items:
|
|
936
|
+
for it in items[:n]:
|
|
937
|
+
d = it.get("data") or {}
|
|
938
|
+
content = d.get("content", "") or d.get("pic", "") or ""
|
|
939
|
+
title = d.get("title", "") or content[:40]
|
|
940
|
+
results.append({
|
|
941
|
+
"title": title[:80] or "金十快讯",
|
|
942
|
+
"url": "https://www.jin10.com/",
|
|
943
|
+
"snippet": f"{it.get('time', '')} | {content[:150]}",
|
|
944
|
+
"source": "jin10",
|
|
945
|
+
})
|
|
946
|
+
return results
|
|
947
|
+
return _engine
|
|
948
|
+
|
|
949
|
+
# ── Octen AI 搜索引擎 ─────────────────────────────────────────────────────────
|
|
950
|
+
|
|
951
|
+
def _build_octen_engine(spec: dict[str, Any]) -> Any:
|
|
952
|
+
"""Octen AI 搜索(高速语义搜索 + broad-search 多查询分解)
|
|
953
|
+
|
|
954
|
+
支持两种模式:
|
|
955
|
+
- 标准搜索: /search(单次查询)
|
|
956
|
+
- 广域搜索: /broad-search(自动分解为子查询并行执行)
|
|
957
|
+
"""
|
|
958
|
+
timeout = spec.get("timeout", 10)
|
|
959
|
+
|
|
960
|
+
@safe_search
|
|
961
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, depth: str = "fast", **kwargs) -> list[dict[str, Any]]:
|
|
962
|
+
to = _timeout or timeout
|
|
963
|
+
api_key = get_env(["ARGO_OCTEN_API_KEY", "OCTEN_API_KEY"])
|
|
964
|
+
if not api_key:
|
|
965
|
+
logger.warning("OCTEN_API_KEY 未设置")
|
|
966
|
+
return []
|
|
967
|
+
|
|
968
|
+
# broad-search 模式(多子查询分解),支持 depth 参数
|
|
969
|
+
use_broad = depth in ("deep", "balanced")
|
|
970
|
+
if use_broad:
|
|
971
|
+
url = "https://api.octen.ai/broad-search"
|
|
972
|
+
max_q = 3 if depth == "balanced" else 5
|
|
973
|
+
body = json.dumps({
|
|
974
|
+
"query": query,
|
|
975
|
+
"max_queries": max_q,
|
|
976
|
+
"search_options": {
|
|
977
|
+
"count": min(n, 5),
|
|
978
|
+
"topic": "general",
|
|
979
|
+
"safesearch": "off",
|
|
980
|
+
"highlight": {"enable": True, "max_tokens": 512},
|
|
981
|
+
"full_content": {"enable": False},
|
|
982
|
+
"include_images": False,
|
|
983
|
+
},
|
|
984
|
+
}).encode("utf-8")
|
|
985
|
+
else:
|
|
986
|
+
url = "https://api.octen.ai/search"
|
|
987
|
+
body = json.dumps({
|
|
988
|
+
"query": query,
|
|
989
|
+
"count": min(n, 10),
|
|
990
|
+
"topic": "general",
|
|
991
|
+
"safesearch": "off",
|
|
992
|
+
"highlight": {"enable": True, "max_tokens": 512},
|
|
993
|
+
"full_content": {"enable": False},
|
|
994
|
+
"include_images": False,
|
|
995
|
+
}).encode("utf-8")
|
|
996
|
+
|
|
997
|
+
headers = {"X-Api-Key": api_key, "Content-Type": "application/json"}
|
|
998
|
+
req = urllib.request.Request(url, data=body, headers=headers)
|
|
999
|
+
|
|
1000
|
+
try:
|
|
1001
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
1002
|
+
data = json.loads(resp.read().decode("utf-8"))
|
|
1003
|
+
items = data.get("data", {})
|
|
1004
|
+
results = []
|
|
1005
|
+
|
|
1006
|
+
if use_broad:
|
|
1007
|
+
# broad-search: 遍历所有子查询结果
|
|
1008
|
+
seen = set()
|
|
1009
|
+
for group in items.get("search_results", []):
|
|
1010
|
+
for r in group.get("results", []):
|
|
1011
|
+
url_key = r.get("url", "")
|
|
1012
|
+
if url_key and url_key not in seen:
|
|
1013
|
+
seen.add(url_key)
|
|
1014
|
+
results.append({
|
|
1015
|
+
"title": r.get("title", ""),
|
|
1016
|
+
"url": url_key,
|
|
1017
|
+
"snippet": r.get("highlight", "")[:300],
|
|
1018
|
+
"source": "octen",
|
|
1019
|
+
"score": rank_score(0.8, len(results)),
|
|
1020
|
+
})
|
|
1021
|
+
if len(results) >= n:
|
|
1022
|
+
return results
|
|
1023
|
+
else:
|
|
1024
|
+
for r in items.get("results", [])[:n]:
|
|
1025
|
+
results.append({
|
|
1026
|
+
"title": r.get("title", ""),
|
|
1027
|
+
"url": r.get("url", ""),
|
|
1028
|
+
"snippet": r.get("highlight", "")[:300],
|
|
1029
|
+
"source": "octen",
|
|
1030
|
+
"score": rank_score(next((v for v in (r.get("score"),) if v not in (None, "")), 0.5), len(results)), # 哨兵判空:显式 0 分不得膨胀成 0.5
|
|
1031
|
+
})
|
|
1032
|
+
|
|
1033
|
+
# 如果结果太少,补充 quick 数据源的查询(不走 broad 保底)
|
|
1034
|
+
if not results and not use_broad:
|
|
1035
|
+
logger.info("Octen 标准搜索无结果,兜底空列表返回")
|
|
1036
|
+
return results
|
|
1037
|
+
|
|
1038
|
+
except Exception as e:
|
|
1039
|
+
logger.warning(f"Octen 引擎失败: {e}")
|
|
1040
|
+
return []
|
|
1041
|
+
return _engine
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
# ── PubChem + ChEMBL 化学引擎(chem 域) ────────────────────────────────────
|
|
1045
|
+
|
|
1046
|
+
_PUBCHEM_API = "https://pubchem.ncbi.nlm.nih.gov/rest/pug"
|
|
1047
|
+
_CHEMBL_API = "https://www.ebi.ac.uk/chembl/api/data"
|
|
1048
|
+
_RE_FORMULA = re.compile(r"^[A-Z][a-z]?\d*([A-Z][a-z]?\d*)*$")
|
|
1049
|
+
# 检索噪声词:剥掉后用核心化合物名打 API(「阿司匹林分子式」→「阿司匹林」)
|
|
1050
|
+
_CHEM_NOISE_RE = re.compile(
|
|
1051
|
+
r"(?i)\b(?:molecular\s+weight|chemical\s+formula|cas\s*number|iupac|smiles|"
|
|
1052
|
+
r"formula|weight|compound|molecule|mw)\b|"
|
|
1053
|
+
r"分子式|分子量|摩尔质量|化学式|结构式|化合物|药物|CAS号?"
|
|
1054
|
+
)
|
|
1055
|
+
# 常用中文药名 → PubChem 英文名(中文直查常 miss,ChEMBL 会乱返回无关分子)
|
|
1056
|
+
_CHEM_CN_ALIASES: dict[str, str] = {
|
|
1057
|
+
"阿司匹林": "aspirin",
|
|
1058
|
+
"乙酰水杨酸": "aspirin",
|
|
1059
|
+
"布洛芬": "ibuprofen",
|
|
1060
|
+
"对乙酰氨基酚": "acetaminophen",
|
|
1061
|
+
"扑热息痛": "acetaminophen",
|
|
1062
|
+
"咖啡因": "caffeine",
|
|
1063
|
+
"葡萄糖": "glucose",
|
|
1064
|
+
"乙醇": "ethanol",
|
|
1065
|
+
"甲醇": "methanol",
|
|
1066
|
+
"胆固醇": "cholesterol",
|
|
1067
|
+
"青霉素": "penicillin",
|
|
1068
|
+
"阿莫西林": "amoxicillin",
|
|
1069
|
+
"二甲双胍": "metformin",
|
|
1070
|
+
"奥美拉唑": "omeprazole",
|
|
1071
|
+
"硝苯地平": "nifedipine",
|
|
1072
|
+
"氨氯地平": "amlodipine",
|
|
1073
|
+
"阿托伐他汀": "atorvastatin",
|
|
1074
|
+
"氯吡格雷": "clopidogrel",
|
|
1075
|
+
"华法林": "warfarin",
|
|
1076
|
+
"胰岛素": "insulin",
|
|
1077
|
+
"吗啡": "morphine",
|
|
1078
|
+
"可待因": "codeine",
|
|
1079
|
+
"尼古丁": "nicotine",
|
|
1080
|
+
"维生素c": "ascorbic acid",
|
|
1081
|
+
"维生素C": "ascorbic acid",
|
|
1082
|
+
"抗坏血酸": "ascorbic acid",
|
|
1083
|
+
}
|
|
1084
|
+
_CHEM_TOKEN_NOISE = frozenset({
|
|
1085
|
+
"分子量", "分子式", "化学", "化学式", "结构式", "化合物", "药物",
|
|
1086
|
+
"weight", "formula", "molecular", "chemical", "compound", "molecule",
|
|
1087
|
+
"cid", "chembl", "smiles", "iupac", "cas", "mw", "acid", "hydrochloride",
|
|
1088
|
+
})
|
|
1089
|
+
|
|
1090
|
+
|
|
1091
|
+
def _strip_chem_noise(query: str) -> str:
|
|
1092
|
+
"""去掉「分子式/分子量」等修饰,保留化合物核心名。"""
|
|
1093
|
+
cleaned = _CHEM_NOISE_RE.sub(" ", query)
|
|
1094
|
+
cleaned = re.sub(r"\s+", " ", cleaned).strip(" ,;|/")
|
|
1095
|
+
return cleaned or query.strip()
|
|
1096
|
+
|
|
1097
|
+
|
|
1098
|
+
def _chem_lookup_names(query: str) -> list[str]:
|
|
1099
|
+
"""生成 PubChem/ChEMBL 查询候选:核心名 + 中文别名英文化。"""
|
|
1100
|
+
raw = query.strip()
|
|
1101
|
+
core = _strip_chem_noise(raw)
|
|
1102
|
+
names: list[str] = []
|
|
1103
|
+
for candidate in (core, raw):
|
|
1104
|
+
if candidate and candidate not in names:
|
|
1105
|
+
names.append(candidate)
|
|
1106
|
+
# 中文别名:核心或原文含中文药名时追加英文
|
|
1107
|
+
hay = f"{core} {raw}"
|
|
1108
|
+
for cn, en in _CHEM_CN_ALIASES.items():
|
|
1109
|
+
if cn in hay and en not in names:
|
|
1110
|
+
names.append(en)
|
|
1111
|
+
return names
|
|
1112
|
+
|
|
1113
|
+
|
|
1114
|
+
def _chem_tokens(text: str) -> set[str]:
|
|
1115
|
+
"""化合物相关 token:英文词 + 连续中文 + 数字。"""
|
|
1116
|
+
if not text:
|
|
1117
|
+
return set()
|
|
1118
|
+
low = text.lower()
|
|
1119
|
+
tokens = set(re.findall(r"[a-z][a-z0-9\-]{1,}", low))
|
|
1120
|
+
tokens.update(cjk_term_grams(text))
|
|
1121
|
+
tokens.update(re.findall(r"\d{2,}", text))
|
|
1122
|
+
return {t for t in tokens if t not in _CHEM_TOKEN_NOISE and len(t) >= 2}
|
|
1123
|
+
|
|
1124
|
+
|
|
1125
|
+
def _chem_result_overlaps(query: str, title: str, snippet: str = "",
|
|
1126
|
+
pref_name: str = "") -> bool:
|
|
1127
|
+
"""结果与查询是否相关:token 交集或核心名子串命中。
|
|
1128
|
+
|
|
1129
|
+
无重叠则视为假阳性(ChEMBL 对中文常返回无关分子)。
|
|
1130
|
+
"""
|
|
1131
|
+
q_tokens = _chem_tokens(query)
|
|
1132
|
+
core = _strip_chem_noise(query).lower()
|
|
1133
|
+
if core:
|
|
1134
|
+
q_tokens |= _chem_tokens(core)
|
|
1135
|
+
for cn, en in _CHEM_CN_ALIASES.items():
|
|
1136
|
+
if cn in query or cn in core:
|
|
1137
|
+
q_tokens.add(en.lower())
|
|
1138
|
+
for part in en.lower().split():
|
|
1139
|
+
if len(part) >= 2:
|
|
1140
|
+
q_tokens.add(part)
|
|
1141
|
+
doc = _chem_tokens(f"{title} {snippet} {pref_name}")
|
|
1142
|
+
if not q_tokens:
|
|
1143
|
+
return True # 无法判定时不误杀
|
|
1144
|
+
if q_tokens & doc:
|
|
1145
|
+
return True
|
|
1146
|
+
hay = f"{title} {snippet} {pref_name}".lower()
|
|
1147
|
+
if core and len(core) >= 2 and core in hay:
|
|
1148
|
+
return True
|
|
1149
|
+
# 英文别名整词
|
|
1150
|
+
for cn, en in _CHEM_CN_ALIASES.items():
|
|
1151
|
+
if cn in query and en.lower() in hay:
|
|
1152
|
+
return True
|
|
1153
|
+
return False
|
|
1154
|
+
|
|
1155
|
+
|
|
1156
|
+
def _build_pubchem_engine(spec: dict[str, Any]) -> Any:
|
|
1157
|
+
"""化学/药学化合物检索:PubChem PUG REST 主路径 + ChEMBL 保底。
|
|
1158
|
+
|
|
1159
|
+
支持化合物名、分子式、CAS 号、IUPAC 名查询,返回分子式/分子量/IUPAC/SMILES。
|
|
1160
|
+
质量闸门:ChEMBL 结果须与查询 token 重叠,否则丢弃当 no-results(宁空勿假)。
|
|
1161
|
+
"""
|
|
1162
|
+
timeout = spec.get("timeout", 10)
|
|
1163
|
+
_HEADERS = {"User-Agent": "argo-search/2.6 (unified-search@local)", "Accept": "application/json"}
|
|
1164
|
+
|
|
1165
|
+
def _jget(url: str, to: float) -> dict:
|
|
1166
|
+
req = urllib.request.Request(url, headers=_HEADERS)
|
|
1167
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
1168
|
+
return json.loads(resp.read().decode("utf-8", "replace"))
|
|
1169
|
+
|
|
1170
|
+
@safe_search
|
|
1171
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
1172
|
+
q = query.strip()
|
|
1173
|
+
if not q:
|
|
1174
|
+
return []
|
|
1175
|
+
to = _timeout or timeout
|
|
1176
|
+
results: list[dict[str, Any]] = []
|
|
1177
|
+
lookup_names = _chem_lookup_names(q)
|
|
1178
|
+
display_name = _strip_chem_noise(q) or q
|
|
1179
|
+
|
|
1180
|
+
# 1. PubChem 主路径:多候选名 → CID → 属性(formula 端点异步,不在此轮询)
|
|
1181
|
+
cid = ""
|
|
1182
|
+
hit_name = display_name
|
|
1183
|
+
for name in lookup_names:
|
|
1184
|
+
try:
|
|
1185
|
+
d = _jget(
|
|
1186
|
+
f"{_PUBCHEM_API}/compound/name/{urllib.parse.quote(name)}/cids/JSON",
|
|
1187
|
+
to,
|
|
1188
|
+
)
|
|
1189
|
+
cids = (d.get("IdentifierList") or {}).get("CID") or []
|
|
1190
|
+
if cids:
|
|
1191
|
+
cid = str(cids[0])
|
|
1192
|
+
hit_name = name
|
|
1193
|
+
break
|
|
1194
|
+
except Exception:
|
|
1195
|
+
continue
|
|
1196
|
+
if cid:
|
|
1197
|
+
try:
|
|
1198
|
+
d = _jget(
|
|
1199
|
+
f"{_PUBCHEM_API}/compound/cid/{cid}/property/"
|
|
1200
|
+
f"MolecularFormula,MolecularWeight,IUPACName,CanonicalSMILES/JSON", to)
|
|
1201
|
+
props = ((d.get("PropertyTable") or {}).get("Properties") or [{}])[0]
|
|
1202
|
+
formula = props.get("MolecularFormula", "")
|
|
1203
|
+
mw = props.get("MolecularWeight", "")
|
|
1204
|
+
iupac = props.get("IUPACName", "") or ""
|
|
1205
|
+
smiles = props.get("CanonicalSMILES", "") or ""
|
|
1206
|
+
title = f"{display_name} (PubChem CID {cid})"
|
|
1207
|
+
if hit_name.lower() != display_name.lower():
|
|
1208
|
+
title = f"{display_name} / {hit_name} (PubChem CID {cid})"
|
|
1209
|
+
if formula:
|
|
1210
|
+
title += f" 分子式 {formula}"
|
|
1211
|
+
if mw:
|
|
1212
|
+
title += f" 分子量 {mw}"
|
|
1213
|
+
snippet = f"CID {cid}"
|
|
1214
|
+
if iupac:
|
|
1215
|
+
snippet += f" | IUPAC: {iupac}"
|
|
1216
|
+
if smiles:
|
|
1217
|
+
snippet += f" | SMILES: {smiles}"
|
|
1218
|
+
results.append({
|
|
1219
|
+
"title": title,
|
|
1220
|
+
"url": f"https://pubchem.ncbi.nlm.nih.gov/compound/{cid}",
|
|
1221
|
+
"snippet": snippet[:400],
|
|
1222
|
+
"source": "pubchem",
|
|
1223
|
+
"score": 0.95,
|
|
1224
|
+
})
|
|
1225
|
+
except Exception as e:
|
|
1226
|
+
logger.warning(f"PubChem 属性失败: {e}")
|
|
1227
|
+
|
|
1228
|
+
# 2. ChEMBL 保底:仅保留与查询重叠的结果(无重叠 = 空,禁止交差评)
|
|
1229
|
+
if not results:
|
|
1230
|
+
seen_ids: set[str] = set()
|
|
1231
|
+
for name in lookup_names:
|
|
1232
|
+
try:
|
|
1233
|
+
d = _jget(
|
|
1234
|
+
f"{_CHEMBL_API}/molecule/search.json?"
|
|
1235
|
+
f"q={urllib.parse.quote(name)}&limit={min(n, 5)}",
|
|
1236
|
+
to,
|
|
1237
|
+
)
|
|
1238
|
+
except Exception as e:
|
|
1239
|
+
logger.warning(f"ChEMBL 失败 ({name}): {e}")
|
|
1240
|
+
continue
|
|
1241
|
+
for m in (d.get("molecules") or [])[:n]:
|
|
1242
|
+
props = m.get("molecule_properties") or {}
|
|
1243
|
+
chembl_id = m.get("molecule_chembl_id", "")
|
|
1244
|
+
if chembl_id in seen_ids:
|
|
1245
|
+
continue
|
|
1246
|
+
pref = (m.get("pref_name") or "") or ""
|
|
1247
|
+
title_base = pref or chembl_id or name
|
|
1248
|
+
title = title_base
|
|
1249
|
+
if props.get("mw_freebase"):
|
|
1250
|
+
title += f" 分子量 {props['mw_freebase']}"
|
|
1251
|
+
parts = [f"ChEMBL {chembl_id}"]
|
|
1252
|
+
if props.get("full_molformula"):
|
|
1253
|
+
parts.append(f"分子式 {props['full_molformula']}")
|
|
1254
|
+
if props.get("canonical_smiles"):
|
|
1255
|
+
parts.append(f"SMILES {props['canonical_smiles'][:100]}")
|
|
1256
|
+
snippet = " | ".join(parts)[:400]
|
|
1257
|
+
# 质量闸门:与原查询或当前 lookup 名无重叠则丢弃
|
|
1258
|
+
if not (
|
|
1259
|
+
_chem_result_overlaps(q, title, snippet, pref)
|
|
1260
|
+
or _chem_result_overlaps(name, title, snippet, pref)
|
|
1261
|
+
):
|
|
1262
|
+
continue
|
|
1263
|
+
seen_ids.add(chembl_id)
|
|
1264
|
+
results.append({
|
|
1265
|
+
"title": title,
|
|
1266
|
+
"url": (f"https://www.ebi.ac.uk/chembl/explore/compound/{chembl_id}"
|
|
1267
|
+
if chembl_id else "https://www.ebi.ac.uk/chembl/"),
|
|
1268
|
+
"snippet": snippet,
|
|
1269
|
+
"source": "chembl",
|
|
1270
|
+
"score": rank_score(0.9, len(results)),
|
|
1271
|
+
})
|
|
1272
|
+
if results:
|
|
1273
|
+
break
|
|
1274
|
+
if not results:
|
|
1275
|
+
logger.info("ChEMBL 结果与查询无重叠,返回空(宁空勿假)")
|
|
1276
|
+
return results[: max(n, 3)]
|
|
1277
|
+
|
|
1278
|
+
return _engine
|
|
1279
|
+
|
|
1280
|
+
|
|
1281
|
+
# ── GBIF 生物多样性引擎(species 域) ────────────────────────────────────────
|
|
1282
|
+
|
|
1283
|
+
_GBIF_API = "https://api.gbif.org/v1"
|
|
1284
|
+
_GBIF_RANK_ORDER = {"SPECIES": 3, "SUBSPECIES": 2, "GENUS": 1}
|
|
1285
|
+
|
|
1286
|
+
|
|
1287
|
+
def _build_gbif_engine(spec: dict[str, Any]) -> Any:
|
|
1288
|
+
"""GBIF 物种检索(api.gbif.org/v1/species/search,免认证)。
|
|
1289
|
+
|
|
1290
|
+
学名/俗名搜索,优先学名包含查询词的 ACCEPTED 物种条目。
|
|
1291
|
+
"""
|
|
1292
|
+
timeout = spec.get("timeout", 12)
|
|
1293
|
+
_HEADERS = {"User-Agent": "argo-search/2.6 (unified-search@local)"}
|
|
1294
|
+
|
|
1295
|
+
def _jget(url: str, to: float) -> dict:
|
|
1296
|
+
req = urllib.request.Request(url, headers=_HEADERS)
|
|
1297
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
1298
|
+
return json.loads(resp.read().decode("utf-8", "replace"))
|
|
1299
|
+
|
|
1300
|
+
@safe_search
|
|
1301
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
1302
|
+
q = query.strip()
|
|
1303
|
+
if not q:
|
|
1304
|
+
return []
|
|
1305
|
+
# 剥离中文查询噪声词(学名/物种/俗名等);GBIF 只对拉丁学名/英文俗名可靠,
|
|
1306
|
+
# 纯中文俗名(如「大熊猫」)搜出来的是无关属种,直接放弃
|
|
1307
|
+
q = re.sub(r"(学名|物种|俗名|拉丁名|生物|分类|是什么|有哪些|查询|搜索)", "", q).strip()
|
|
1308
|
+
if not re.search(r"[A-Za-z]{2,}", q):
|
|
1309
|
+
return []
|
|
1310
|
+
to = _timeout or timeout
|
|
1311
|
+
ql = q.lower()
|
|
1312
|
+
url = f"{_GBIF_API}/species/search?q={urllib.parse.quote(q)}&limit={min(n * 3, 20)}"
|
|
1313
|
+
try:
|
|
1314
|
+
d = _jget(url, to)
|
|
1315
|
+
except Exception as e:
|
|
1316
|
+
logger.warning(f"GBIF 失败: {e}")
|
|
1317
|
+
return []
|
|
1318
|
+
|
|
1319
|
+
def _key(r: dict) -> tuple:
|
|
1320
|
+
sn = (r.get("scientificName") or "").lower()
|
|
1321
|
+
name_hit = ql in sn # 学名包含查询词优先(俗名匹配的噪声大)
|
|
1322
|
+
status_ok = (r.get("taxonomicStatus") or "") == "ACCEPTED"
|
|
1323
|
+
rank = _GBIF_RANK_ORDER.get(r.get("rank"), 0)
|
|
1324
|
+
return (name_hit, status_ok, rank)
|
|
1325
|
+
|
|
1326
|
+
items = sorted((r for r in (d.get("results") or []) if r.get("scientificName")), key=_key, reverse=True)
|
|
1327
|
+
results = []
|
|
1328
|
+
for _rk9, r in enumerate(items[:n]):
|
|
1329
|
+
sci = r.get("scientificName", "")
|
|
1330
|
+
rank = r.get("rank", "")
|
|
1331
|
+
kingdom = r.get("kingdom") or r.get("kingdomKey") or ""
|
|
1332
|
+
status = r.get("taxonomicStatus") or ""
|
|
1333
|
+
key = r.get("nubKey") or r.get("key")
|
|
1334
|
+
title = f"{sci} ({rank})"
|
|
1335
|
+
if status == "ACCEPTED":
|
|
1336
|
+
title = f"{sci}(有效名)"
|
|
1337
|
+
snip = f"界 {kingdom} | 分类 {rank}"
|
|
1338
|
+
if status:
|
|
1339
|
+
snip += f" | 状态 {status}"
|
|
1340
|
+
if r.get("vernacularName"):
|
|
1341
|
+
snip += f" | 俗名 {r['vernacularName']}"
|
|
1342
|
+
results.append({
|
|
1343
|
+
"title": title,
|
|
1344
|
+
"url": f"https://www.gbif.org/species/{key}" if key else "https://www.gbif.org/",
|
|
1345
|
+
"snippet": snip,
|
|
1346
|
+
"source": "gbif",
|
|
1347
|
+
"score": rank_score(0.92, _rk9),
|
|
1348
|
+
})
|
|
1349
|
+
return results[: max(n, 3)]
|
|
1350
|
+
|
|
1351
|
+
return _engine
|
|
1352
|
+
|
|
1353
|
+
|
|
1354
|
+
# ── RFC Editor / IETF 标准引擎(tech 域) ────────────────────────────────────
|
|
1355
|
+
|
|
1356
|
+
_DATATRACKER_API = "https://datatracker.ietf.org/api/v1"
|
|
1357
|
+
|
|
1358
|
+
|
|
1359
|
+
def _build_rfc_editor_engine(spec: dict[str, Any]) -> Any:
|
|
1360
|
+
"""RFC / IETF 标准文档检索(datatracker.ietf.org API,免认证)。
|
|
1361
|
+
|
|
1362
|
+
「RFC 9000」直接定位单篇;「QUIC RFC」按标题搜索 RFC 类型文档。
|
|
1363
|
+
"""
|
|
1364
|
+
timeout = spec.get("timeout", 15)
|
|
1365
|
+
_HEADERS = {"User-Agent": "argo-search/2.6 (unified-search@local)", "Accept": "application/json"}
|
|
1366
|
+
|
|
1367
|
+
def _jget(url: str, to: float) -> dict:
|
|
1368
|
+
req = urllib.request.Request(url, headers=_HEADERS)
|
|
1369
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
1370
|
+
return json.loads(resp.read().decode("utf-8", "replace"))
|
|
1371
|
+
|
|
1372
|
+
def _mk(r: dict) -> dict[str, Any]:
|
|
1373
|
+
name = r.get("name", "")
|
|
1374
|
+
num = re.search(r"(\d+)", name)
|
|
1375
|
+
title = r.get("title") or ""
|
|
1376
|
+
url = (f"https://www.rfc-editor.org/rfc/rfc{num.group(1)}.txt"
|
|
1377
|
+
if num else "https://www.rfc-editor.org/")
|
|
1378
|
+
snip = "Internet 标准文档(IETF)"
|
|
1379
|
+
if r.get("abstract"):
|
|
1380
|
+
snip = r["abstract"][:300]
|
|
1381
|
+
return {
|
|
1382
|
+
"title": f"{name.upper()}: {title}" if num else title,
|
|
1383
|
+
"url": url,
|
|
1384
|
+
"snippet": snip,
|
|
1385
|
+
"source": "rfc_editor",
|
|
1386
|
+
"score": 0.93,
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
@safe_search
|
|
1390
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
1391
|
+
q = query.strip()
|
|
1392
|
+
if not q:
|
|
1393
|
+
return []
|
|
1394
|
+
to = _timeout or timeout
|
|
1395
|
+
# 1. 直接 RFC 编号
|
|
1396
|
+
m = re.search(r"(?i)(?:rfc\s*|RFC)(\d{2,5})", q)
|
|
1397
|
+
if m:
|
|
1398
|
+
num = m.group(1)
|
|
1399
|
+
try:
|
|
1400
|
+
d = _jget(f"{_DATATRACKER_API}/doc/document/rfc{num}/", to)
|
|
1401
|
+
if d.get("name"):
|
|
1402
|
+
return [_mk(d)][: max(n, 3)]
|
|
1403
|
+
except Exception:
|
|
1404
|
+
pass
|
|
1405
|
+
# 2. 标题搜索:剥掉 RFC/标准等噪声词后按标题检索
|
|
1406
|
+
search_q = re.sub(r"(?i)\b(rfc|ietf|internet standard|standard)\b|互联网标准|标准文档|文档", " ", q)
|
|
1407
|
+
search_q = re.sub(r"\s+", " ", search_q).strip()
|
|
1408
|
+
if not search_q:
|
|
1409
|
+
return []
|
|
1410
|
+
url = (f"{_DATATRACKER_API}/doc/document/?title__icontains="
|
|
1411
|
+
f"{urllib.parse.quote(search_q)}&type__slug=rfc&limit={min(n, 10)}")
|
|
1412
|
+
try:
|
|
1413
|
+
d = _jget(url, to)
|
|
1414
|
+
except Exception as e:
|
|
1415
|
+
logger.warning(f"RFC 搜索失败: {e}")
|
|
1416
|
+
return []
|
|
1417
|
+
return [_mk(o) for o in (d.get("objects") or [])[:n]][: max(n, 3)]
|
|
1418
|
+
|
|
1419
|
+
return _engine
|
|
1420
|
+
|
|
1421
|
+
|
|
1422
|
+
|
|
1423
|
+
# ── UniProt 蛋白质/基因组引擎 ────────────────────────────────────────────────
|
|
1424
|
+
|
|
1425
|
+
_UNIPROT_API = "https://rest.uniprot.org/uniprotkb/search"
|
|
1426
|
+
|
|
1427
|
+
# 高频蛋白中文名 → 英文检索词(UniProt 不索引中文)
|
|
1428
|
+
_UNIPROT_CN = {
|
|
1429
|
+
"胰岛素": "insulin", "血红蛋白": "hemoglobin", "肌红蛋白": "myoglobin",
|
|
1430
|
+
"白蛋白": "albumin", "胶原蛋白": "collagen", "角蛋白": "keratin",
|
|
1431
|
+
"胰蛋白酶": "trypsin", "溶菌酶": "lysozyme", "细胞色素c": "cytochrome c",
|
|
1432
|
+
"细胞色素": "cytochrome", "泛素": "ubiquitin", "免疫球蛋白": "immunoglobulin",
|
|
1433
|
+
"抗体": "antibody", "转铁蛋白": "transferrin", "纤维蛋白原": "fibrinogen",
|
|
1434
|
+
"表皮生长因子受体": "EGFR", "生长激素": "growth hormone",
|
|
1435
|
+
"血管内皮生长因子": "VEGF", "肿瘤坏死因子": "TNF", "白介素": "interleukin",
|
|
1436
|
+
"干扰素": "interferon", "淀粉酶": "amylase", "胃蛋白酶": "pepsin",
|
|
1437
|
+
"弹性蛋白": "elastin", "肌动蛋白": "actin", "肌球蛋白": "myosin",
|
|
1438
|
+
"微管蛋白": "tubulin", "组蛋白": "histone", "核糖体蛋白": "ribosomal protein",
|
|
1439
|
+
"谷胱甘肽过氧化物酶": "glutathione peroxidase", "超氧化物歧化酶": "superoxide dismutase",
|
|
1440
|
+
"载脂蛋白": "apolipoprotein", "脂蛋白": "lipoprotein", "受体蛋白": "receptor protein",
|
|
1441
|
+
"蛋白激酶": "protein kinase", "磷酸酶": "phosphatase", "转录因子": "transcription factor",
|
|
1442
|
+
"癌基因": "oncogene", "抑癌基因": "tumor suppressor", "p53蛋白": "p53",
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
|
|
1446
|
+
def _ascii_tokens(q: str) -> str:
|
|
1447
|
+
"""提取查询中的 ASCII 字母数字 token(UniProt/PDB 只索引英文)。"""
|
|
1448
|
+
toks = re.findall(r"[A-Za-z0-9][A-Za-z0-9\-\.\*]*", q)
|
|
1449
|
+
return " ".join(toks).strip()
|
|
1450
|
+
|
|
1451
|
+
|
|
1452
|
+
def _build_uniprot_engine(spec: dict[str, Any]) -> Any:
|
|
1453
|
+
"""UniProt 蛋白质检索(rest.uniprot.org/uniprotkb/search,免认证)。
|
|
1454
|
+
|
|
1455
|
+
基因名/蛋白名/物种查询,返回 accession、推荐名、基因名与物种。
|
|
1456
|
+
中英混合查询只取 ASCII token(UniProt 不索引中文),纯中文查高频蛋白名映射表。
|
|
1457
|
+
"""
|
|
1458
|
+
timeout = spec.get("timeout", 12)
|
|
1459
|
+
_HEADERS = {"User-Agent": "argo-search/2.6 (unified-search@local)", "Accept": "application/json"}
|
|
1460
|
+
|
|
1461
|
+
@safe_search
|
|
1462
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
1463
|
+
q_raw = query.strip()
|
|
1464
|
+
if not q_raw:
|
|
1465
|
+
return []
|
|
1466
|
+
q = _ascii_tokens(q_raw)
|
|
1467
|
+
if not q:
|
|
1468
|
+
q = _UNIPROT_CN.get(q_raw, "")
|
|
1469
|
+
if not q:
|
|
1470
|
+
return []
|
|
1471
|
+
to = _timeout or timeout
|
|
1472
|
+
url = f"{_UNIPROT_API}?" + urllib.parse.urlencode({
|
|
1473
|
+
"query": q, "format": "json", "size": min(n, 25),
|
|
1474
|
+
})
|
|
1475
|
+
try:
|
|
1476
|
+
with http_open(urllib.request.Request(url, headers=_HEADERS), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
1477
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
1478
|
+
except Exception as e:
|
|
1479
|
+
logger.warning(f"UniProt 失败: {e}")
|
|
1480
|
+
return []
|
|
1481
|
+
results = []
|
|
1482
|
+
for _rk10, x in enumerate((data.get("results") or [])[:n]):
|
|
1483
|
+
acc = x.get("primaryAccession", "")
|
|
1484
|
+
pd = x.get("proteinDescription") or {}
|
|
1485
|
+
rn = ((pd.get("recommendedName") or {}).get("fullName") or {}).get("value", "")
|
|
1486
|
+
genes = x.get("genes") or []
|
|
1487
|
+
gene = ""
|
|
1488
|
+
if genes and isinstance(genes[0], dict):
|
|
1489
|
+
gene = ((genes[0].get("geneName") or {}).get("value", ""))
|
|
1490
|
+
org = ((x.get("organism") or {}).get("scientificName") or "")
|
|
1491
|
+
title = rn or (f"{acc}" if acc else q)
|
|
1492
|
+
snip = " · ".join(p for p in (org, gene) if p)
|
|
1493
|
+
results.append({
|
|
1494
|
+
"title": f"{title} ({acc})" if acc else title,
|
|
1495
|
+
"url": f"https://www.uniprot.org/uniprotkb/{acc}" if acc else "",
|
|
1496
|
+
"snippet": snip[:300],
|
|
1497
|
+
"source": "uniprot",
|
|
1498
|
+
"score": rank_score(0.9, _rk10),
|
|
1499
|
+
})
|
|
1500
|
+
return results
|
|
1501
|
+
return _engine
|
|
1502
|
+
|
|
1503
|
+
|
|
1504
|
+
# ── RCSB PDB 蛋白质结构引擎 ──────────────────────────────────────────────────
|
|
1505
|
+
|
|
1506
|
+
_PDB_SEARCH_API = "https://search.rcsb.org/rcsbsearch/v2/query"
|
|
1507
|
+
|
|
1508
|
+
|
|
1509
|
+
def _build_rcsb_pdb_engine(spec: dict[str, Any]) -> Any:
|
|
1510
|
+
"""RCSB PDB 蛋白质结构检索(search.rcsb.org v2,免认证)。
|
|
1511
|
+
|
|
1512
|
+
全文搜索返回 PDB ID 列表,链接到 rcsb.org/structure/{id}。
|
|
1513
|
+
"""
|
|
1514
|
+
timeout = spec.get("timeout", 15)
|
|
1515
|
+
_HEADERS = {"User-Agent": "argo-search/2.6 (unified-search@local)",
|
|
1516
|
+
"Accept": "application/json", "Content-Type": "application/json"}
|
|
1517
|
+
|
|
1518
|
+
@safe_search
|
|
1519
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
1520
|
+
q = _ascii_tokens(query)
|
|
1521
|
+
if not q:
|
|
1522
|
+
return []
|
|
1523
|
+
to = _timeout or timeout
|
|
1524
|
+
body = json.dumps({
|
|
1525
|
+
"query": {"type": "terminal", "service": "full_text", "parameters": {"value": q}},
|
|
1526
|
+
"return_type": "entry",
|
|
1527
|
+
"request_options": {"return_all_hits": True},
|
|
1528
|
+
}).encode("utf-8")
|
|
1529
|
+
try:
|
|
1530
|
+
req = urllib.request.Request(_PDB_SEARCH_API, data=body, headers=_HEADERS, method="POST")
|
|
1531
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
1532
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
1533
|
+
except Exception as e:
|
|
1534
|
+
logger.warning(f"RCSB PDB 失败: {e}")
|
|
1535
|
+
return []
|
|
1536
|
+
results = []
|
|
1537
|
+
for _rk11, x in enumerate((data.get("result_set") or [])[:n]):
|
|
1538
|
+
pid = x.get("identifier", "")
|
|
1539
|
+
if not pid:
|
|
1540
|
+
continue
|
|
1541
|
+
results.append({
|
|
1542
|
+
"title": f"PDB 结构 {pid}({q})",
|
|
1543
|
+
"url": f"https://www.rcsb.org/structure/{pid}",
|
|
1544
|
+
"snippet": f"RCSB PDB 实验结构 {pid},分辨率 {x.get('score', '')}"[:300],
|
|
1545
|
+
"source": "rcsb_pdb",
|
|
1546
|
+
"score": rank_score(0.85, _rk11),
|
|
1547
|
+
})
|
|
1548
|
+
return results
|
|
1549
|
+
return _engine
|
|
1550
|
+
|
|
1551
|
+
|
|
1552
|
+
# ── CourtListener 美国判例引擎 ───────────────────────────────────────────────
|
|
1553
|
+
|
|
1554
|
+
_COURTLISTENER_API = "https://www.courtlistener.com/api/rest/v4/search/"
|
|
1555
|
+
|
|
1556
|
+
|
|
1557
|
+
def _build_courtlistener_engine(spec: dict[str, Any]) -> Any:
|
|
1558
|
+
"""CourtListener 美国判例检索(courtlistener.com/api/rest/v4,匿名可用)。
|
|
1559
|
+
|
|
1560
|
+
注意 2026-05 起限额骤降(免费档约 5 次/分、125 次/天),qps 已压到 1。
|
|
1561
|
+
"""
|
|
1562
|
+
timeout = spec.get("timeout", 12)
|
|
1563
|
+
_HEADERS = {"User-Agent": "argo-search/2.6 (unified-search@local)", "Accept": "application/json"}
|
|
1564
|
+
|
|
1565
|
+
@safe_search
|
|
1566
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
1567
|
+
q = query.strip()
|
|
1568
|
+
if not q:
|
|
1569
|
+
return []
|
|
1570
|
+
to = _timeout or timeout
|
|
1571
|
+
url = _COURTLISTENER_API + "?" + urllib.parse.urlencode({
|
|
1572
|
+
"q": q, "format": "json", "page_size": min(n, 10), "order_by": "score desc",
|
|
1573
|
+
})
|
|
1574
|
+
try:
|
|
1575
|
+
with http_open(urllib.request.Request(url, headers=_HEADERS), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
1576
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
1577
|
+
except Exception as e:
|
|
1578
|
+
logger.warning(f"CourtListener 失败: {e}")
|
|
1579
|
+
return []
|
|
1580
|
+
results = []
|
|
1581
|
+
for _rk12, x in enumerate((data.get("results") or [])[:n]):
|
|
1582
|
+
name = x.get("caseName") or q
|
|
1583
|
+
rel = x.get("absolute_url") or ""
|
|
1584
|
+
date = (x.get("dateFiled") or "")[:10]
|
|
1585
|
+
snip = str(x.get("snippet") or "")[:280]
|
|
1586
|
+
body_txt = str(x.get("plain_text") or "")[:280]
|
|
1587
|
+
snip = snip or body_txt
|
|
1588
|
+
parts = [p for p in (date, snip) if p]
|
|
1589
|
+
results.append({
|
|
1590
|
+
"title": name,
|
|
1591
|
+
"url": f"https://www.courtlistener.com{rel}" if rel else "",
|
|
1592
|
+
"snippet": " · ".join(parts)[:300],
|
|
1593
|
+
"source": "courtlistener",
|
|
1594
|
+
"score": rank_score(0.9, _rk12),
|
|
1595
|
+
})
|
|
1596
|
+
return results
|
|
1597
|
+
return _engine
|
|
1598
|
+
|
|
1599
|
+
|
|
1600
|
+
# ── Project Gutenberg 公版书引擎(Gutendex) ─────────────────────────────────
|
|
1601
|
+
|
|
1602
|
+
_GUTENDEX_API = "https://gutendex.com/books"
|
|
1603
|
+
|
|
1604
|
+
|
|
1605
|
+
def _build_gutenberg_engine(spec: dict[str, Any]) -> Any:
|
|
1606
|
+
"""Project Gutenberg 公版书检索(gutendex.com,免认证)。
|
|
1607
|
+
|
|
1608
|
+
返回书名/作者/语言与 Gutenberg 书目页链接;Gutenberg 提供结构化全文。
|
|
1609
|
+
"""
|
|
1610
|
+
timeout = spec.get("timeout", 15)
|
|
1611
|
+
_HEADERS = {"User-Agent": "argo-search/2.6 (unified-search@local)", "Accept": "application/json"}
|
|
1612
|
+
|
|
1613
|
+
@safe_search
|
|
1614
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
1615
|
+
q = query.strip()
|
|
1616
|
+
if not q:
|
|
1617
|
+
return []
|
|
1618
|
+
to = _timeout or timeout
|
|
1619
|
+
url = _GUTENDEX_API + "?" + urllib.parse.urlencode({"search": q, "page_size": min(n, 10)})
|
|
1620
|
+
try:
|
|
1621
|
+
with http_open(urllib.request.Request(url, headers=_HEADERS), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
1622
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
1623
|
+
except Exception as e:
|
|
1624
|
+
logger.warning(f"Gutendex 失败: {e}")
|
|
1625
|
+
return []
|
|
1626
|
+
results = []
|
|
1627
|
+
for _rk13, x in enumerate((data.get("results") or [])[:n]):
|
|
1628
|
+
title = x.get("title") or q
|
|
1629
|
+
gid = x.get("id") or ""
|
|
1630
|
+
authors = ", ".join((a.get("name") or "") for a in (x.get("authors") or [])[:2])
|
|
1631
|
+
langs = ",".join(x.get("languages") or [])
|
|
1632
|
+
fmt = x.get("formats") or {}
|
|
1633
|
+
txt = fmt.get("text/plain; charset=us-ascii") or fmt.get("text/html")
|
|
1634
|
+
snip = " · ".join(p for p in (authors, langs, "公版书全文") if p)
|
|
1635
|
+
_row = {
|
|
1636
|
+
"title": title,
|
|
1637
|
+
"url": f"https://www.gutenberg.org/ebooks/{gid}" if gid else txt or "",
|
|
1638
|
+
"snippet": snip[:300],
|
|
1639
|
+
"source": "gutenberg",
|
|
1640
|
+
"score": rank_score(0.85, _rk13),
|
|
1641
|
+
}
|
|
1642
|
+
# 源内全文直出:`/ebooks/{id}` 是下载门户页(实测首段全是 noprint
|
|
1643
|
+
# 脚本),不是正文;gutendex 本来就把纯文本 URL 放在 formats 里,
|
|
1644
|
+
# 此前只在缺 id 时当保底,正常路径直接丢掉。这里单独带上,供下游
|
|
1645
|
+
# 取正文时优先使用(省掉从门户页找下载链这一步)。
|
|
1646
|
+
if txt:
|
|
1647
|
+
_row["full_text_url"] = txt
|
|
1648
|
+
results.append(_row)
|
|
1649
|
+
return results
|
|
1650
|
+
return _engine
|
|
1651
|
+
|
|
1652
|
+
|
|
1653
|
+
# ── Wayback CDX 已删除内容检索引擎 ───────────────────────────────────────────
|
|
1654
|
+
|
|
1655
|
+
_CDX_API = "https://web.archive.org/cdx/search/cdx"
|
|
1656
|
+
|
|
1657
|
+
|
|
1658
|
+
def _build_wayback_cdx_engine(spec: dict[str, Any]) -> Any:
|
|
1659
|
+
"""Wayback Machine CDX 快照检索(web.archive.org/cdx,免认证)。
|
|
1660
|
+
|
|
1661
|
+
输入域名/URL,返回历史快照时间戳与状态码,是查已删除页面的唯一路径。
|
|
1662
|
+
"""
|
|
1663
|
+
timeout = spec.get("timeout", 20)
|
|
1664
|
+
_HEADERS = {"User-Agent": "argo-search/2.6 (unified-search@local)", "Accept": "application/json"}
|
|
1665
|
+
|
|
1666
|
+
@safe_search
|
|
1667
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
1668
|
+
q = query.strip().lower()
|
|
1669
|
+
if not q:
|
|
1670
|
+
return []
|
|
1671
|
+
# 剥离常见协议前缀,CDX 用裸域名/host 查询
|
|
1672
|
+
q = re.sub(r"^https?://", "", q).strip()
|
|
1673
|
+
if not q:
|
|
1674
|
+
return []
|
|
1675
|
+
to = _timeout or timeout
|
|
1676
|
+
url = _CDX_API + "?" + urllib.parse.urlencode({
|
|
1677
|
+
"url": q, "output": "json", "limit": min(n, 10),
|
|
1678
|
+
"fl": "timestamp,original,statuscode",
|
|
1679
|
+
})
|
|
1680
|
+
try:
|
|
1681
|
+
with http_open(urllib.request.Request(url, headers=_HEADERS), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
1682
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
1683
|
+
except Exception as e:
|
|
1684
|
+
logger.warning(f"Wayback CDX 失败: {e}")
|
|
1685
|
+
return []
|
|
1686
|
+
rows = data if isinstance(data, list) else []
|
|
1687
|
+
# 带 fl 时首行是列名
|
|
1688
|
+
if rows and rows[0] and rows[0][0] == "timestamp":
|
|
1689
|
+
rows = rows[1:]
|
|
1690
|
+
results = []
|
|
1691
|
+
for _rk14, r in enumerate(rows[:n]):
|
|
1692
|
+
if len(r) < 3:
|
|
1693
|
+
continue
|
|
1694
|
+
ts, orig, status = r[0], r[1], r[2]
|
|
1695
|
+
# CDX 时间戳 YYYYMMDDHHMMSS → 标准 published_at(最早快照语义)
|
|
1696
|
+
published_at = ""
|
|
1697
|
+
try:
|
|
1698
|
+
published_at = datetime.strptime(ts, "%Y%m%d%H%M%S").isoformat(timespec="seconds")
|
|
1699
|
+
except ValueError:
|
|
1700
|
+
pass
|
|
1701
|
+
item = {
|
|
1702
|
+
"title": f"快照 {orig}({ts[:8]})",
|
|
1703
|
+
"url": f"https://web.archive.org/web/{ts}/{orig}",
|
|
1704
|
+
"snippet": f"Wayback 快照 {ts} · HTTP {status}"[:300],
|
|
1705
|
+
"source": "wayback_cdx",
|
|
1706
|
+
"score": rank_score(0.8, _rk14),
|
|
1707
|
+
}
|
|
1708
|
+
if published_at:
|
|
1709
|
+
item["published_at"] = published_at
|
|
1710
|
+
results.append(item)
|
|
1711
|
+
return results
|
|
1712
|
+
return _engine
|
|
1713
|
+
|
|
1714
|
+
|
|
1715
|
+
# ── USGS 地震引擎 ────────────────────────────────────────────────────────────
|
|
1716
|
+
|
|
1717
|
+
_USGS_EQ_API = "https://earthquake.usgs.gov/fdsnws/event/1/query"
|
|
1718
|
+
|
|
1719
|
+
# 中文地点词 → (place 匹配正则, bbox 地理框或 None)
|
|
1720
|
+
# USGS place 是英文;美国国内震 place 用「城市, 州缩写」格式("6 km NW of Pinnacles, CA")。
|
|
1721
|
+
# bbox 可让「台湾地震」这类查询不依赖全球 top-50,直接框选该地区全部地震。
|
|
1722
|
+
_USGS_PLACE_CN = {
|
|
1723
|
+
"日本": ("japan", dict(minlatitude=24, maxlatitude=47, minlongitude=122, maxlongitude=147)),
|
|
1724
|
+
"台湾": ("taiwan", dict(minlatitude=20, maxlatitude=26, minlongitude=119, maxlongitude=124)),
|
|
1725
|
+
"加州": (r"california|, ca\b", dict(minlatitude=32, maxlatitude=42, minlongitude=-125, maxlongitude=-114)),
|
|
1726
|
+
"中国": ("china", None), "香港": ("hong kong", None),
|
|
1727
|
+
"美国": (r", (ca|nv|ak|or|wa|hi|az|ut|mt|id|nm|tx|ok|ar|mo|ky|tn|sc|nc|ga)\b", None),
|
|
1728
|
+
"阿拉斯加": ("alaska", dict(minlatitude=48, maxlatitude=72, minlongitude=-170, maxlongitude=-130)),
|
|
1729
|
+
"夏威夷": ("hawaii", None),
|
|
1730
|
+
"印尼": ("indonesia", None), "菲律宾": ("philippines", None), "新西兰": ("new zealand", None),
|
|
1731
|
+
"智利": ("chile", None), "秘鲁": ("peru", None), "墨西哥": ("mexico", None), "土耳其": ("turkey", None),
|
|
1732
|
+
"伊朗": ("iran", None), "印度": ("india", None), "尼泊尔": ("nepal", None), "意大利": ("italy", None),
|
|
1733
|
+
"希腊": ("greece", None), "冰岛": ("iceland", None), "阿富汗": ("afghanistan", None), "缅甸": ("myanmar", None),
|
|
1734
|
+
"巴布亚": ("papua", None), "所罗门": ("solomon", None), "汤加": ("tonga", None), "瓦努阿图": ("vanuatu", None),
|
|
1735
|
+
"斐济": ("fiji", None), "关岛": ("guam", None), "千岛群岛": ("kuril", None),
|
|
1736
|
+
}
|
|
1737
|
+
|
|
1738
|
+
|
|
1739
|
+
def _build_usgs_engine(spec: dict[str, Any]) -> Any:
|
|
1740
|
+
"""USGS 地震检索(earthquake.usgs.gov FDSNWS,免认证)。
|
|
1741
|
+
|
|
1742
|
+
返回最近 30 天 M2.5+ 地震;查询含地点词(如「日本」「加州」)时客户端侧
|
|
1743
|
+
按 place 字段过滤,命中地点相关地震。
|
|
1744
|
+
"""
|
|
1745
|
+
timeout = spec.get("timeout", 12)
|
|
1746
|
+
_HEADERS = {"User-Agent": "argo-search/2.6 (unified-search@local)", "Accept": "application/json"}
|
|
1747
|
+
|
|
1748
|
+
@safe_search
|
|
1749
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
1750
|
+
q = query.strip()
|
|
1751
|
+
if not q:
|
|
1752
|
+
return []
|
|
1753
|
+
to = _timeout or timeout
|
|
1754
|
+
# 提取地点词:去掉「地震/earthquake/震级」等噪声
|
|
1755
|
+
place_kw = re.sub(r"(?i)(地震|earthquake|seismic|震级|magnitude|quakes?|最近|最新)", " ", q)
|
|
1756
|
+
place_kw = re.sub(r"\s+", " ", place_kw).strip()
|
|
1757
|
+
params = {
|
|
1758
|
+
"format": "geojson", "limit": 50,
|
|
1759
|
+
"starttime": time.strftime("%Y-%m-%d", time.gmtime(time.time() - 30 * 86400)),
|
|
1760
|
+
"minmagnitude": "2.5",
|
|
1761
|
+
}
|
|
1762
|
+
if place_kw:
|
|
1763
|
+
kw = place_kw.lower()
|
|
1764
|
+
bbox = None
|
|
1765
|
+
# 中文地点词查映射表(正则 + 可选 bbox);英文词直接按原文匹配 place
|
|
1766
|
+
if not re.search(r"[a-z]", kw):
|
|
1767
|
+
entry = _USGS_PLACE_CN.get(place_kw)
|
|
1768
|
+
if entry:
|
|
1769
|
+
kw, bbox = entry
|
|
1770
|
+
if bbox:
|
|
1771
|
+
params.update(bbox)
|
|
1772
|
+
# bbox 精确框选时无需放大样本,直接按时间取
|
|
1773
|
+
params["limit"] = min(n * 3, 30)
|
|
1774
|
+
else:
|
|
1775
|
+
# 无 bbox 时按震级降序取样本,避免目标地点被全球密集地震挤出 top
|
|
1776
|
+
params["orderby"] = "magnitude"
|
|
1777
|
+
url = _USGS_EQ_API + "?" + urllib.parse.urlencode(params)
|
|
1778
|
+
try:
|
|
1779
|
+
with http_open(urllib.request.Request(url, headers=_HEADERS), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
1780
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
1781
|
+
except Exception as e:
|
|
1782
|
+
logger.warning(f"USGS 失败: {e}")
|
|
1783
|
+
return []
|
|
1784
|
+
feats = data.get("features") or []
|
|
1785
|
+
if place_kw:
|
|
1786
|
+
if kw:
|
|
1787
|
+
feats = [f for f in feats if re.search(kw, str(f.get("properties", {}).get("place", "")).lower())]
|
|
1788
|
+
results = []
|
|
1789
|
+
for _rk15, f in enumerate(feats[:n]):
|
|
1790
|
+
p = f.get("properties", {})
|
|
1791
|
+
mag = p.get("mag")
|
|
1792
|
+
place = p.get("place", "")
|
|
1793
|
+
ts = (p.get("time") or 0) // 1000
|
|
1794
|
+
url_ = p.get("url", "")
|
|
1795
|
+
results.append({
|
|
1796
|
+
"title": f"M{mag} {place}",
|
|
1797
|
+
"url": url_ or "",
|
|
1798
|
+
"snippet": f"USGS 地震 · {ts and time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime(ts))}"[:300],
|
|
1799
|
+
"source": "usgs",
|
|
1800
|
+
"score": rank_score(0.85, _rk15),
|
|
1801
|
+
})
|
|
1802
|
+
return results
|
|
1803
|
+
return _engine
|
|
1804
|
+
|
|
1805
|
+
|
|
1806
|
+
# ── NASA CMR 地球科学数据目录引擎 ────────────────────────────────────────────
|
|
1807
|
+
|
|
1808
|
+
_CMR_API = "https://cmr.earthdata.nasa.gov/search/collections.json"
|
|
1809
|
+
|
|
1810
|
+
|
|
1811
|
+
def _build_nasa_cmr_engine(spec: dict[str, Any]) -> Any:
|
|
1812
|
+
"""NASA CMR 地球科学数据目录检索(cmr.earthdata.nasa.gov,免认证)。
|
|
1813
|
+
|
|
1814
|
+
MODIS/卫星/遥感等关键词 → 数据集集合(含概念 ID,链接 Earthdata Search)。
|
|
1815
|
+
"""
|
|
1816
|
+
timeout = spec.get("timeout", 12)
|
|
1817
|
+
_HEADERS = {"User-Agent": "argo-search/2.6 (unified-search@local)", "Accept": "application/json"}
|
|
1818
|
+
|
|
1819
|
+
@safe_search
|
|
1820
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
1821
|
+
q = _ascii_tokens(query)
|
|
1822
|
+
if not q:
|
|
1823
|
+
return []
|
|
1824
|
+
to = _timeout or timeout
|
|
1825
|
+
url = _CMR_API + "?" + urllib.parse.urlencode({"keyword": q, "page_size": min(n, 10)})
|
|
1826
|
+
try:
|
|
1827
|
+
with http_open(urllib.request.Request(url, headers=_HEADERS), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
1828
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
1829
|
+
except Exception as e:
|
|
1830
|
+
logger.warning(f"NASA CMR 失败: {e}")
|
|
1831
|
+
return []
|
|
1832
|
+
results = []
|
|
1833
|
+
for _rk16, x in enumerate(((data.get("feed") or {}).get("entry") or [])[:n]):
|
|
1834
|
+
title = x.get("title") or q
|
|
1835
|
+
cid = x.get("id") or ""
|
|
1836
|
+
summary = (x.get("summary") or "")[:200]
|
|
1837
|
+
results.append({
|
|
1838
|
+
"title": title,
|
|
1839
|
+
"url": f"https://search.earthdata.nasa.gov/search/granules?p={cid}" if cid else "",
|
|
1840
|
+
"snippet": summary[:300],
|
|
1841
|
+
"source": "nasa_cmr",
|
|
1842
|
+
"score": rank_score(0.8, _rk16),
|
|
1843
|
+
})
|
|
1844
|
+
return results
|
|
1845
|
+
return _engine
|
|
1846
|
+
|
|
1847
|
+
|
|
1848
|
+
# ── IMDb 影视 suggestion API(免认证) ────────────────────────────────────────
|
|
1849
|
+
|
|
1850
|
+
_IMDB_SUGGEST = "https://v2.sg.media-imdb.com/suggestion/{first}/{q}.json"
|
|
1851
|
+
_IMDB_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36"
|
|
1852
|
+
|
|
1853
|
+
|
|
1854
|
+
# 影视问句噪声词:整句进 suggestion 常 0 命中(如 "Interstellar film cast")
|
|
1855
|
+
_IMDB_STOP = re.compile(
|
|
1856
|
+
r"(?i)\b(movie|film|tv\s*show|tv\s*series|cinema|cast|starring|director|directed\s+by|"
|
|
1857
|
+
r"screenwriter|written\s+by|actor|actress|producer|box\s*office|trailer|review|"
|
|
1858
|
+
r"imdb|豆瓣|电影|电视剧|剧集|影视|影片|导演|主演|演员|编剧|制片|上映|首映|票房|"
|
|
1859
|
+
r"影评|奥斯卡|金像奖|金马奖|网剧|美剧|韩剧|日剧|英剧)\b"
|
|
1860
|
+
)
|
|
1861
|
+
|
|
1862
|
+
|
|
1863
|
+
def _imdb_clean_query(query: str) -> str:
|
|
1864
|
+
"""去掉影视角色/事实词,保留片名/人名主体。"""
|
|
1865
|
+
q = (query or "").strip()
|
|
1866
|
+
if not q:
|
|
1867
|
+
return ""
|
|
1868
|
+
cleaned = _IMDB_STOP.sub(" ", q)
|
|
1869
|
+
cleaned = re.sub(r"[??!!,,.。::;;]+", " ", cleaned)
|
|
1870
|
+
cleaned = re.sub(r"\s+", " ", cleaned).strip()
|
|
1871
|
+
# 清洗后过短则回退原 query(避免只剩空)
|
|
1872
|
+
if len(cleaned) < 2:
|
|
1873
|
+
return q
|
|
1874
|
+
return cleaned
|
|
1875
|
+
|
|
1876
|
+
|
|
1877
|
+
def _build_imdb_engine(spec: dict[str, Any]) -> Any:
|
|
1878
|
+
"""IMDb 自动补全 suggestion API(v2.sg.media-imdb.com,免认证)。
|
|
1879
|
+
|
|
1880
|
+
路径:/suggestion/{首字符}/{query}.json
|
|
1881
|
+
返回电影/剧集/人物,带 tt/nm ID,可拼正式 IMDb URL。
|
|
1882
|
+
"""
|
|
1883
|
+
timeout = spec.get("timeout", 8)
|
|
1884
|
+
source_name = spec.get("_name", "imdb")
|
|
1885
|
+
|
|
1886
|
+
def _fetch_suggest(q: str, to: float) -> list[dict[str, Any]]:
|
|
1887
|
+
if not q:
|
|
1888
|
+
return []
|
|
1889
|
+
first = q[0].lower()
|
|
1890
|
+
if not (("a" <= first <= "z") or ("0" <= first <= "9")):
|
|
1891
|
+
first = "_"
|
|
1892
|
+
path_q = urllib.parse.quote(q[:80], safe="")
|
|
1893
|
+
url = _IMDB_SUGGEST.format(first=first, q=path_q)
|
|
1894
|
+
try:
|
|
1895
|
+
req = urllib.request.Request(url, headers={"User-Agent": _IMDB_UA, "Accept": "application/json"})
|
|
1896
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
1897
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
1898
|
+
except Exception as e:
|
|
1899
|
+
logger.warning(f"IMDb 失败: {e}")
|
|
1900
|
+
return []
|
|
1901
|
+
return [x for x in (data.get("d") or []) if isinstance(x, dict)]
|
|
1902
|
+
|
|
1903
|
+
@safe_search
|
|
1904
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
1905
|
+
raw = (query or "").strip()
|
|
1906
|
+
if not raw:
|
|
1907
|
+
return []
|
|
1908
|
+
to = _timeout or timeout
|
|
1909
|
+
cleaned = _imdb_clean_query(raw)
|
|
1910
|
+
# 先试清洗词,空则回退原句
|
|
1911
|
+
items = _fetch_suggest(cleaned, to)
|
|
1912
|
+
if not items and cleaned != raw:
|
|
1913
|
+
items = _fetch_suggest(raw, to)
|
|
1914
|
+
|
|
1915
|
+
# 按标题与查询 token 重叠 + feature 优先排序
|
|
1916
|
+
q_tokens = {t.lower() for t in re.findall(r"[A-Za-z0-9\u4e00-\u9fff]{2,}", cleaned or raw)}
|
|
1917
|
+
ranked: list[tuple[float, dict[str, Any]]] = []
|
|
1918
|
+
for item in items:
|
|
1919
|
+
title = item.get("l") or ""
|
|
1920
|
+
iid = item.get("id") or ""
|
|
1921
|
+
if not title and not iid:
|
|
1922
|
+
continue
|
|
1923
|
+
t_low = title.lower()
|
|
1924
|
+
t_tokens = {t.lower() for t in re.findall(r"[A-Za-z0-9\u4e00-\u9fff]{2,}", title)}
|
|
1925
|
+
overlap = len(q_tokens & t_tokens) / max(len(q_tokens), 1)
|
|
1926
|
+
# 精确/前缀加分;tt 影片优先于 nm 人名(片名查询时)
|
|
1927
|
+
exact = 1.0 if t_low == (cleaned or raw).lower() else 0.0
|
|
1928
|
+
prefix = 0.3 if t_low.startswith((cleaned or raw).lower()[: min(12, len(cleaned or raw))]) else 0.0
|
|
1929
|
+
kind_bonus = 0.15 if str(iid).startswith("tt") else (0.05 if str(iid).startswith("nm") else 0.0)
|
|
1930
|
+
year = item.get("y") or 0
|
|
1931
|
+
try:
|
|
1932
|
+
year_n = int(year) if year else 0
|
|
1933
|
+
except (TypeError, ValueError):
|
|
1934
|
+
year_n = 0
|
|
1935
|
+
# 近年作品轻微加分(同名旧片靠后)
|
|
1936
|
+
recency = min(max(year_n - 1980, 0), 50) / 500.0
|
|
1937
|
+
score = 0.5 + overlap + exact + prefix + kind_bonus + recency
|
|
1938
|
+
ranked.append((score, item))
|
|
1939
|
+
ranked.sort(key=lambda x: -x[0])
|
|
1940
|
+
|
|
1941
|
+
results = []
|
|
1942
|
+
for score, item in ranked[: max(n, 1)]:
|
|
1943
|
+
title = item.get("l") or ""
|
|
1944
|
+
iid = item.get("id") or ""
|
|
1945
|
+
year = item.get("y")
|
|
1946
|
+
kind = item.get("q") or item.get("qid") or ""
|
|
1947
|
+
cast = item.get("s") or ""
|
|
1948
|
+
parts = [p for p in (str(year) if year else "", kind, cast) if p]
|
|
1949
|
+
if str(iid).startswith("tt"):
|
|
1950
|
+
href = f"https://www.imdb.com/title/{iid}/"
|
|
1951
|
+
elif str(iid).startswith("nm"):
|
|
1952
|
+
href = f"https://www.imdb.com/name/{iid}/"
|
|
1953
|
+
else:
|
|
1954
|
+
href = f"https://www.imdb.com/find/?q={urllib.parse.quote(title)}"
|
|
1955
|
+
results.append({
|
|
1956
|
+
"title": title,
|
|
1957
|
+
"url": href,
|
|
1958
|
+
"snippet": " · ".join(parts)[:300],
|
|
1959
|
+
"source": source_name,
|
|
1960
|
+
"score": min(0.95, 0.7 + score * 0.15),
|
|
1961
|
+
})
|
|
1962
|
+
if len(results) >= n:
|
|
1963
|
+
break
|
|
1964
|
+
return results
|
|
1965
|
+
return _engine
|
|
1966
|
+
|
|
1967
|
+
|
|
1968
|
+
# ── iTunes 媒体库(中文需 country=cn;专辑问 entity=album;播客问 entity=podcast)──
|
|
1969
|
+
|
|
1970
|
+
# 媒体意图词。引擎可被 --engine 直接调用,那时拿不到路由域的上下文,
|
|
1971
|
+
# 意图只能从查询词本身读——命中播客词即把 media 从 music 切到 podcast。
|
|
1972
|
+
_PODCAST_HINT_RE = re.compile(r"(?i)(播客|podcast|电台|单集|一期|这期|那期|episode)")
|
|
1973
|
+
# 单集意图:用户要的是某期节目(带时长),而非节目本体
|
|
1974
|
+
_PODCAST_EPISODE_RE = re.compile(r"(?i)(单集|一期|这期|那期|episode|第\s*\d+\s*集|#\s*\d+)")
|
|
1975
|
+
_ALBUM_HINT_RE = re.compile(r"(?i)(专辑|album|discography|唱片)")
|
|
1976
|
+
# 意图词只用于定实体,须从检索词里剥掉,否则「播客」这类词会污染 term
|
|
1977
|
+
_MEDIA_INTENT_WORD_RE = re.compile(
|
|
1978
|
+
r"(?i)(专辑|新专辑|唱片|音乐|歌曲|单曲|播客|电台|单集|一期|这期|那期|"
|
|
1979
|
+
r"album|discography|music|song|songs|single|podcast|podcasts|episode)"
|
|
1980
|
+
)
|
|
1981
|
+
# 节目级时长必须二跳取样(见下方 docstring),逐条补会让请求数随结果数膨胀。
|
|
1982
|
+
# 只补首条:播客名查询通常只有一个正解(实测「张小珺 商业访谈录」Apple 只回 1 条)。
|
|
1983
|
+
_ITUNES_SHOW_ENRICH_MAX = 1
|
|
1984
|
+
# 文字稿/feed 拉取的字节上限:预览只需几百字,2MB 足够防超大文件吃内存
|
|
1985
|
+
_TRANSCRIPT_MAX_BYTES = 2 * 1024 * 1024
|
|
1986
|
+
|
|
1987
|
+
|
|
1988
|
+
def _itunes_date(raw: Any) -> str:
|
|
1989
|
+
"""Apple 的 releaseDate 形如 2026-09-03T00:00:00Z,裁到日期部分供展示。"""
|
|
1990
|
+
s = str(raw or "").strip()
|
|
1991
|
+
return s[:10] if len(s) >= 10 else ""
|
|
1992
|
+
|
|
1993
|
+
|
|
1994
|
+
def _transcript_to_text(raw: str, ctype: str) -> str:
|
|
1995
|
+
"""文字稿原文 → 纯文本。VTT/SRT 去时间轴与样式行,HTML 去 tag。
|
|
1996
|
+
|
|
1997
|
+
Podcasting 2.0 文字稿的 type 值不统一(text/vtt、application/srt、
|
|
1998
|
+
裸 text/plain 都见过),按字符串包含判断而非全等。
|
|
1999
|
+
"""
|
|
2000
|
+
if "html" in ctype:
|
|
2001
|
+
text = re.sub(r"<[^>]+>", " ", raw)
|
|
2002
|
+
text = _unescape(text)
|
|
2003
|
+
elif "vtt" in ctype or "srt" in ctype or "plain" in ctype:
|
|
2004
|
+
lines: list[str] = []
|
|
2005
|
+
for ln in raw.splitlines():
|
|
2006
|
+
s = ln.strip()
|
|
2007
|
+
if (not s or s == "WEBVTT"
|
|
2008
|
+
or s.startswith(("NOTE", "STYLE", "REGION"))
|
|
2009
|
+
or "-->" in s or s.isdigit()):
|
|
2010
|
+
continue
|
|
2011
|
+
s = re.sub(r"<[^>]+>", "", s) # <c> 高亮、<v 说话人> 内联标签
|
|
2012
|
+
if s:
|
|
2013
|
+
lines.append(s)
|
|
2014
|
+
text = " ".join(lines)
|
|
2015
|
+
else:
|
|
2016
|
+
text = raw
|
|
2017
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
2018
|
+
|
|
2019
|
+
|
|
2020
|
+
def _build_itunes_engine(spec: dict[str, Any]) -> Any:
|
|
2021
|
+
"""iTunes Search API:按语言与意图调 country / media / entity。
|
|
2022
|
+
|
|
2023
|
+
三种媒体形态(播客为 2026-09-15 补入):
|
|
2024
|
+
- 播客节目 media=podcast&entity=podcast → 集数(trackCount)、更新日期、类型
|
|
2025
|
+
- 播客单集 media=podcast&entity=podcastEpisode → 时长(trackTimeMillis)、发布日期
|
|
2026
|
+
- 音乐 media=music(专辑问 entity=album)
|
|
2027
|
+
|
|
2028
|
+
播客时长只能取自单集:**节目对象上的 trackTimeMillis 不是时长**。实测 12 档
|
|
2029
|
+
节目只有 6 档等于最新单集的秒数,另 6 档明显偏小(忽左忽右 3491 而最新单集
|
|
2030
|
+
5739),单位与含义上游都未文档化——取信它会静默给出错值。所以节目级的
|
|
2031
|
+
「每期多长」走二跳单集取样,见 _recent_episode_duration。
|
|
2032
|
+
"""
|
|
2033
|
+
timeout = spec.get("timeout", 8)
|
|
2034
|
+
source_name = spec.get("_name", "itunes")
|
|
2035
|
+
engine_tag = spec.get("_name", "")
|
|
2036
|
+
base = spec.get("url") or "https://itunes.apple.com/search"
|
|
2037
|
+
lookup_base = (base.replace("/search", "/lookup") if "/search" in base
|
|
2038
|
+
else "https://itunes.apple.com/lookup")
|
|
2039
|
+
ua_headers = {"User-Agent": "argo-search/2.6 (itunes)", "Accept": "application/json"}
|
|
2040
|
+
|
|
2041
|
+
def _get_json(url: str, to: float) -> dict[str, Any]:
|
|
2042
|
+
req = urllib.request.Request(url, headers=ua_headers)
|
|
2043
|
+
with http_open(req, timeout=to, engine=engine_tag) as resp:
|
|
2044
|
+
return json.loads(resp.read().decode("utf-8", "replace"))
|
|
2045
|
+
|
|
2046
|
+
def _recent_episode_duration(collection_id: Any, country: str, to: float) -> tuple[int | None, int]:
|
|
2047
|
+
"""节目近 5 期单集时长的中位数 → (中位分钟, 样本数)。
|
|
2048
|
+
|
|
2049
|
+
取中位数而非最新一期:访谈类单集长度波动大,一期 4 小时长访谈代表不了
|
|
2050
|
+
整个节目。任何失败一律退回 (None, 0),节目本体照常返回——时长是补充
|
|
2051
|
+
信息,不该让整条结果缺失。
|
|
2052
|
+
"""
|
|
2053
|
+
import statistics
|
|
2054
|
+
if not collection_id:
|
|
2055
|
+
return None, 0
|
|
2056
|
+
params = {"id": str(collection_id), "entity": "podcastEpisode", "limit": "5"}
|
|
2057
|
+
if country:
|
|
2058
|
+
params["country"] = country
|
|
2059
|
+
try:
|
|
2060
|
+
data = _get_json(f"{lookup_base}?{urllib.parse.urlencode(params)}", to)
|
|
2061
|
+
except Exception as e:
|
|
2062
|
+
logger.debug(f"iTunes 单集取样失败: {e}")
|
|
2063
|
+
return None, 0
|
|
2064
|
+
mins = []
|
|
2065
|
+
for it in data.get("results") or []:
|
|
2066
|
+
if not isinstance(it, dict):
|
|
2067
|
+
continue
|
|
2068
|
+
if str(it.get("wrapperType") or "").lower() != "podcastepisode":
|
|
2069
|
+
continue
|
|
2070
|
+
ms = it.get("trackTimeMillis")
|
|
2071
|
+
if isinstance(ms, (int, float)) and ms > 0:
|
|
2072
|
+
mins.append(ms / 60000.0)
|
|
2073
|
+
if not mins:
|
|
2074
|
+
return None, 0
|
|
2075
|
+
return int(round(statistics.median(mins))), len(mins)
|
|
2076
|
+
|
|
2077
|
+
def _show_transcript(feed_url: str, to: float) -> tuple[str, str]:
|
|
2078
|
+
"""节目官方文字稿(Podcasting 2.0):feed 里 <podcast:transcript> 标签
|
|
2079
|
+
指向创作者自己挂的全文,不是 ASR 猜的——这是播客转录里唯一免费且
|
|
2080
|
+
准确的通道(英文生态采纳率高,中文节目普遍没挂)。
|
|
2081
|
+
|
|
2082
|
+
取 feed 中第一份可解析格式的文字稿(最新一期):搜索对象是「节目」,
|
|
2083
|
+
最新一期代表当前内容;历史各期的文字稿同挂在 feed,下游拿着
|
|
2084
|
+
transcript_url 可回读全文。任何失败一律 ("", ""),节目本体照常返回。
|
|
2085
|
+
application/json(Podcasting 2.0 的机器格式)本版不解析,静默跳过。
|
|
2086
|
+
"""
|
|
2087
|
+
if not feed_url:
|
|
2088
|
+
return "", ""
|
|
2089
|
+
feed_url = str(feed_url).strip()
|
|
2090
|
+
if not feed_url.lower().startswith(("https://", "http://")):
|
|
2091
|
+
return "", ""
|
|
2092
|
+
try:
|
|
2093
|
+
req = urllib.request.Request(feed_url, headers=ua_headers)
|
|
2094
|
+
with http_open(req, timeout=to, engine=engine_tag) as resp:
|
|
2095
|
+
feed_xml = resp.read(_TRANSCRIPT_MAX_BYTES).decode("utf-8", "replace")
|
|
2096
|
+
except Exception as e:
|
|
2097
|
+
logger.debug(f"iTunes 文字稿 feed 拉取失败: {e}")
|
|
2098
|
+
return "", ""
|
|
2099
|
+
ok_types = ("text/vtt", "text/srt", "text/plain", "text/html")
|
|
2100
|
+
ok_exts = ("vtt", "srt", "txt", "html", "htm")
|
|
2101
|
+
for m in re.finditer(r"<podcast:transcript\b[^>]*>", feed_xml):
|
|
2102
|
+
tag = m.group(0)
|
|
2103
|
+
# 属性可能是单引号或双引号;type 可能带 ;charset= 参数
|
|
2104
|
+
url_m = re.search(r"""\burl=("([^"]+)"|'([^']+)')""", tag)
|
|
2105
|
+
if not url_m:
|
|
2106
|
+
continue
|
|
2107
|
+
raw_url = url_m.group(2) or url_m.group(3)
|
|
2108
|
+
type_m = re.search(r"""\btype=("([^"]+)"|'([^']+)')""", tag)
|
|
2109
|
+
ctype = (type_m.group(2) or type_m.group(3) if type_m else "").lower()
|
|
2110
|
+
ctype = ctype.split(";", 1)[0].strip()
|
|
2111
|
+
try:
|
|
2112
|
+
parsed = urllib.parse.urlsplit(raw_url)
|
|
2113
|
+
if parsed.scheme == "":
|
|
2114
|
+
parsed = urllib.parse.urlsplit(urllib.parse.urljoin(feed_url, raw_url))
|
|
2115
|
+
# 文字稿 URL 来自第三方 feed 内容:仅接受 https,防 file://
|
|
2116
|
+
# 读本地文件、防内网地址(SSRF);urlsplit 对畸形输入抛
|
|
2117
|
+
# ValueError,一并拦下
|
|
2118
|
+
if parsed.scheme != "https" or not parsed.hostname:
|
|
2119
|
+
continue
|
|
2120
|
+
t_url = urllib.parse.urlunsplit(parsed)
|
|
2121
|
+
except ValueError:
|
|
2122
|
+
continue
|
|
2123
|
+
if not ctype:
|
|
2124
|
+
ext = parsed.path.rsplit(".", 1)[-1].lower() if "." in parsed.path.rsplit("?", 1)[0] else ""
|
|
2125
|
+
if ext in ok_exts:
|
|
2126
|
+
ctype = "text/plain" if ext in ("txt", "srt") else f"text/{ext}"
|
|
2127
|
+
if ctype not in ok_types:
|
|
2128
|
+
continue
|
|
2129
|
+
try:
|
|
2130
|
+
req = urllib.request.Request(t_url, headers=ua_headers)
|
|
2131
|
+
with http_open(req, timeout=to, engine=engine_tag) as resp:
|
|
2132
|
+
raw = resp.read(_TRANSCRIPT_MAX_BYTES).decode("utf-8", "replace")
|
|
2133
|
+
except Exception as e:
|
|
2134
|
+
logger.debug(f"iTunes 文字稿下载失败: {e}")
|
|
2135
|
+
continue # 这份坏了不放弃整条节目,试下一份候选
|
|
2136
|
+
text = _transcript_to_text(raw, ctype)
|
|
2137
|
+
if text:
|
|
2138
|
+
return t_url, text
|
|
2139
|
+
return "", ""
|
|
2140
|
+
|
|
2141
|
+
@safe_search
|
|
2142
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
2143
|
+
q = (query or "").strip()
|
|
2144
|
+
if not q:
|
|
2145
|
+
return []
|
|
2146
|
+
to = _timeout or timeout
|
|
2147
|
+
want_podcast = bool(_PODCAST_HINT_RE.search(q))
|
|
2148
|
+
want_episode = want_podcast and bool(_PODCAST_EPISODE_RE.search(q))
|
|
2149
|
+
want_album = bool(_ALBUM_HINT_RE.search(q)) and not want_podcast
|
|
2150
|
+
# 去掉意图词,保留实体名
|
|
2151
|
+
term = _MEDIA_INTENT_WORD_RE.sub(" ", q)
|
|
2152
|
+
term = re.sub(r"\s+", " ", term).strip() or q
|
|
2153
|
+
has_zh = bool(re.search(r"[\u4e00-\u9fff]", term))
|
|
2154
|
+
country = "cn" if has_zh else ""
|
|
2155
|
+
params: dict[str, str] = {
|
|
2156
|
+
"term": term,
|
|
2157
|
+
"limit": str(min(max(n, 1), 25)),
|
|
2158
|
+
}
|
|
2159
|
+
if want_podcast:
|
|
2160
|
+
params["media"] = "podcast"
|
|
2161
|
+
params["entity"] = "podcastEpisode" if want_episode else "podcast"
|
|
2162
|
+
else:
|
|
2163
|
+
params["media"] = "music"
|
|
2164
|
+
if want_album:
|
|
2165
|
+
params["entity"] = "album"
|
|
2166
|
+
if country:
|
|
2167
|
+
params["country"] = country
|
|
2168
|
+
params["lang"] = "zh_cn"
|
|
2169
|
+
try:
|
|
2170
|
+
data = _get_json(f"{base}?{urllib.parse.urlencode(params)}", to)
|
|
2171
|
+
except Exception as e:
|
|
2172
|
+
logger.warning(f"iTunes 失败: {e}")
|
|
2173
|
+
return []
|
|
2174
|
+
items = data.get("results") or []
|
|
2175
|
+
# 相关度:实体名与 term 重叠。播客单集的标题里不含节目名,故比对
|
|
2176
|
+
# 「标题 + 摘要」——摘要首段就是《节目名》,否则搜节目名拿单集会全被滤掉。
|
|
2177
|
+
t_keys = {t.lower() for t in re.findall(r"[A-Za-z0-9\u4e00-\u9fff]{2,}", term)}
|
|
2178
|
+
base_score = 0.88 if want_album else 0.85
|
|
2179
|
+
out: list[dict[str, Any]] = []
|
|
2180
|
+
for it in items:
|
|
2181
|
+
if not isinstance(it, dict):
|
|
2182
|
+
continue
|
|
2183
|
+
is_episode = want_podcast and (
|
|
2184
|
+
str(it.get("wrapperType") or "").lower() == "podcastepisode"
|
|
2185
|
+
or (not it.get("wrapperType") and it.get("episodeUrl"))
|
|
2186
|
+
)
|
|
2187
|
+
if want_podcast and not is_episode:
|
|
2188
|
+
title = str(it.get("collectionName") or it.get("trackName") or "").strip()
|
|
2189
|
+
href = it.get("collectionViewUrl") or it.get("feedUrl") or ""
|
|
2190
|
+
count = it.get("trackCount")
|
|
2191
|
+
bits = ["播客"]
|
|
2192
|
+
if isinstance(count, int) and count > 0:
|
|
2193
|
+
bits.append(f"{count} 集")
|
|
2194
|
+
genre = str(it.get("primaryGenreName") or "").strip()
|
|
2195
|
+
if genre:
|
|
2196
|
+
bits.append(genre)
|
|
2197
|
+
updated = _itunes_date(it.get("releaseDate"))
|
|
2198
|
+
if updated:
|
|
2199
|
+
bits.append(f"更新 {updated}")
|
|
2200
|
+
row: dict[str, Any] = {
|
|
2201
|
+
"title": title,
|
|
2202
|
+
"url": str(href or "").strip(),
|
|
2203
|
+
"snippet": " · ".join(bits)[:300],
|
|
2204
|
+
"source": source_name,
|
|
2205
|
+
}
|
|
2206
|
+
if isinstance(count, int) and count > 0:
|
|
2207
|
+
row["episode_count"] = count
|
|
2208
|
+
# published_at 是既有的一等字段(--fields agent 与 --sort 都认它)。
|
|
2209
|
+
# 只给播客挂:音乐的 releaseDate 不在本次范围内,而挂上会改变
|
|
2210
|
+
# 音乐查询在 --sort newest 下的重排行为(releaseDate≠引擎相关序)。
|
|
2211
|
+
_published = str(it.get("releaseDate") or "").strip()
|
|
2212
|
+
if _published:
|
|
2213
|
+
row["published_at"] = _published[:64]
|
|
2214
|
+
elif is_episode:
|
|
2215
|
+
title = str(it.get("trackName") or "").strip()
|
|
2216
|
+
show = str(it.get("collectionName") or "").strip()
|
|
2217
|
+
bits = [f"《{show}》"] if show else []
|
|
2218
|
+
ms = it.get("trackTimeMillis")
|
|
2219
|
+
row = {
|
|
2220
|
+
"title": title,
|
|
2221
|
+
"url": str(it.get("trackViewUrl") or it.get("episodeUrl") or "").strip(),
|
|
2222
|
+
"snippet": "",
|
|
2223
|
+
"source": source_name,
|
|
2224
|
+
}
|
|
2225
|
+
if isinstance(ms, (int, float)) and ms > 0:
|
|
2226
|
+
mins = int(round(ms / 60000.0))
|
|
2227
|
+
row["duration_minutes"] = mins
|
|
2228
|
+
bits.append(f"时长 {mins} 分")
|
|
2229
|
+
pub_date = _itunes_date(it.get("releaseDate"))
|
|
2230
|
+
if pub_date:
|
|
2231
|
+
bits.append(pub_date)
|
|
2232
|
+
_published = str(it.get("releaseDate") or "").strip()
|
|
2233
|
+
if _published:
|
|
2234
|
+
row["published_at"] = _published[:64]
|
|
2235
|
+
row["snippet"] = " · ".join(bits)[:300]
|
|
2236
|
+
else:
|
|
2237
|
+
if want_album:
|
|
2238
|
+
title = str(it.get("collectionName") or it.get("trackName") or "").strip()
|
|
2239
|
+
href = it.get("collectionViewUrl") or it.get("trackViewUrl") or ""
|
|
2240
|
+
else:
|
|
2241
|
+
title = str(it.get("trackName") or it.get("collectionName") or "").strip()
|
|
2242
|
+
href = it.get("trackViewUrl") or it.get("collectionViewUrl") or ""
|
|
2243
|
+
row = {
|
|
2244
|
+
"title": title,
|
|
2245
|
+
"url": str(href or "").strip(),
|
|
2246
|
+
"snippet": str(it.get("artistName") or "").strip()[:300],
|
|
2247
|
+
"source": source_name,
|
|
2248
|
+
}
|
|
2249
|
+
if not row["title"] or not row["url"]:
|
|
2250
|
+
continue
|
|
2251
|
+
blob = f"{row['title']} {row['snippet']}".lower()
|
|
2252
|
+
if t_keys and not any(k in blob for k in t_keys):
|
|
2253
|
+
continue
|
|
2254
|
+
row["score"] = rank_score(base_score, len(out))
|
|
2255
|
+
out.append(row)
|
|
2256
|
+
if len(out) >= n:
|
|
2257
|
+
break
|
|
2258
|
+
# 节目级时长补在筛选之后:被滤掉的结果不该付二跳的代价
|
|
2259
|
+
if want_podcast and not want_episode:
|
|
2260
|
+
enriched = 0
|
|
2261
|
+
for it in items:
|
|
2262
|
+
if enriched >= _ITUNES_SHOW_ENRICH_MAX:
|
|
2263
|
+
break
|
|
2264
|
+
if not isinstance(it, dict):
|
|
2265
|
+
continue
|
|
2266
|
+
name = str(it.get("collectionName") or "").strip()
|
|
2267
|
+
target = next((r for r in out if r["title"] == name), None)
|
|
2268
|
+
if target is None or "duration_minutes" in target:
|
|
2269
|
+
continue
|
|
2270
|
+
mid, samples = _recent_episode_duration(it.get("collectionId"), country, to)
|
|
2271
|
+
if mid is not None:
|
|
2272
|
+
target["duration_minutes"] = mid
|
|
2273
|
+
target["snippet"] = f"{target['snippet']} · 近 {samples} 期中位 {mid} 分"
|
|
2274
|
+
enriched += 1
|
|
2275
|
+
# 文字稿与时长共用 enrich 配额(前 1 条节目):官方全文预览的
|
|
2276
|
+
# 信息密度远高于「N 集 · 类型」元数据行,有文字稿时 snippet
|
|
2277
|
+
# 拼接全文开头,episode_count/published_at 等结构字段不动
|
|
2278
|
+
feed_url = str(it.get("feedUrl") or "").strip()
|
|
2279
|
+
if feed_url:
|
|
2280
|
+
t_url, t_prev = _show_transcript(feed_url, to)
|
|
2281
|
+
if t_url:
|
|
2282
|
+
target["transcript_url"] = t_url
|
|
2283
|
+
if t_prev:
|
|
2284
|
+
target["snippet"] = f"{target['snippet']} | {t_prev}"[:300]
|
|
2285
|
+
enriched += 1
|
|
2286
|
+
return out
|
|
2287
|
+
return _engine
|
|
2288
|
+
|
|
2289
|
+
|
|
2290
|
+
# ── TheSportsDB 体育(免认证 test key=3) ────────────────────────────────────
|
|
2291
|
+
|
|
2292
|
+
_SPORTSDB_BASE = "https://www.thesportsdb.com/api/v1/json/3"
|
|
2293
|
+
_SPORTSDB_UA = "argo-search/2.6 (unified-search@local)"
|
|
2294
|
+
|
|
2295
|
+
# 高频中文球员/球队 → TheSportsDB 英文检索名(公开 API 基本只认拉丁文)
|
|
2296
|
+
_SPORTS_ZH_ALIASES: dict[str, str] = {
|
|
2297
|
+
"梅西": "Messi",
|
|
2298
|
+
"C罗": "Ronaldo",
|
|
2299
|
+
"C羅": "Ronaldo",
|
|
2300
|
+
"罗纳尔多": "Ronaldo",
|
|
2301
|
+
"Cristiano": "Ronaldo",
|
|
2302
|
+
"内马尔": "Neymar",
|
|
2303
|
+
"姆巴佩": "Mbappe",
|
|
2304
|
+
"哈兰德": "Haaland",
|
|
2305
|
+
# 单姓 Curry 会命中 Eddy Curry;中文「库里」默认指 Stephen Curry
|
|
2306
|
+
"库里": "Stephen Curry",
|
|
2307
|
+
"斯蒂芬库里": "Stephen Curry",
|
|
2308
|
+
"史蒂芬库里": "Stephen Curry",
|
|
2309
|
+
"詹姆斯": "LeBron James",
|
|
2310
|
+
"勒布朗": "LeBron",
|
|
2311
|
+
"杜兰特": "Durant",
|
|
2312
|
+
"科比": "Kobe",
|
|
2313
|
+
"乔丹": "Jordan",
|
|
2314
|
+
"姚明": "Yao Ming",
|
|
2315
|
+
"易建联": "Yi Jianlian",
|
|
2316
|
+
"皇马": "Real Madrid",
|
|
2317
|
+
"巴萨": "Barcelona",
|
|
2318
|
+
"曼联": "Manchester United",
|
|
2319
|
+
"曼城": "Manchester City",
|
|
2320
|
+
"湖人": "Lakers",
|
|
2321
|
+
"勇士": "Warriors",
|
|
2322
|
+
"凯尔特人": "Celtics",
|
|
2323
|
+
}
|
|
2324
|
+
|
|
2325
|
+
|
|
2326
|
+
def _sports_expand_aliases(query: str) -> list[str]:
|
|
2327
|
+
"""中文别名展开为英文检索变体。"""
|
|
2328
|
+
out: list[str] = []
|
|
2329
|
+
for zh, en in _SPORTS_ZH_ALIASES.items():
|
|
2330
|
+
if zh in query:
|
|
2331
|
+
if en not in out:
|
|
2332
|
+
out.append(en)
|
|
2333
|
+
return out
|
|
2334
|
+
|
|
2335
|
+
|
|
2336
|
+
def _build_thesportsdb_engine(spec: dict[str, Any]) -> Any:
|
|
2337
|
+
"""TheSportsDB 球员/球队/赛事搜索(api key=3 公开测试钥,免认证)。
|
|
2338
|
+
|
|
2339
|
+
并行查 searchplayers / searchteams / searchevents,合并去重。
|
|
2340
|
+
"""
|
|
2341
|
+
timeout = spec.get("timeout", 10)
|
|
2342
|
+
source_name = spec.get("_name", "thesportsdb")
|
|
2343
|
+
|
|
2344
|
+
def _fetch(path: str, params: dict[str, str], to: float) -> Any:
|
|
2345
|
+
url = f"{_SPORTSDB_BASE}/{path}?" + urllib.parse.urlencode(params)
|
|
2346
|
+
req = urllib.request.Request(
|
|
2347
|
+
url, headers={"User-Agent": _SPORTSDB_UA, "Accept": "application/json"},
|
|
2348
|
+
)
|
|
2349
|
+
with http_open(req, timeout=to, engine=spec.get("_name", "")) as resp:
|
|
2350
|
+
return json.loads(resp.read().decode("utf-8", "replace"))
|
|
2351
|
+
|
|
2352
|
+
@safe_search
|
|
2353
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
2354
|
+
q = (query or "").strip()
|
|
2355
|
+
if not q:
|
|
2356
|
+
return []
|
|
2357
|
+
to = _timeout or timeout
|
|
2358
|
+
# 多词查询(如 "NBA finals MVP 2024")整句常 0 命中 → 短语 + token 变体
|
|
2359
|
+
stop = {
|
|
2360
|
+
"finals", "final", "mvp", "vs", "the", "and", "of", "in", "for", "a", "an",
|
|
2361
|
+
"winner", "winners", "who", "won", "what", "which", "year", "years",
|
|
2362
|
+
"team", "club", "player", "coach", "manager", "roster", "squad",
|
|
2363
|
+
"决赛", "半决赛", "总冠军", "联赛", "球员", "俱乐部", "进球", "冠军",
|
|
2364
|
+
"球队", "球星", "效力", "夺冠", "卫冕", "助攻", "得分王",
|
|
2365
|
+
}
|
|
2366
|
+
tokens = re.findall(r"[A-Za-z\u4e00-\u9fff]{2,}", q)
|
|
2367
|
+
# 纯数字年号单独收集,用于 event 年份保持一致,不当主检索词
|
|
2368
|
+
# 注意:必须整年捕获;r"(19|20)\d{2}" 的 findall 只会返回 "20",
|
|
2369
|
+
# 导致 "20" in "2016-11-24" 恒真、年份过滤失效。
|
|
2370
|
+
years = re.findall(r"\b((?:19|20)\d{2})\b", q)
|
|
2371
|
+
meaningful = [t for t in tokens if t.lower() not in stop and not re.fullmatch(r"\d{4}", t)]
|
|
2372
|
+
|
|
2373
|
+
variants: list[str] = []
|
|
2374
|
+
# 中文别名优先(梅西→Messi)
|
|
2375
|
+
for alias in _sports_expand_aliases(q):
|
|
2376
|
+
if alias not in variants:
|
|
2377
|
+
variants.append(alias)
|
|
2378
|
+
# 双词短语优先(World Cup / LeBron James)
|
|
2379
|
+
for i in range(len(meaningful) - 1):
|
|
2380
|
+
phrase = f"{meaningful[i]} {meaningful[i + 1]}"
|
|
2381
|
+
if phrase not in variants:
|
|
2382
|
+
variants.append(phrase)
|
|
2383
|
+
# 整句(短查询,非纯中文)
|
|
2384
|
+
if len(q) <= 40 and q not in variants and re.search(r"[A-Za-z]", q):
|
|
2385
|
+
variants.insert(0, q)
|
|
2386
|
+
for t in meaningful:
|
|
2387
|
+
# 纯中文 token 对英文 API 无效,跳过(已由别名覆盖)
|
|
2388
|
+
if re.fullmatch(r"[\u4e00-\u9fff]+", t):
|
|
2389
|
+
continue
|
|
2390
|
+
if t not in variants:
|
|
2391
|
+
variants.append(t)
|
|
2392
|
+
if len(variants) >= 5:
|
|
2393
|
+
break
|
|
2394
|
+
if not variants:
|
|
2395
|
+
variants = [q]
|
|
2396
|
+
|
|
2397
|
+
# 意图:club/team/球队 → 优先 team/player;event 词 → 保留 event
|
|
2398
|
+
q_low = q.lower()
|
|
2399
|
+
prefer_entity = bool(re.search(
|
|
2400
|
+
r"(?i)(team|club|player|coach|球队|球星|俱乐部|效力|球员|roster|squad)", q,
|
|
2401
|
+
))
|
|
2402
|
+
prefer_event = bool(re.search(
|
|
2403
|
+
r"(?i)(finals?|mvp|cup|championship|winner|won|决赛|总冠军|世界杯|欧冠|季后赛)", q,
|
|
2404
|
+
)) and not prefer_entity
|
|
2405
|
+
|
|
2406
|
+
bucket: list[dict[str, Any]] = []
|
|
2407
|
+
seen: set[str] = set()
|
|
2408
|
+
|
|
2409
|
+
endpoints = (
|
|
2410
|
+
("searchplayers.php", "p", "player", "strPlayer", "idPlayer",
|
|
2411
|
+
lambda x: " · ".join(p for p in (
|
|
2412
|
+
x.get("strTeam") or "",
|
|
2413
|
+
x.get("strSport") or "",
|
|
2414
|
+
x.get("strPosition") or "",
|
|
2415
|
+
x.get("strNationality") or "",
|
|
2416
|
+
) if p)),
|
|
2417
|
+
("searchteams.php", "t", "team", "strTeam", "idTeam",
|
|
2418
|
+
lambda x: " · ".join(p for p in (
|
|
2419
|
+
x.get("strSport") or "",
|
|
2420
|
+
x.get("strLeague") or "",
|
|
2421
|
+
x.get("strCountry") or "",
|
|
2422
|
+
) if p)),
|
|
2423
|
+
("searchevents.php", "e", "event", "strEvent", "idEvent",
|
|
2424
|
+
lambda x: " · ".join(p for p in (
|
|
2425
|
+
x.get("strLeague") or "",
|
|
2426
|
+
x.get("strSport") or "",
|
|
2427
|
+
x.get("dateEvent") or x.get("strTimestamp") or "",
|
|
2428
|
+
) if p)),
|
|
2429
|
+
)
|
|
2430
|
+
list_keys = {
|
|
2431
|
+
"searchplayers.php": "player",
|
|
2432
|
+
"searchteams.php": "teams",
|
|
2433
|
+
"searchevents.php": "event",
|
|
2434
|
+
}
|
|
2435
|
+
url_tpl = {
|
|
2436
|
+
"player": "https://www.thesportsdb.com/player/{id}",
|
|
2437
|
+
"team": "https://www.thesportsdb.com/team/{id}",
|
|
2438
|
+
"event": "https://www.thesportsdb.com/event/{id}",
|
|
2439
|
+
}
|
|
2440
|
+
score_of = {"player": 0.86, "team": 0.84, "event": 0.82}
|
|
2441
|
+
# 实体问优先 player/team;赛事问放宽 event
|
|
2442
|
+
if prefer_entity:
|
|
2443
|
+
endpoints = endpoints[:2] + endpoints[2:] # 顺序已是 p,t,e
|
|
2444
|
+
elif prefer_event:
|
|
2445
|
+
endpoints = (endpoints[2], endpoints[0], endpoints[1])
|
|
2446
|
+
|
|
2447
|
+
q_token_set = {t.lower() for t in meaningful}
|
|
2448
|
+
|
|
2449
|
+
for v in variants:
|
|
2450
|
+
for path, param_k, kind, title_k, id_k, snip_fn in endpoints:
|
|
2451
|
+
try:
|
|
2452
|
+
data = _fetch(path, {param_k: v}, to)
|
|
2453
|
+
except Exception as e:
|
|
2454
|
+
logger.warning(f"TheSportsDB {path} 失败: {e}")
|
|
2455
|
+
continue
|
|
2456
|
+
items = data.get(list_keys[path]) if isinstance(data, dict) else None
|
|
2457
|
+
if not items:
|
|
2458
|
+
continue
|
|
2459
|
+
for x in items[: max(n, 5)]:
|
|
2460
|
+
if not isinstance(x, dict):
|
|
2461
|
+
continue
|
|
2462
|
+
title = x.get(title_k) or ""
|
|
2463
|
+
iid = str(x.get(id_k) or "")
|
|
2464
|
+
key = f"{kind}:{iid or title}"
|
|
2465
|
+
if not title or key in seen:
|
|
2466
|
+
continue
|
|
2467
|
+
# 相关度:标题/摘要 token 与查询重叠;event 尤其严格
|
|
2468
|
+
snip = snip_fn(x) or ""
|
|
2469
|
+
sport = str(x.get("strSport") or "")
|
|
2470
|
+
league = str(x.get("strLeague") or "")
|
|
2471
|
+
blob = f"{title} {snip} {sport} {league}".lower()
|
|
2472
|
+
t_tokens = set(re.findall(r"[a-z0-9\u4e00-\u9fff]{2,}", blob))
|
|
2473
|
+
overlap = len(q_token_set & t_tokens) if q_token_set else 1
|
|
2474
|
+
# 「世界杯/World Cup」默认足球;公开 API 常塞高尔夫/篮网等噪声 → 非足球一律丢
|
|
2475
|
+
world_cup_q = bool(re.search(r"(?i)world\s*cup|世界杯|fifa", q))
|
|
2476
|
+
if world_cup_q:
|
|
2477
|
+
if not re.search(r"(?i)soccer|football|fifa", f"{title} {sport} {league} {snip}"):
|
|
2478
|
+
continue
|
|
2479
|
+
if kind == "event":
|
|
2480
|
+
# 无实质重叠 → 丢弃噪声 event(如 World Cup → World Sand Greens)
|
|
2481
|
+
if q_token_set and overlap == 0:
|
|
2482
|
+
continue
|
|
2483
|
+
# 有年份且 API 给了日期:年份不对直接丢(避免 2022 问到 2016 Golf)
|
|
2484
|
+
date_s = str(x.get("dateEvent") or x.get("strTimestamp") or "")
|
|
2485
|
+
if years and date_s and not any(y in date_s for y in years):
|
|
2486
|
+
continue
|
|
2487
|
+
# 双词短语(如 World Cup)若在标题中,要求更高相关;单 token 过宽
|
|
2488
|
+
phrase_hit = any(
|
|
2489
|
+
len(p.split()) >= 2 and p.lower() in title.lower()
|
|
2490
|
+
for p in variants[:3]
|
|
2491
|
+
)
|
|
2492
|
+
if q_token_set and overlap < 2 and not phrase_hit:
|
|
2493
|
+
continue
|
|
2494
|
+
rel = 0.35 + 0.15 * overlap + (0.2 if phrase_hit else 0)
|
|
2495
|
+
if years and date_s and any(y in date_s for y in years):
|
|
2496
|
+
rel += 0.2
|
|
2497
|
+
if world_cup_q and re.search(r"(?i)soccer|football|fifa", blob):
|
|
2498
|
+
rel += 0.25
|
|
2499
|
+
else:
|
|
2500
|
+
# 赛事问勿用弱 token 沾上球员(Metta World Peace)
|
|
2501
|
+
if prefer_event and overlap < 2 and v.lower() not in title.lower():
|
|
2502
|
+
continue
|
|
2503
|
+
if world_cup_q and kind == "player" and not re.search(
|
|
2504
|
+
r"(?i)soccer|football", f"{sport} {snip}",
|
|
2505
|
+
):
|
|
2506
|
+
continue
|
|
2507
|
+
# player/team:变体本身已是名,重叠可放宽
|
|
2508
|
+
rel = 0.35 + 0.25 * overlap
|
|
2509
|
+
if v.lower() in title.lower():
|
|
2510
|
+
rel += 0.25
|
|
2511
|
+
# 全名变体(Stephen Curry)精确命中加权
|
|
2512
|
+
if " " in v and v.lower() == title.lower():
|
|
2513
|
+
rel += 0.3
|
|
2514
|
+
seen.add(key)
|
|
2515
|
+
href = url_tpl[kind].format(id=iid) if iid else ""
|
|
2516
|
+
base = score_of.get(kind, 0.8)
|
|
2517
|
+
bucket.append({
|
|
2518
|
+
"title": f"[{kind}] {title}",
|
|
2519
|
+
"url": href,
|
|
2520
|
+
"snippet": (snip_fn(x) or kind)[:300],
|
|
2521
|
+
"source": source_name,
|
|
2522
|
+
"score": min(0.95, base * 0.5 + rel),
|
|
2523
|
+
"_kind": kind,
|
|
2524
|
+
"_rel": rel,
|
|
2525
|
+
})
|
|
2526
|
+
# 已有高相关实体则早停
|
|
2527
|
+
if sum(1 for r in bucket if r.get("_rel", 0) >= 0.5) >= n:
|
|
2528
|
+
break
|
|
2529
|
+
|
|
2530
|
+
kind_rank = {"player": 0, "team": 1, "event": 2}
|
|
2531
|
+
if prefer_entity:
|
|
2532
|
+
kind_rank = {"player": 0, "team": 0, "event": 3}
|
|
2533
|
+
elif prefer_event:
|
|
2534
|
+
kind_rank = {"event": 0, "player": 1, "team": 2}
|
|
2535
|
+
|
|
2536
|
+
bucket.sort(key=lambda r: (
|
|
2537
|
+
kind_rank.get(r.get("_kind", ""), 9),
|
|
2538
|
+
-float(r.get("_rel", 0)),
|
|
2539
|
+
-float(r.get("score", 0)),
|
|
2540
|
+
))
|
|
2541
|
+
out = []
|
|
2542
|
+
for r in bucket[:n]:
|
|
2543
|
+
r.pop("_kind", None)
|
|
2544
|
+
r.pop("_rel", None)
|
|
2545
|
+
out.append(r)
|
|
2546
|
+
return out
|
|
2547
|
+
return _engine
|
|
2548
|
+
|
|
2549
|
+
|
|
2550
|
+
# ── GDELT 全球事件数据库 ─────────────────────────────────────────────────────
|
|
2551
|
+
|
|
2552
|
+
# ── OpenCorporates 全球公司注册 ──────────────────────────────────────────────
|
|
2553
|
+
|
|
2554
|
+
def _build_opencorporates_engine(spec: dict[str, Any]) -> Any:
|
|
2555
|
+
"""OpenCorporates 全球公司注册(api.opencorporates.com/v0.4/companies/search,免认证)。
|
|
2556
|
+
|
|
2557
|
+
公司尽调/反欺诈:公司名、注册号、司法辖区、成立日期、状态。
|
|
2558
|
+
argo 现有引擎无公司注册数据源,这是尽调类查询的核心空白。
|
|
2559
|
+
"""
|
|
2560
|
+
timeout = spec.get("timeout", 10)
|
|
2561
|
+
|
|
2562
|
+
@safe_search
|
|
2563
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
2564
|
+
to = _timeout or timeout
|
|
2565
|
+
q = query.strip()
|
|
2566
|
+
# 去常见修饰语,取核心公司名
|
|
2567
|
+
q2 = re.sub(r"(?i)\b(company|corporation|inc|limited|ltd|llc|公司|集团|有限|注册|查一下|查询)\b", " ", q)
|
|
2568
|
+
q2 = re.sub(r"\s+", " ", q2).strip() or q
|
|
2569
|
+
url = (
|
|
2570
|
+
"https://api.opencorporates.com/v0.4/companies/search"
|
|
2571
|
+
f"?q={urllib.parse.quote(q2)}&per_page={min(n, 10)}"
|
|
2572
|
+
)
|
|
2573
|
+
headers = {"User-Agent": "argo-search/2.6 (unified-search@local)", "Accept": "application/json"}
|
|
2574
|
+
try:
|
|
2575
|
+
with http_open(urllib.request.Request(url, headers=headers), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
2576
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
2577
|
+
except Exception:
|
|
2578
|
+
return []
|
|
2579
|
+
companies = ((data.get("results") or {}).get("companies")) or []
|
|
2580
|
+
results: list[dict[str, Any]] = []
|
|
2581
|
+
for _rk19, c in enumerate(companies[:n]):
|
|
2582
|
+
comp = c.get("company") or {}
|
|
2583
|
+
name = comp.get("name") or ""
|
|
2584
|
+
if not name:
|
|
2585
|
+
continue
|
|
2586
|
+
num = comp.get("company_number") or ""
|
|
2587
|
+
juris = comp.get("jurisdiction_code") or ""
|
|
2588
|
+
inc_date = (comp.get("incorporation_date") or "")[:10]
|
|
2589
|
+
status = comp.get("current_status") or ""
|
|
2590
|
+
cc = (comp.get("registered_address") or {}).get("country") or ""
|
|
2591
|
+
results.append({
|
|
2592
|
+
"title": name[:200],
|
|
2593
|
+
"url": comp.get("opencorporates_url") or f"https://opencorporates.com/companies/{juris}/{num}",
|
|
2594
|
+
"snippet": (
|
|
2595
|
+
f"{juris}{'/'+num if num else ''}"
|
|
2596
|
+
f"{' · 成立 '+inc_date if inc_date else ''}"
|
|
2597
|
+
f"{' · 状态 '+status if status else ''}"
|
|
2598
|
+
f"{' · '+cc if cc else ''}"
|
|
2599
|
+
)[:300],
|
|
2600
|
+
"source": "opencorporates",
|
|
2601
|
+
"score": rank_score(0.8, _rk19),
|
|
2602
|
+
"company_number": num,
|
|
2603
|
+
"jurisdiction": juris,
|
|
2604
|
+
"incorporation_date": inc_date,
|
|
2605
|
+
"status": status,
|
|
2606
|
+
})
|
|
2607
|
+
return results
|
|
2608
|
+
return _engine
|
|
2609
|
+
|
|
2610
|
+
|
|
2611
|
+
# ── Google Patents 专利搜索 ──────────────────────────────────────────────────
|
|
2612
|
+
|
|
2613
|
+
def _build_google_patents_engine(spec: dict[str, Any]) -> Any:
|
|
2614
|
+
"""Google Patents 专利搜索(patents.google.com/xhr/query,免认证)。
|
|
2615
|
+
|
|
2616
|
+
技术尽调/竞品分析:专利标题、公开号、公开日期、申请人。
|
|
2617
|
+
用 patents.google.com 的公开 XHR 接口,免 key、返回 JSON。
|
|
2618
|
+
argo 现有引擎无专利数据源,这是技术尽调的核心空白。
|
|
2619
|
+
"""
|
|
2620
|
+
timeout = spec.get("timeout", 10)
|
|
2621
|
+
|
|
2622
|
+
@safe_search
|
|
2623
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
2624
|
+
to = _timeout or timeout
|
|
2625
|
+
q = query.strip()
|
|
2626
|
+
# 去常见修饰语
|
|
2627
|
+
q2 = re.sub(r"(?i)\b(patent|专利|查询|搜索|检索)\b", " ", q)
|
|
2628
|
+
q2 = re.sub(r"\s+", " ", q2).strip() or q
|
|
2629
|
+
url = "https://patents.google.com/xhr/query"
|
|
2630
|
+
params = {
|
|
2631
|
+
"url": f"q={urllib.parse.quote(q2)}",
|
|
2632
|
+
"exp": "",
|
|
2633
|
+
}
|
|
2634
|
+
full_url = url + "?" + urllib.parse.urlencode(params)
|
|
2635
|
+
headers = {
|
|
2636
|
+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
|
|
2637
|
+
"Accept": "application/json",
|
|
2638
|
+
}
|
|
2639
|
+
try:
|
|
2640
|
+
with http_open(urllib.request.Request(full_url, headers=headers), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
2641
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
2642
|
+
except Exception:
|
|
2643
|
+
return []
|
|
2644
|
+
results: list[dict[str, Any]] = []
|
|
2645
|
+
try:
|
|
2646
|
+
cluster = (data.get("results") or {}).get("cluster") or []
|
|
2647
|
+
for c in cluster:
|
|
2648
|
+
for r in (c.get("result") or []):
|
|
2649
|
+
if len(results) >= n:
|
|
2650
|
+
break
|
|
2651
|
+
pat = r.get("patent") or {}
|
|
2652
|
+
title = pat.get("title") or ""
|
|
2653
|
+
if not title:
|
|
2654
|
+
continue
|
|
2655
|
+
pub_date = pat.get("publication_date") or ""
|
|
2656
|
+
pub_num = pat.get("publication_number") or ""
|
|
2657
|
+
assignee = (pat.get("assignee") or [{}])[0].get("name") if isinstance(pat.get("assignee"), list) else ""
|
|
2658
|
+
pub_id = pat.get("publication_number") or pub_num or ""
|
|
2659
|
+
results.append({
|
|
2660
|
+
"title": title[:200],
|
|
2661
|
+
"url": f"https://patents.google.com/patent/{pub_id}" if pub_id else "",
|
|
2662
|
+
"snippet": (r.get("snippet") or "")[:300],
|
|
2663
|
+
"source": "google_patents",
|
|
2664
|
+
"score": rank_score(0.8, len(results)),
|
|
2665
|
+
"publication_date": pub_date,
|
|
2666
|
+
"publication_number": pub_num,
|
|
2667
|
+
"assignee": assignee,
|
|
2668
|
+
})
|
|
2669
|
+
except Exception:
|
|
2670
|
+
pass
|
|
2671
|
+
return results
|
|
2672
|
+
return _engine
|
|
2673
|
+
|
|
2674
|
+
|
|
2675
|
+
# ── Marginalia 独立索引引擎 ──────────────────────────────────────────────────
|
|
2676
|
+
|
|
2677
|
+
def _build_marginalia_engine(spec: dict[str, Any]) -> Any:
|
|
2678
|
+
"""Marginalia 独立爬虫索引搜索(api.marginalia.nu,免认证 JSON)。
|
|
2679
|
+
|
|
2680
|
+
非大厂代理的独立索引,专挖长尾非商业页面;JSON 直出。
|
|
2681
|
+
"""
|
|
2682
|
+
timeout = spec.get("timeout", 10)
|
|
2683
|
+
|
|
2684
|
+
@safe_search
|
|
2685
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
2686
|
+
import urllib.parse as up
|
|
2687
|
+
to = _timeout or timeout
|
|
2688
|
+
url = f"https://api.marginalia.nu/public/search/{up.quote(query)}?count={min(n, 10)}"
|
|
2689
|
+
try:
|
|
2690
|
+
with http_open(urllib.request.Request(
|
|
2691
|
+
url, headers={"User-Agent": "argo-search/2.4 (unified-search@local)"}), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
2692
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
2693
|
+
except Exception as e:
|
|
2694
|
+
logger.warning(f"Marginalia 失败: {e}")
|
|
2695
|
+
return []
|
|
2696
|
+
results = []
|
|
2697
|
+
for _rk20, r in enumerate((data.get("results") or [])[:n]):
|
|
2698
|
+
title = r.get("title", "")
|
|
2699
|
+
url_ = r.get("url", "")
|
|
2700
|
+
if not title and not url_:
|
|
2701
|
+
continue
|
|
2702
|
+
results.append({
|
|
2703
|
+
"title": title,
|
|
2704
|
+
"url": url_,
|
|
2705
|
+
"snippet": r.get("description", "")[:300],
|
|
2706
|
+
"source": "marginalia",
|
|
2707
|
+
"score": rank_score(0.7, _rk20),
|
|
2708
|
+
})
|
|
2709
|
+
return results
|
|
2710
|
+
return _engine
|
|
2711
|
+
|
|
2712
|
+
|
|
2713
|
+
# ── Wiby 手工网页索引引擎 ────────────────────────────────────────────────────
|
|
2714
|
+
|
|
2715
|
+
def _build_wiby_engine(spec: dict[str, Any]) -> Any:
|
|
2716
|
+
"""Wiby 老式手工网页索引搜索(wiby.me/json,免认证,注意大写键)。
|
|
2717
|
+
|
|
2718
|
+
专收非商业化内容的老式索引,与主流引擎互补。
|
|
2719
|
+
"""
|
|
2720
|
+
timeout = spec.get("timeout", 10)
|
|
2721
|
+
|
|
2722
|
+
@safe_search
|
|
2723
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
2724
|
+
import urllib.parse as up
|
|
2725
|
+
to = _timeout or timeout
|
|
2726
|
+
url = f"https://wiby.me/json/?q={up.quote(query)}"
|
|
2727
|
+
try:
|
|
2728
|
+
with http_open(urllib.request.Request(
|
|
2729
|
+
url, headers={"User-Agent": "argo-search/2.4 (unified-search@local)"}), timeout=to, engine=spec.get("_name", "")) as resp:
|
|
2730
|
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
|
2731
|
+
except Exception as e:
|
|
2732
|
+
logger.warning(f"Wiby 失败: {e}")
|
|
2733
|
+
return []
|
|
2734
|
+
results = []
|
|
2735
|
+
for _rk21, r in enumerate((data if isinstance(data, list) else [])[:n]):
|
|
2736
|
+
if not isinstance(r, dict):
|
|
2737
|
+
continue
|
|
2738
|
+
title = r.get("Title", r.get("title", ""))
|
|
2739
|
+
url_ = r.get("URL", r.get("url", ""))
|
|
2740
|
+
if not title and not url_:
|
|
2741
|
+
continue
|
|
2742
|
+
results.append({
|
|
2743
|
+
"title": title,
|
|
2744
|
+
"url": url_,
|
|
2745
|
+
"snippet": r.get("Snippet", r.get("Description", ""))[:300],
|
|
2746
|
+
"source": "wiby",
|
|
2747
|
+
"score": rank_score(0.7, _rk21),
|
|
2748
|
+
})
|
|
2749
|
+
return results
|
|
2750
|
+
return _engine
|
|
2751
|
+
|
|
2752
|
+
|
|
2753
|
+
# ── bioRxiv / medRxiv 预印本引擎 ─────────────────────────────────────────────
|
|
2754
|
+
|
|
2755
|
+
# DOI 提取:10. 开头的标准 DOI(bioRxiv DOI 形态 10.1101/2024.01.02.123456)
|
|
2756
|
+
_PREPRINT_DOI_RE = re.compile(r"10\.\d{4,9}/[^\s\"',。;)((]+", re.I)
|
|
2757
|
+
|
|
2758
|
+
|
|
2759
|
+
def _build_biorxiv_engine(spec: dict[str, Any]) -> Any:
|
|
2760
|
+
"""bioRxiv / medRxiv 预印本引擎(API 免认证)。
|
|
2761
|
+
|
|
2762
|
+
bioRxiv API 没有关键词搜索端点——只有两种查询语义是诚实的:
|
|
2763
|
+
① DOI(10.xxxx/… 或 bioRxiv/medRxiv 链接)→ 取单篇详情(标题/作者/摘要)
|
|
2764
|
+
② 「最新 / recent」或空 → 最近 3 天全量列表
|
|
2765
|
+
关键词查询诚实返回空(不伪造结果);关键词发现交给 openalex /
|
|
2766
|
+
semantic_scholar,本引擎负责预印本原文入口与时效性列表。
|
|
2767
|
+
medRxiv 按 DOI 自动路由到 medrxiv 子库。
|
|
2768
|
+
"""
|
|
2769
|
+
timeout = spec.get("timeout", 12)
|
|
2770
|
+
|
|
2771
|
+
@safe_search
|
|
2772
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
2773
|
+
from datetime import date, timedelta
|
|
2774
|
+
to = _timeout or timeout
|
|
2775
|
+
from engines_base import _http_get_raw
|
|
2776
|
+
q = (query or "").strip()
|
|
2777
|
+
m = _PREPRINT_DOI_RE.search(q)
|
|
2778
|
+
headers = {"Accept": "application/json"}
|
|
2779
|
+
if m:
|
|
2780
|
+
doi = m.group(0).rstrip(".")
|
|
2781
|
+
server = "medrxiv" if ("medrxiv" in q.lower()) else "biorxiv"
|
|
2782
|
+
url = f"https://api.biorxiv.org/details/{server}/{doi}"
|
|
2783
|
+
raw = _http_get_raw(url, headers, to, engine="biorxiv")
|
|
2784
|
+
if not raw:
|
|
2785
|
+
return []
|
|
2786
|
+
try:
|
|
2787
|
+
data = json.loads(raw)
|
|
2788
|
+
except json.JSONDecodeError:
|
|
2789
|
+
return []
|
|
2790
|
+
coll = data.get("collection") or []
|
|
2791
|
+
items = coll
|
|
2792
|
+
else:
|
|
2793
|
+
# 时效模式:最近 3 天(含「最新/recent」触发词;关键词一律走时效模式
|
|
2794
|
+
# 的如实空列表——API 无搜索端点,不伪造)
|
|
2795
|
+
end = date.today()
|
|
2796
|
+
start = end - timedelta(days=3)
|
|
2797
|
+
url = (f"https://api.biorxiv.org/details/biorxiv/"
|
|
2798
|
+
f"{start.isoformat()}/{end.isoformat()}/0")
|
|
2799
|
+
raw = _http_get_raw(url, headers, to, engine="biorxiv")
|
|
2800
|
+
if not raw:
|
|
2801
|
+
return []
|
|
2802
|
+
try:
|
|
2803
|
+
data = json.loads(raw)
|
|
2804
|
+
except json.JSONDecodeError:
|
|
2805
|
+
return []
|
|
2806
|
+
coll = data.get("collection") or []
|
|
2807
|
+
# 时效模式下按查询词做本地相关过滤;纯触发词(最新/recent)放行
|
|
2808
|
+
kws = [w for w in q.lower().split() if w not in ("最新", "recent", "预印本")]
|
|
2809
|
+
if kws:
|
|
2810
|
+
coll = [c for c in coll
|
|
2811
|
+
if any(kw in (str(c.get("title") or "") + str(c.get("abstract") or "")).lower()
|
|
2812
|
+
for kw in kws)]
|
|
2813
|
+
items = coll[:n]
|
|
2814
|
+
|
|
2815
|
+
results = []
|
|
2816
|
+
for c in items:
|
|
2817
|
+
doi = str(c.get("doi") or "")
|
|
2818
|
+
results.append({
|
|
2819
|
+
"title": str(c.get("title") or "")[:120],
|
|
2820
|
+
"url": f"https://doi.org/{doi}" if doi else "https://www.biorxiv.org/",
|
|
2821
|
+
"snippet": f"{str(c.get('authors') or '')[:80]} | {str(c.get('abstract') or '')[:220]}",
|
|
2822
|
+
"source": "biorxiv",
|
|
2823
|
+
"score": 0.9 if m else 0.75,
|
|
2824
|
+
"published_at": str(c.get("date") or "")[:10],
|
|
2825
|
+
})
|
|
2826
|
+
return results
|
|
2827
|
+
return _engine
|
|
2828
|
+
|
|
2829
|
+
|
|
2830
|
+
# ── UN Comtrade 双边贸易引擎 ─────────────────────────────────────────────────
|
|
2831
|
+
|
|
2832
|
+
# 报告国词表:查询里的国家名 → Comtrade reporterCode。
|
|
2833
|
+
# 取主要经济体 30+;未收录国家诚实返回空(代码表不硬猜)。
|
|
2834
|
+
_COMTRADE_REPORTERS = {
|
|
2835
|
+
"中国": 156, "美国": 842, "日本": 392, "德国": 276, "韩国": 410,
|
|
2836
|
+
"印度": 699, "英国": 826, "法国": 251, "意大利": 380, "荷兰": 528,
|
|
2837
|
+
"俄罗斯": 643, "巴西": 76, "澳大利亚": 36, "加拿大": 124, "墨西哥": 484,
|
|
2838
|
+
"印尼": 360, "印度尼西亚": 360, "泰国": 764, "越南": 704, "马来西亚": 458,
|
|
2839
|
+
"新加坡": 702, "土耳其": 792, "西班牙": 724, "波兰": 616, "瑞士": 756,
|
|
2840
|
+
"瑞典": 752, "阿联酋": 784, "沙特": 682, "南非": 710, "阿根廷": 32,
|
|
2841
|
+
"智利": 152, "菲律宾": 608, "香港": 344, "挪威": 578, "丹麦": 208,
|
|
2842
|
+
}
|
|
2843
|
+
|
|
2844
|
+
|
|
2845
|
+
def _build_un_comtrade_engine(spec: dict[str, Any]) -> Any:
|
|
2846
|
+
"""UN Comtrade 双边商品贸易引擎(preview 端点免 key)。
|
|
2847
|
+
|
|
2848
|
+
查询形态:「<国家> <HS 码> [年份] [出口]」,如「中国 8541」「美国 8542
|
|
2849
|
+
2024 出口」。未提供国家或 HS 码时诚实返回空——参数解析不做猜测。
|
|
2850
|
+
preview 端点返回代码与数值,不含描述字典(reporter/partner/cmd 的
|
|
2851
|
+
Desc 字段为 null),故标题用查询词原文回显、金额取 primaryValue。
|
|
2852
|
+
每条结果附 comtradeplus 网页链接可核验同一数据。
|
|
2853
|
+
"""
|
|
2854
|
+
timeout = spec.get("timeout", 15)
|
|
2855
|
+
|
|
2856
|
+
@safe_search
|
|
2857
|
+
def _engine(query: str, n: int = 5, _timeout: float | None = None, **kwargs) -> list[dict[str, Any]]:
|
|
2858
|
+
import urllib.parse as up
|
|
2859
|
+
to = _timeout or timeout
|
|
2860
|
+
q = (query or "").strip()
|
|
2861
|
+
reporter = None
|
|
2862
|
+
reporter_name = ""
|
|
2863
|
+
for name, code in _COMTRADE_REPORTERS.items():
|
|
2864
|
+
if name in q:
|
|
2865
|
+
reporter, reporter_name = code, name
|
|
2866
|
+
break
|
|
2867
|
+
if reporter is None:
|
|
2868
|
+
return []
|
|
2869
|
+
year_m = re.search(r"\b(20[0-2]\d)\b", q)
|
|
2870
|
+
period = year_m.group(1) if year_m else "2023"
|
|
2871
|
+
flow = "X" if ("出口" in q or "export" in q.lower()) else "M"
|
|
2872
|
+
flow_name = "出口" if flow == "X" else "进口"
|
|
2873
|
+
cmd = ""
|
|
2874
|
+
for tok in re.findall(r"\d{4,6}", q):
|
|
2875
|
+
if tok == period:
|
|
2876
|
+
continue
|
|
2877
|
+
cmd = tok
|
|
2878
|
+
break
|
|
2879
|
+
if not cmd:
|
|
2880
|
+
return []
|
|
2881
|
+
params = {
|
|
2882
|
+
"reporterCode": reporter,
|
|
2883
|
+
"period": period,
|
|
2884
|
+
"cmdCode": cmd,
|
|
2885
|
+
"flowCode": flow,
|
|
2886
|
+
}
|
|
2887
|
+
url = ("https://comtradeapi.un.org/public/v1/preview/C/A/HS?"
|
|
2888
|
+
+ up.urlencode(params))
|
|
2889
|
+
from engines_base import _http_get_raw
|
|
2890
|
+
raw = _http_get_raw(url, {"Accept": "application/json"}, to,
|
|
2891
|
+
engine="un_comtrade")
|
|
2892
|
+
if not raw:
|
|
2893
|
+
return []
|
|
2894
|
+
try:
|
|
2895
|
+
data = json.loads(raw)
|
|
2896
|
+
except json.JSONDecodeError:
|
|
2897
|
+
return []
|
|
2898
|
+
rows = data.get("data") or []
|
|
2899
|
+
if not rows:
|
|
2900
|
+
return []
|
|
2901
|
+
rows.sort(key=lambda r: float(r.get("primaryValue") or 0), reverse=True)
|
|
2902
|
+
results = []
|
|
2903
|
+
for r in rows[:n]:
|
|
2904
|
+
partner = r.get("partnerCode")
|
|
2905
|
+
value = r.get("primaryValue")
|
|
2906
|
+
value_str = f"{float(value):,.0f} USD" if value else "金额未披露"
|
|
2907
|
+
# title/url 均带伙伴维度:融合层按 title+url 去重,同一伙伴维度
|
|
2908
|
+
# 才算重复;URL 带 PartnerAreas 使每条结果可独立核验
|
|
2909
|
+
partner_url = (
|
|
2910
|
+
"https://comtradeplus.un.org/TradeFlow?Frequency=A&Classification=HS"
|
|
2911
|
+
f"&Commodities={cmd}&Flow={flow}&Reporters={reporter}"
|
|
2912
|
+
f"&PartnerAreas={partner}&Period={period}")
|
|
2913
|
+
label = "全球合计" if partner in (0, 1) else f"伙伴 {partner}"
|
|
2914
|
+
results.append({
|
|
2915
|
+
"title": f"{reporter_name} {flow_name} HS{cmd} {period} → {label}",
|
|
2916
|
+
"url": partner_url,
|
|
2917
|
+
"snippet": value_str + (f" | 净重 {r.get('netWgt')}" if r.get('netWgt') else ""),
|
|
2918
|
+
"source": "un_comtrade",
|
|
2919
|
+
"score": 0.95 if partner in (0, 1) else 0.75,
|
|
2920
|
+
"published_at": f"{period}-12-31",
|
|
2921
|
+
})
|
|
2922
|
+
return results
|
|
2923
|
+
return _engine
|