argo-search 1.0.1 → 2.9.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (463) hide show
  1. package/LICENSE +201 -21
  2. package/README.en.md +822 -0
  3. package/README.es.md +864 -0
  4. package/README.ja.md +864 -0
  5. package/README.ko.md +864 -0
  6. package/README.md +867 -320
  7. package/SKILL.md +135 -0
  8. package/assets/readme/hero.svg +121 -0
  9. package/assets/readme/made-with-beautify.svg +10 -0
  10. package/assets/readme/proof-routes.svg +73 -0
  11. package/assets/readme/settings-keys.png +0 -0
  12. package/assets/readme/settings-tools.png +0 -0
  13. package/assets/readme/why-better.svg +104 -0
  14. package/assets/readme/workflow.svg +83 -0
  15. package/backends/domain_profiles.json +2308 -96
  16. package/backends/engine_registry.yaml +3314 -503
  17. package/backends/query_synonyms_cn.json +35 -0
  18. package/backends/quota_profiles.json +2409 -161
  19. package/backends/source_types_cn.json +83 -0
  20. package/backends/text_lexicon.json +44 -0
  21. package/bin/argo +703 -0
  22. package/bin/argo.js +71 -39
  23. package/config.yaml +4976 -363
  24. package/cordis.patch.yml +21 -0
  25. package/data/countries.json +40 -0
  26. package/data/lang_matrix/lang_capability.json +842 -0
  27. package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
  28. package/data/regions_cn.json +1 -0
  29. package/docs/ADDING_NEW_ENGINE.md +184 -0
  30. package/docs/ARGO_INTRO.md +124 -0
  31. package/docs/DATA_SOURCE_GAPS.md +81 -0
  32. package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
  33. package/docs/ENGINE_CATALOG.md +564 -0
  34. package/docs/MCP_SETUP.md +106 -0
  35. package/docs/PLATFORM_PATHS.md +111 -0
  36. package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
  37. package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
  38. package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
  39. package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
  40. package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
  41. package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
  42. package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
  43. package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
  44. package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
  45. package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
  46. package/docs/SEARCH_ARCHIVE.md +115 -0
  47. package/docs/assets/architecture.svg +104 -0
  48. package/docs/assets/hero.svg +98 -0
  49. package/docs/engines/bangumi.md +27 -0
  50. package/docs/engines/biorxiv.md +27 -0
  51. package/docs/engines/clawhub.md +27 -0
  52. package/docs/engines/crt_sh.md +27 -0
  53. package/docs/engines/datacite.md +27 -0
  54. package/docs/engines/deps_dev.md +27 -0
  55. package/docs/engines/douban_movie.md +27 -0
  56. package/docs/engines/endoflife.md +27 -0
  57. package/docs/engines/flk_law.md +27 -0
  58. package/docs/engines/fxtwitter.md +27 -0
  59. package/docs/engines/google_news.md +27 -0
  60. package/docs/engines/gov_regulations.md +27 -0
  61. package/docs/engines/guqian.md +27 -0
  62. package/docs/engines/hackernews.md +27 -0
  63. package/docs/engines/iplant.md +27 -0
  64. package/docs/engines/keenable.md +27 -0
  65. package/docs/engines/met_museum.md +27 -0
  66. package/docs/engines/nasa_images.md +27 -0
  67. package/docs/engines/netease_music.md +27 -0
  68. package/docs/engines/npm.md +27 -0
  69. package/docs/engines/nvd.md +27 -0
  70. package/docs/engines/openreview.md +27 -0
  71. package/docs/engines/openstd.md +27 -0
  72. package/docs/engines/parallel_free.md +27 -0
  73. package/docs/engines/people_daily.md +27 -0
  74. package/docs/engines/realtime_index.md +27 -0
  75. package/docs/engines/sec_edgar.md +27 -0
  76. package/docs/engines/seltz.md +27 -0
  77. package/docs/engines/skillsmp.md +27 -0
  78. package/docs/engines/sspai.md +27 -0
  79. package/docs/engines/std_samr.md +27 -0
  80. package/docs/engines/train.md +27 -0
  81. package/docs/engines/un_comtrade.md +27 -0
  82. package/docs/engines/wikisource.md +27 -0
  83. package/docs/engines/zdic.md +27 -0
  84. package/docs/engines/zenodo.md +27 -0
  85. package/docs/engines/zhihu_hot_app.md +27 -0
  86. package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
  87. package/engines/README.md +10 -0
  88. package/engines/_template_cli.yaml +39 -0
  89. package/engines/_template_http.yaml +37 -0
  90. package/engines/plugins/_template_plugin.py +50 -0
  91. package/engines/specs/_cisa_kev.yaml +54 -0
  92. package/engines/specs/_osv.yaml +63 -0
  93. package/engines/specs/aljazeera.yaml +43 -0
  94. package/engines/specs/antara.yaml +43 -0
  95. package/engines/specs/artic.yaml +20 -0
  96. package/engines/specs/aviation_weather.yaml +39 -0
  97. package/engines/specs/bangumi.yaml +21 -0
  98. package/engines/specs/biorxiv.yaml +24 -0
  99. package/engines/specs/carbon_intensity.yaml +32 -0
  100. package/engines/specs/cinii.yaml +44 -0
  101. package/engines/specs/clawhub.yaml +49 -0
  102. package/engines/specs/cleveland.yaml +19 -0
  103. package/engines/specs/cn_ai_news.yaml +43 -0
  104. package/engines/specs/core.yaml +35 -0
  105. package/engines/specs/crt_sh.yaml +38 -0
  106. package/engines/specs/ct24.yaml +43 -0
  107. package/engines/specs/datacite.yaml +27 -0
  108. package/engines/specs/datagov_uk.yaml +41 -0
  109. package/engines/specs/ddgs_images.yaml +45 -0
  110. package/engines/specs/deepseek_search.yaml +74 -0
  111. package/engines/specs/deezer.yaml +20 -0
  112. package/engines/specs/deps_dev.yaml +21 -0
  113. package/engines/specs/digup.yaml +36 -0
  114. package/engines/specs/doi.yaml +39 -0
  115. package/engines/specs/douban_movie.yaml +22 -0
  116. package/engines/specs/dw_news.yaml +47 -0
  117. package/engines/specs/egov_law.yaml +21 -0
  118. package/engines/specs/elpais.yaml +43 -0
  119. package/engines/specs/endoflife.yaml +21 -0
  120. package/engines/specs/energy_charts.yaml +21 -0
  121. package/engines/specs/factcheck_org.yaml +52 -0
  122. package/engines/specs/faz.yaml +43 -0
  123. package/engines/specs/federal_register.yaml +57 -0
  124. package/engines/specs/firecrawl.yaml +53 -0
  125. package/engines/specs/flk_law.yaml +24 -0
  126. package/engines/specs/folha.yaml +43 -0
  127. package/engines/specs/france24.yaml +44 -0
  128. package/engines/specs/full_fact.yaml +47 -0
  129. package/engines/specs/fxtwitter.yaml +31 -0
  130. package/engines/specs/gbfs_nyc.yaml +32 -0
  131. package/engines/specs/gdacs.yaml +21 -0
  132. package/engines/specs/gdelt.yaml +64 -0
  133. package/engines/specs/google_news.yaml +22 -0
  134. package/engines/specs/gov_regulations.yaml +38 -0
  135. package/engines/specs/guardian_rss.yaml +59 -0
  136. package/engines/specs/hurriyet.yaml +43 -0
  137. package/engines/specs/iplant.yaml +25 -0
  138. package/engines/specs/jikan.yaml +24 -0
  139. package/engines/specs/jolpica.yaml +20 -0
  140. package/engines/specs/k10plus.yaml +20 -0
  141. package/engines/specs/keenable.yaml +44 -0
  142. package/engines/specs/lefigaro.yaml +43 -0
  143. package/engines/specs/listenbrainz.yaml +19 -0
  144. package/engines/specs/matichon.yaml +43 -0
  145. package/engines/specs/met_museum.yaml +21 -0
  146. package/engines/specs/nasa_images.yaml +45 -0
  147. package/engines/specs/netease_music.yaml +38 -0
  148. package/engines/specs/nhk.yaml +43 -0
  149. package/engines/specs/nhtsa_vpic.yaml +20 -0
  150. package/engines/specs/noaa_swpc.yaml +20 -0
  151. package/engines/specs/nvd.yaml +38 -0
  152. package/engines/specs/obis.yaml +20 -0
  153. package/engines/specs/opencitations.yaml +52 -0
  154. package/engines/specs/openf1.yaml +21 -0
  155. package/engines/specs/openfoodfacts.yaml +45 -0
  156. package/engines/specs/openligadb.yaml +20 -0
  157. package/engines/specs/openreview.yaml +37 -0
  158. package/engines/specs/openstd.yaml +21 -0
  159. package/engines/specs/people_daily.yaml +22 -0
  160. package/engines/specs/realtime_index.yaml +34 -0
  161. package/engines/specs/ror.yaml +20 -0
  162. package/engines/specs/satnogs.yaml +20 -0
  163. package/engines/specs/sec_edgar.yaml +46 -0
  164. package/engines/specs/skillsmp.yaml +50 -0
  165. package/engines/specs/sogou_weixin_browser.yaml +30 -0
  166. package/engines/specs/soilgrids.yaml +25 -0
  167. package/engines/specs/sspai.yaml +36 -0
  168. package/engines/specs/stackexchange.yaml +46 -0
  169. package/engines/specs/std_samr.yaml +21 -0
  170. package/engines/specs/tass.yaml +43 -0
  171. package/engines/specs/tle_mirror.yaml +21 -0
  172. package/engines/specs/train.yaml +37 -0
  173. package/engines/specs/tvmaze.yaml +20 -0
  174. package/engines/specs/twitter_syndication.yaml +31 -0
  175. package/engines/specs/un_comtrade.yaml +23 -0
  176. package/engines/specs/unpaywall.yaml +54 -0
  177. package/engines/specs/vnpress.yaml +43 -0
  178. package/engines/specs/weather.yaml +32 -0
  179. package/engines/specs/who_don.yaml +21 -0
  180. package/engines/specs/who_gho.yaml +20 -0
  181. package/engines/specs/wikimedia_commons.yaml +34 -0
  182. package/engines/specs/wikisource.yaml +22 -0
  183. package/engines/specs/worms.yaml +19 -0
  184. package/engines/specs/x_trending.yaml +34 -0
  185. package/engines/specs/yna.yaml +43 -0
  186. package/engines/specs/ynet.yaml +43 -0
  187. package/engines/specs/zbmath.yaml +40 -0
  188. package/engines/specs/zdic.yaml +22 -0
  189. package/engines/specs/zenodo.yaml +27 -0
  190. package/engines/specs/zhihu_hot_app.yaml +35 -0
  191. package/mcp/clients.yaml +74 -0
  192. package/package.json +51 -12
  193. package/packages/dsh-plugin/README.md +134 -0
  194. package/packages/dsh-plugin/cordis.patch.yml +48 -0
  195. package/packages/dsh-plugin/dsh/client.js +1244 -0
  196. package/packages/dsh-plugin/dsh/index.js +2290 -0
  197. package/packages/dsh-plugin/dsh/native-tools.mjs +743 -0
  198. package/packages/dsh-plugin/package.json +38 -0
  199. package/packages/dsh-plugin/test/client.test.js +232 -0
  200. package/packages/dsh-plugin/test/config.test.js +30 -0
  201. package/packages/dsh-plugin/test/index.test.js +1338 -0
  202. package/references/query-templates.md +96 -0
  203. package/references/query-templates.yaml +118 -0
  204. package/scripts/ab_eval.py +219 -0
  205. package/scripts/ab_eval_p0p1.py +266 -0
  206. package/scripts/adaptive.py +271 -70
  207. package/scripts/answer.py +154 -0
  208. package/scripts/archive_run.py +591 -0
  209. package/scripts/argo_engine_registry.py +172 -0
  210. package/scripts/argo_paths.py +728 -0
  211. package/scripts/article.py +136 -0
  212. package/scripts/arxiv_source.py +201 -0
  213. package/scripts/badcase_ledger.py +252 -0
  214. package/scripts/batch_probe.py +241 -0
  215. package/scripts/benchmark.py +19 -0
  216. package/scripts/bounded_run.py +144 -0
  217. package/scripts/browser_auth.py +323 -0
  218. package/scripts/burstiness_detector.py +147 -0
  219. package/scripts/cache.py +1078 -67
  220. package/scripts/cache_guard.py +271 -0
  221. package/scripts/cache_key_vdom.py +56 -0
  222. package/scripts/cache_similarity.py +170 -0
  223. package/scripts/caliber_isolation.py +195 -0
  224. package/scripts/calibrate_lowquality.py +210 -0
  225. package/scripts/candidate_envelope.py +366 -0
  226. package/scripts/chrome_cdp.py +871 -0
  227. package/scripts/circuit_breaker.py +389 -0
  228. package/scripts/citations.py +334 -0
  229. package/scripts/cite_cli.py +59 -0
  230. package/scripts/cjk_tokens.py +95 -0
  231. package/scripts/clarify.py +47 -24
  232. package/scripts/cli_io.py +240 -0
  233. package/scripts/config.py +683 -45
  234. package/scripts/config_cache.py +402 -0
  235. package/scripts/conn_pool.py +220 -0
  236. package/scripts/content_security.py +685 -0
  237. package/scripts/content_signals.py +730 -0
  238. package/scripts/crawl.py +74 -35
  239. package/scripts/daemon.py +866 -0
  240. package/scripts/digup_bridge.py +214 -0
  241. package/scripts/engine_admission.py +366 -0
  242. package/scripts/engine_dispatch.py +1019 -0
  243. package/scripts/engine_env.py +534 -0
  244. package/scripts/engine_failure.py +411 -0
  245. package/scripts/engine_families.py +732 -0
  246. package/scripts/engine_policy.py +218 -0
  247. package/scripts/engine_requires.py +190 -0
  248. package/scripts/engine_status.py +322 -0
  249. package/scripts/engine_validate.py +638 -0
  250. package/scripts/engines.py +778 -294
  251. package/scripts/engines_base.py +1521 -0
  252. package/scripts/engines_builders.py +298 -0
  253. package/scripts/engines_builders_batch10.py +159 -0
  254. package/scripts/engines_builders_batch11.py +174 -0
  255. package/scripts/engines_builders_batch12.py +103 -0
  256. package/scripts/engines_builders_batch13.py +340 -0
  257. package/scripts/engines_builders_batch9.py +1468 -0
  258. package/scripts/engines_builders_cn.py +2283 -0
  259. package/scripts/engines_builders_data.py +2923 -0
  260. package/scripts/engines_builders_data_macro.py +775 -0
  261. package/scripts/engines_builders_feeds.py +255 -0
  262. package/scripts/engines_builders_intl.py +1239 -0
  263. package/scripts/engines_builders_search.py +440 -0
  264. package/scripts/engines_builders_tech.py +1360 -0
  265. package/scripts/evidence.py +588 -162
  266. package/scripts/evidence_loop.py +592 -0
  267. package/scripts/evidence_tier.py +298 -0
  268. package/scripts/except_sets.py +25 -0
  269. package/scripts/extract.py +55 -16
  270. package/scripts/fact_align.py +207 -0
  271. package/scripts/fact_crosscheck.py +230 -0
  272. package/scripts/fetch.py +25 -12
  273. package/scripts/fetch_browser.py +87 -0
  274. package/scripts/fetch_jsrun.py +122 -0
  275. package/scripts/fetch_quality.py +225 -0
  276. package/scripts/fetch_render_tinyfish.py +117 -0
  277. package/scripts/fetch_v3.py +1779 -0
  278. package/scripts/focus_extract.py +350 -0
  279. package/scripts/fulltext_store.py +169 -0
  280. package/scripts/fxtwitter_api.py +325 -0
  281. package/scripts/gen_engine_catalog.py +364 -0
  282. package/scripts/gen_native_tools.py +91 -0
  283. package/scripts/gen_routing_snapshot.py +184 -0
  284. package/scripts/health_check.py +140 -402
  285. package/scripts/health_probe.py +23 -8
  286. package/scripts/hot_state.py +134 -0
  287. package/scripts/http_client.py +1053 -0
  288. package/scripts/image_ops.py +430 -0
  289. package/scripts/image_vision/vision_probe +0 -0
  290. package/scripts/image_vision/vision_probe.swift +126 -0
  291. package/scripts/install.ps1 +148 -0
  292. package/scripts/install.sh +206 -0
  293. package/scripts/job.py +1186 -0
  294. package/scripts/jump_resolver.py +235 -0
  295. package/scripts/lang_capability.py +150 -0
  296. package/scripts/lang_detect.py +289 -0
  297. package/scripts/lang_matrix_probe.py +329 -0
  298. package/scripts/lang_pref.py +311 -0
  299. package/scripts/link_source.py +340 -0
  300. package/scripts/local_engines.py +321 -0
  301. package/scripts/local_image.py +970 -0
  302. package/scripts/local_seek.py +366 -0
  303. package/scripts/macro_countries.py +82 -0
  304. package/scripts/matrix_search_eval.py +1105 -0
  305. package/scripts/mcp_diag.py +8 -3
  306. package/scripts/mcp_handlers.py +1074 -0
  307. package/scripts/mcp_handlers_surface.py +157 -0
  308. package/scripts/mcp_launch.sh +122 -0
  309. package/scripts/mcp_server.py +118 -417
  310. package/scripts/mcp_setup.py +407 -0
  311. package/scripts/mcp_tools.py +330 -0
  312. package/scripts/mcp_transport.py +191 -0
  313. package/scripts/min_version_scan.py +335 -0
  314. package/scripts/net_proxy.py +258 -0
  315. package/scripts/network_aware.py +152 -0
  316. package/scripts/paper.py +274 -0
  317. package/scripts/pdf_extract.py +518 -0
  318. package/scripts/phrase_filter.py +66 -0
  319. package/scripts/plan.py +422 -0
  320. package/scripts/postprocess_signals.py +150 -0
  321. package/scripts/pre-commit +48 -0
  322. package/scripts/query_enhance.py +157 -0
  323. package/scripts/query_operators.py +196 -0
  324. package/scripts/query_rewriter.py +116 -7
  325. package/scripts/query_signals.py +300 -0
  326. package/scripts/query_templates.py +167 -0
  327. package/scripts/query_understanding.py +411 -0
  328. package/scripts/query_variants.py +298 -0
  329. package/scripts/quota.py +488 -76
  330. package/scripts/rank_signals.py +775 -0
  331. package/scripts/ranking_eval.py +319 -0
  332. package/scripts/readability_extract.py +433 -0
  333. package/scripts/recompute.py +415 -0
  334. package/scripts/recovery.py +616 -0
  335. package/scripts/recovery_l0.py +160 -0
  336. package/scripts/redskill/redskill_engine.py +332 -0
  337. package/scripts/regression_p0p1.py +318 -0
  338. package/scripts/replay_eval.py +640 -0
  339. package/scripts/rerank_tfidf.py +225 -0
  340. package/scripts/research.py +591 -403
  341. package/scripts/research_cli.py +361 -0
  342. package/scripts/research_dossier.py +492 -0
  343. package/scripts/research_expand.py +146 -0
  344. package/scripts/research_gates.py +272 -0
  345. package/scripts/research_report.py +209 -0
  346. package/scripts/research_strategy.py +37 -0
  347. package/scripts/research_work_packages.py +195 -0
  348. package/scripts/result_lang.py +311 -0
  349. package/scripts/robots_guard.py +259 -0
  350. package/scripts/route.py +921 -254
  351. package/scripts/route_bangs.py +40 -0
  352. package/scripts/route_cache.py +225 -0
  353. package/scripts/route_combo.py +910 -0
  354. package/scripts/route_domains.py +328 -0
  355. package/scripts/route_lang.py +605 -0
  356. package/scripts/route_log.py +63 -0
  357. package/scripts/route_policy.py +250 -0
  358. package/scripts/rrf_classic.py +66 -0
  359. package/scripts/screenshot.py +73 -0
  360. package/scripts/search.py +897 -405
  361. package/scripts/search_attribution.py +88 -0
  362. package/scripts/search_benchmark.py +407 -0
  363. package/scripts/search_cli.py +627 -0
  364. package/scripts/search_entry.py +389 -0
  365. package/scripts/search_output.py +606 -0
  366. package/scripts/search_pipeline.py +1077 -0
  367. package/scripts/search_rank.py +991 -0
  368. package/scripts/search_types.py +27 -12
  369. package/scripts/seek_locator.py +68 -0
  370. package/scripts/semantic_evidence.py +215 -0
  371. package/scripts/serp_guard.py +228 -0
  372. package/scripts/serp_kill_log.py +51 -0
  373. package/scripts/single_flight.py +120 -0
  374. package/scripts/social_engines/__init__.py +10 -0
  375. package/scripts/social_engines/bilibili_engine.py +131 -0
  376. package/scripts/social_engines/hackernews_engine.py +113 -0
  377. package/scripts/social_engines/reddit_engine.py +172 -0
  378. package/scripts/social_engines/twitter_engine.py +397 -0
  379. package/scripts/social_engines/v2ex_engine.py +101 -0
  380. package/scripts/social_engines/weibo_engine.py +128 -0
  381. package/scripts/social_engines/xiaohongshu_engine.py +137 -0
  382. package/scripts/social_engines/zhihu_engine.py +139 -0
  383. package/scripts/social_research.py +201 -0
  384. package/scripts/sogou_weixin_browser.py +318 -0
  385. package/scripts/source_ledger.py +249 -0
  386. package/scripts/stage_timing.py +58 -0
  387. package/scripts/stats_cli.py +95 -0
  388. package/scripts/straggler_policy.py +60 -0
  389. package/scripts/stylometry_detector.py +231 -0
  390. package/scripts/sync_backends.py +353 -0
  391. package/scripts/sync_installs.py +29 -0
  392. package/scripts/template_cli.py +127 -0
  393. package/scripts/text_lexicon.py +72 -0
  394. package/scripts/tfidf_router.py +109 -37
  395. package/scripts/time_utils.py +233 -0
  396. package/scripts/topic_research_profiles.py +643 -0
  397. package/scripts/train.py +737 -0
  398. package/scripts/tweet.py +596 -0
  399. package/scripts/url_canon.py +217 -0
  400. package/scripts/url_optimize.py +56 -0
  401. package/scripts/url_safety.py +236 -0
  402. package/scripts/usage_log.py +116 -0
  403. package/scripts/usage_log_clean.py +113 -0
  404. package/scripts/v2ex_nodes.py +311 -0
  405. package/scripts/variant_recall.py +188 -0
  406. package/scripts/watch.py +205 -0
  407. package/scripts/wechat_article.py +258 -0
  408. package/scripts/wide_research.py +402 -0
  409. package/scripts/wx.py +349 -0
  410. package/scripts/x_article_fetch.py +141 -0
  411. package/scripts/x_search.py +234 -0
  412. package/scripts/x_trending.py +189 -0
  413. package/scripts/yaml_load.py +67 -0
  414. package/sub-skills/ego-search/SKILL.md +253 -0
  415. package/sub-skills/ego-search/references/browser-runtime.md +43 -0
  416. package/sub-skills/ego-search/references/install.md +67 -0
  417. package/sub-skills/ego-search/references/original-ego-upgrade.md +51 -0
  418. package/sub-skills/ego-search/scripts/ego_search.py +800 -0
  419. package/sub-skills/ego-search/scripts/merge.py +226 -0
  420. package/sub-skills/ego-search/scripts/quality.py +64 -0
  421. package/sub-skills/ego-search/scripts/runtime.py +172 -0
  422. package/sub-skills/ego-search/scripts/safety.py +109 -0
  423. package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
  424. package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
  425. package/sub-skills/ego-search/tests/test_query_normalize.py +68 -0
  426. package/sub-skills/js-run/SKILL.md +67 -0
  427. package/sub-skills/js-run/scripts/jsrun.py +112 -0
  428. package/sub-skills/js-run/scripts/shim.js +188 -0
  429. package/sub-skills/js-run/tests/fixtures/challenge_like.js +27 -0
  430. package/sub-skills/js-run/tests/test_jsrun.py +165 -0
  431. package/sub-skills/local-search/SKILL.md +50 -53
  432. package/sub-skills/local-search/bm25.py +162 -0
  433. package/sub-skills/local-search/config.yaml +152 -55
  434. package/sub-skills/local-search/engine_registry.py +91 -26
  435. package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
  436. package/sub-skills/local-search/local_search_adapter.py +13 -2
  437. package/sub-skills/local-search/parse_maps.yaml +40 -42
  438. package/sub-skills/local-search/search_v3.py +1182 -65
  439. package/sub-skills/local-search/smart_router.py +47 -7
  440. package/sub-skills/local-seek/SKILL.md +110 -0
  441. package/sub-skills/local-seek/config/domains.yaml +20 -0
  442. package/sub-skills/local-seek/references/auto-behaviors.md +24 -0
  443. package/sub-skills/local-seek/references/fd-mdfind-recipes.md +62 -0
  444. package/sub-skills/local-seek/references/git-integration.md +54 -0
  445. package/sub-skills/local-seek/references/remote-handoff.md +46 -0
  446. package/sub-skills/local-seek/references/rg-recipes.md +96 -0
  447. package/sub-skills/local-seek/references/strategies.md +52 -0
  448. package/sub-skills/local-seek/references/structural-search.md +59 -0
  449. package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
  450. package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
  451. package/sub-skills/local-seek/scripts/seek.py +1465 -0
  452. package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
  453. package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
  454. package/sub-skills/local-seek/scripts/seek_rank.py +193 -0
  455. package/sub-skills/local-seek/tests/test_content_rank.py +222 -0
  456. package/sub-skills/local-seek/tests/test_determinism_20261006.py +145 -0
  457. package/sub-skills/local-seek/tests/test_noise_tier.py +101 -0
  458. package/sub-skills/local-seek/tests/test_pcre2_probe.py +136 -0
  459. package/sub-skills/local-seek/tests/test_rank_pinyin.py +77 -0
  460. package/sub-skills/local-seek/tests/test_regression_20261003.py +169 -0
  461. package/sub-skills/local-seek/tests/test_regression_20261006.py +146 -0
  462. package/sub-skills/local-seek/tests/test_seek_degradation.py +111 -0
  463. package/sub-skills/local-seek/tests/test_seek_fts.py +313 -0
@@ -1,20 +1,15 @@
1
1
  #!/usr/bin/env python3
2
- """
3
- search_types.py — 统一类型系统
4
-
5
- 定义 SearchResult 数据类和 normalize_result 统一转换,
6
- 将所有引擎的原始输出统一为结构化格式。
2
+ """search_types.py — 统一类型系统(已废弃,勿接入)
7
3
 
8
- 字段规范:
9
- - title: 结果标题(必填,截断到 200 字符)
10
- - url: 结果链接(可选)
11
- - snippet: 摘要/片段(截断到 300 字符)
12
- - score: 相关性评分(默认 0.5)
13
- - source: 引擎来源
14
- - metadata: 元数据字典
4
+ 早期设计声明的统一 schema 层,从未被任何模块 import(grep 全仓库无引用):
5
+ - 引擎输出直接透传 dict,运行时防御在 engines_base/safe_search 与
6
+ search.py 的 execute_search 里完成
7
+ - 保留本文件仅避免旧版外部脚本 import 断裂;新代码禁止使用
15
8
  """
16
9
  from __future__ import annotations
17
10
 
11
+ from enum import Enum
12
+
18
13
  from dataclasses import dataclass, field, asdict
19
14
  from typing import Any
20
15
 
@@ -28,6 +23,7 @@ class SearchResult:
28
23
  score: float = 0.5
29
24
  source: str = ""
30
25
  metadata: dict = field(default_factory=dict)
26
+ social_meta: dict | None = None
31
27
 
32
28
  def to_dict(self) -> dict[str, Any]:
33
29
  """转为字典,省略空值/默认值字段。"""
@@ -42,6 +38,8 @@ class SearchResult:
42
38
  d["source"] = self.source
43
39
  if self.metadata:
44
40
  d["metadata"] = self.metadata
41
+ if self.social_meta:
42
+ d["social_meta"] = self.social_meta
45
43
  return d
46
44
 
47
45
 
@@ -69,4 +67,21 @@ def normalize_result(item: dict, engine: str, default_score: float = 0.5) -> Sea
69
67
  score=score,
70
68
  source=item.get("source", engine),
71
69
  metadata=item.get("metadata", {}),
70
+ social_meta=item.get("social_meta"),
72
71
  )
72
+
73
+
74
+ class Stage(str, Enum):
75
+ """进度回调的阶段名(on_progress 契约)。
76
+
77
+ 从 search.py 搬来这里:执行层与加工层(search_pipeline)都要发进度事件,
78
+ 枚举放在任何一侧都会让另一侧反向依赖;search_types 是本仓共享类型的家。
79
+ """
80
+
81
+ START = "start"
82
+ CACHE_HIT = "cache_hit"
83
+ ROUTING = "routing"
84
+ SEARCHING = "searching"
85
+ MERGING = "merging"
86
+ DONE = "done"
87
+ ERROR = "error"
@@ -0,0 +1,68 @@
1
+ #!/usr/bin/env python3
2
+ """seek_locator.py — local-seek 子技能的「安装感知」定位(唯一来源,不写死主机路径)。
3
+
4
+ 本地搜索(argo_local_search / --include-local)需要 local-seek/scripts/seek.py。
5
+ local-seek 随 argo 打包在 sub-skills/ 下(安装感知:寻址锚点是本文件的真实安装位置),
6
+ 也可能作为独立 skill 装在别处。此模块把发现逻辑收紧到一处,供 search.py 与
7
+ mcp_handlers.py 共用,消除两处各自硬编码 __file__ 相对路径与 ~/.agents/skills 等
8
+ 主机路径的重复(Fowler: Duplicated Code 反面却是各自为政、易漂移)。
9
+
10
+ 保持一致 SKILL.md 纪律「禁止在产品代码写死主机 skill 路径」:不再出现
11
+ ~/.agents/skills / ~/.claude/skills 这类字面量,改由环境变量承载自定义/遗留位置。
12
+
13
+ 查找顺序(命中即返回,全部未命中则返回打包位置,由调用方据此 fail 提示):
14
+ 1. ARGO_LOCAL_SEEK_PATH 显式指定 seek.py 的绝对/相对路径(任意自定义位置)
15
+ 2. <ARGO_ROOT>/sub-skills/local-seek/scripts/seek.py
16
+ —— 打包子技能,锚点为本模块的安装位置(symlink 会解析回来源)
17
+ 3. ARGO_LOCAL_SEEK_ROOTS os.pathsep 分隔的候选根目录,逐个检查
18
+ <根>/local-seek/scripts/seek.py(承载遗留/独立装法)
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import os
24
+ from typing import Iterator
25
+
26
+ ARGO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
27
+ _BUNDLED = os.path.join(ARGO_ROOT, "sub-skills", "local-seek", "scripts", "seek.py")
28
+
29
+
30
+ def _candidates() -> Iterator[str]:
31
+ """按优先级产出候选 seek.py 路径(仅路径,未校验存在性)。
32
+
33
+ 优先级:显式文件覆盖 > 显式候选根 > 打包位置(默认)。
34
+ 显式配置(ARGO_LOCAL_SEEK_PATH / ARGO_LOCAL_SEEK_ROOTS)代表用户点名了
35
+ 装在哪,理应先于总是存在的打包子技能;打包位置只作保底。
36
+ """
37
+ override = (os.environ.get("ARGO_LOCAL_SEEK_PATH") or "").strip()
38
+ if override:
39
+ yield os.path.realpath(os.path.expanduser(override))
40
+ roots = (os.environ.get("ARGO_LOCAL_SEEK_ROOTS") or "").strip()
41
+ if roots:
42
+ for root in roots.split(os.pathsep):
43
+ root = root.strip()
44
+ if not root:
45
+ continue
46
+ base = os.path.realpath(os.path.expanduser(root))
47
+ candidate = os.path.join(base, "local-seek", "scripts", "seek.py")
48
+ if os.path.isfile(candidate):
49
+ yield candidate
50
+ yield _BUNDLED
51
+
52
+
53
+ def resolve_seek_py() -> str:
54
+ """返回 local-seek/scripts/seek.py 的绝对路径(命中即返回,未命中返回打包位置)。"""
55
+ for cand in _candidates():
56
+ if cand and os.path.isfile(cand):
57
+ return cand
58
+ return _BUNDLED
59
+
60
+
61
+ def seek_py_exists() -> bool:
62
+ """本地搜索能力是否可用(seek.py 实际存在)。"""
63
+ return os.path.isfile(resolve_seek_py())
64
+
65
+
66
+ if __name__ == "__main__": # pragma: no cover
67
+ import sys
68
+ print(resolve_seek_py())
@@ -0,0 +1,215 @@
1
+ #!/usr/bin/env python3
2
+ """semantic_evidence.py — 可选的语义证据层(classifier.dev,默认关闭)。
3
+
4
+ ## 为什么存在
5
+
6
+ argo 的正文吸收分是**正则密度**(has_numbers / has_definition / …),它测的是
7
+ 「这段文本长得像不像证据块」,不测「它是否支持这条查询的主张」。2026-09-19 的
8
+ 实测(reports/classifier-dev-argo/A、B):密度最高页判 neutral、密度最低页判
9
+ supports 0.99,两者近乎反向;而假前提主张(查询预设「降息」、现实是「加息」)
10
+ 密度分在结构上无法表达,语义判定 5/5 判 contradicts(conf 0.95~1.00)。
11
+
12
+ ## 定位(与热路径解耦)
13
+
14
+ - **只在显式核验链路调用**(`search --verify` / `research --verify`),不进搜索
15
+ 热路径:热命中 78~82ms 且网络为零,一次分类调用 200~680ms,放进热路径是净损失
16
+ - **默认关闭**:config.yaml `semantic_evidence.enabled: false`;显式开启用环境变量
17
+ `ARGO_SEMANTIC_EVIDENCE=1`(env 优先于 config)
18
+ - **fail-open**:超时 / HTTP 错误 / 429 / 解析失败 / 条数不匹配 → 返回 None,
19
+ 调用方行为与未开启时逐位一致(不加字段、不改排序、不改公式)
20
+ - **只加并列字段**:`semantic_support` 挂在结果与 verified 条目上,可信度公式
21
+ (selection 0.40 / absorption 0.35 / freshness 0.15 / original 0.10)不动
22
+
23
+ ## 外部服务约束(2026-09-19 实测,见 reports/classifier-dev-argo/D)
24
+
25
+ - 无密钥、HTTP 通道稳定(1000 条 1.83s、顺序严格保真);CLI 与 MCP 通道当前不可用
26
+ (npm/PyPI 404、MCP tools/call 返回 not_found),因此只走 HTTP
27
+ - 必须带真实 User-Agent(python urllib 默认 UA 被边缘 403)——复用 HttpClient 的
28
+ UA 轮换与 429 退避
29
+ - 标签集必须显式含「none of these」:服务永远返回给定标签之一,不加会把无关文本
30
+ 硬塞进某个标签
31
+ - 同输入置信度有 ±0.08 漂移(标签稳定),所以只做 ≥0.7 的粗门控;实测 conf ≥0.7
32
+ 时严格错误为 0(51 条 entailment 样本)
33
+ - 限额按出口 IP:fast 3000 次/分钟、20000 次/天;空串输入会让整批 400,调用前过滤
34
+ """
35
+
36
+ from __future__ import annotations
37
+
38
+ import json
39
+ import logging
40
+ from typing import Any, Optional
41
+
42
+ logger = logging.getLogger("unified_search.semantic_evidence")
43
+
44
+ DEFAULT_ENDPOINT = "https://classifier.dev"
45
+ DEFAULT_TIMEOUT_S = 5.0
46
+ # conf ≥ 0.7 才给结论:低于此值的判定不可信(B 实测 4 个严格错误全部 ≤0.67)
47
+ SUPPORT_THRESHOLD = 0.7
48
+ # 单条文本上限:核验链路取回的正文 ≤8000 字,再高无收益(A 实测 400→7798 字
49
+ # 置信度反而从 0.73 升到 0.98,说明长文不劣化,但也没必要超发)
50
+ MAX_TEXT_CHARS = 8000
51
+
52
+ SUPPORT_LABELS = (
53
+ "supports the query",
54
+ "contradicts the query",
55
+ "neutral",
56
+ "none of these",
57
+ )
58
+ SOURCE_TYPE_LABELS = (
59
+ "primary data or official announcement",
60
+ "secondary reporting",
61
+ "analysis or opinion",
62
+ "social narrative",
63
+ "none of these",
64
+ )
65
+
66
+
67
+ def _config() -> dict[str, Any]:
68
+ """读 config.yaml 的 semantic_evidence 段(缺失时用内置默认)。"""
69
+ try:
70
+ from config import load_config
71
+ cfg = load_config().get("semantic_evidence")
72
+ return cfg if isinstance(cfg, dict) else {}
73
+ except Exception: # pragma: no cover - 配置不可用按默认关处理
74
+ return {}
75
+
76
+
77
+ def enabled() -> bool:
78
+ """开关判定:环境变量优先,其次 config.yaml;两者都缺省 → 关闭。
79
+
80
+ 用 engine_env.get_env/env_flag 读(全仓唯一的布尔解析),所以
81
+ `ARGO_SEMANTIC_EVIDENCE=off`、写进 ~/.config/argo/env 的写法都生效。
82
+ """
83
+ try:
84
+ from engine_env import env_flag, get_env
85
+ if get_env("ARGO_SEMANTIC_EVIDENCE").strip():
86
+ return env_flag("ARGO_SEMANTIC_EVIDENCE", default=False)
87
+ except Exception: # pragma: no cover - 环境层不可用时不冒险开启
88
+ return False
89
+ return bool(_config().get("enabled", False))
90
+
91
+
92
+ def _endpoint() -> str:
93
+ return str(_config().get("endpoint") or DEFAULT_ENDPOINT).rstrip("/")
94
+
95
+
96
+ def _timeout() -> float:
97
+ try:
98
+ return float(_config().get("timeout_s") or DEFAULT_TIMEOUT_S)
99
+ except (TypeError, ValueError):
100
+ return DEFAULT_TIMEOUT_S
101
+
102
+
103
+ def classify(texts: list[str], labels: tuple[str, ...] | list[str], *,
104
+ instructions: str | None = None,
105
+ timeout: float | None = None,
106
+ tier: str | None = None) -> Optional[list[dict[str, Any]]]:
107
+ """一次批量分类(≤1000 条)。任何异常/非 200/条数不匹配 → None。
108
+
109
+ 返回按输入顺序排列的 [{label, confidence, scores}];调用方负责判定阈值。
110
+ 空文本会被过滤后再发(空串会让整批 400),返回列表与输入等长、空缺位为 None。
111
+ """
112
+ if not texts or len(labels) < 2:
113
+ return None
114
+ keep = [(i, t) for i, t in enumerate(texts) if (t or "").strip()]
115
+ if not keep:
116
+ return None
117
+ body: dict[str, Any] = {
118
+ "labels": [str(x) for x in labels],
119
+ "inputs": [t[:MAX_TEXT_CHARS] for _i, t in keep],
120
+ "tier": tier or str(_config().get("tier") or "fast"),
121
+ }
122
+ if instructions:
123
+ body["instructions"] = instructions
124
+ try:
125
+ from http_client import HttpClient
126
+ resp = HttpClient(timeout=timeout or _timeout(), max_retries=1,
127
+ jitter=False).post(_endpoint(), body)
128
+ if resp.get("status") != 200:
129
+ logger.debug(f"语义分类非 200: {resp.get('status')} {resp.get('error')}")
130
+ return None
131
+ data = json.loads(resp.get("text") or "{}")
132
+ results = data.get("results")
133
+ if not isinstance(results, list) or len(results) != len(keep):
134
+ logger.debug("语义分类返回条数与输入不匹配,按失败处理")
135
+ return None
136
+ except Exception as e: # fail-open:语义层任何问题都不阻断核验
137
+ logger.debug(f"语义分类失败: {type(e).__name__}")
138
+ return None
139
+ out: list[Optional[dict[str, Any]]] = [None] * len(texts)
140
+ for (idx, _t), r in zip(keep, results):
141
+ if isinstance(r, dict):
142
+ out[idx] = r
143
+ return out # type: ignore[return-value]
144
+
145
+
146
+ def assess_support(query: str, items: list[dict[str, Any]]) -> Optional[dict[str, dict]]:
147
+ """对已抓取的正文批量判定「是否支持查询隐含的主张」。
148
+
149
+ items: [{"url": str, "title": str, "text": str}]
150
+ 返回 {url: {label, confidence, supports, contradicts}};未达阈值时
151
+ supports/contradicts 均为 False(原始 label 与 confidence 仍原样保留,
152
+ 调用方可自行换阈值)。失败返回 None。
153
+ """
154
+ usable = [it for it in items if (it.get("text") or "").strip() and it.get("url")]
155
+ if not usable:
156
+ return None
157
+ texts = [f"{it.get('title') or ''}\n{(it.get('text') or '')[:MAX_TEXT_CHARS]}"
158
+ for it in usable]
159
+ instructions = (
160
+ f"Judge whether each passage supports the claim implied by this query: {query!r}. "
161
+ "Answer 'supports the query' when the passage backs the claim, "
162
+ "'contradicts the query' when it states the opposite or a conflicting figure, "
163
+ "'neutral' when it is on-topic but neither supports nor contradicts, "
164
+ "and 'none of these' when it is unrelated to the query."
165
+ )
166
+ results = classify(texts, SUPPORT_LABELS, instructions=instructions)
167
+ if results is None:
168
+ return None
169
+ out: dict[str, dict] = {}
170
+ for it, r in zip(usable, results):
171
+ if not r:
172
+ continue
173
+ label = str(r.get("label") or "")
174
+ conf = r.get("confidence")
175
+ conf_f = float(conf) if isinstance(conf, (int, float)) else 0.0
176
+ out[str(it["url"])] = {
177
+ "label": label,
178
+ "confidence": conf_f,
179
+ "supports": label == "supports the query" and conf_f >= SUPPORT_THRESHOLD,
180
+ "contradicts": label == "contradicts the query" and conf_f >= SUPPORT_THRESHOLD,
181
+ }
182
+ return out or None
183
+
184
+
185
+ def assess_source_types(items: list[dict[str, Any]]) -> Optional[dict[str, dict]]:
186
+ """来源类型分级(研究取证链路用):一手数据 / 二手报道 / 分析观点 / 社交叙事。
187
+
188
+ items: [{"url": str, "title": str, "text": str}](text 可为摘要)
189
+ """
190
+ usable = [it for it in items if it.get("url") and
191
+ ((it.get("text") or "").strip() or (it.get("title") or "").strip())]
192
+ if not usable:
193
+ return None
194
+ texts = [f"{it.get('title') or ''}\n{(it.get('text') or '')[:MAX_TEXT_CHARS]}"
195
+ for it in usable]
196
+ instructions = (
197
+ "Classify each source by its nature: 'primary data or official announcement' "
198
+ "for original datasets, filings, official releases; 'secondary reporting' for "
199
+ "news coverage of someone else's work; 'analysis or opinion' for commentary; "
200
+ "'social narrative' for forum/social posts."
201
+ )
202
+ results = classify(texts, SOURCE_TYPE_LABELS, instructions=instructions)
203
+ if results is None:
204
+ return None
205
+ out: dict[str, dict] = {}
206
+ for it, r in zip(usable, results):
207
+ if not r:
208
+ continue
209
+ label = str(r.get("label") or "")
210
+ conf = r.get("confidence")
211
+ out[str(it["url"])] = {
212
+ "label": label,
213
+ "confidence": float(conf) if isinstance(conf, (int, float)) else 0.0,
214
+ }
215
+ return out or None
@@ -0,0 +1,228 @@
1
+ #!/usr/bin/env python3
2
+ """serp_guard — HTML SERP 垃圾结果守卫(判定规则单点)。
3
+
4
+ 事故背景(dsh-free-search issue #38/#35 同款事故,argo 的 local_* 引擎同样
5
+ 暴露):html 引擎解析的是无 API 的网页 SERP,上游偶发把缓存 SERP / 与查询
6
+ 完全无关的结果页当正常页返回,解析层照常抽出标题摘要,垃圾条目静默混进
7
+ 聚合,调用方把「答非所问」当成「网上没有」——失败伪装成功是红线,这类
8
+ 混入是它的隐形变体。
9
+
10
+ 裁决口径(全部量化,无模糊地带):
11
+
12
+ 1. query token:CJK 连续段取相邻二元组、拉丁/数字连续段取小写词。
13
+ token 少于 2 个时恒 False(无裁决权)——单 token 的巧合重叠率太高,
14
+ 拦了误杀大于漏放。单字 CJK 查询(如「猫」)只有 0 个二元组,双字
15
+ CJK 查询(如「猫粮」)只有 1 个,都天然落在无裁决区。
16
+ 2. results 只取前 5 条;每条取 title+snippet。URL 不参与:缓存/跳转
17
+ 链接的 URL 常回显 query 参数(uddg=、q=),会正好把垃圾页放行,
18
+ 标题+摘要才是「这页在答什么」的信号。
19
+ 3. 前 5 条全部与 query token 零交集 → 整页判垃圾(True)。
20
+
21
+ 为什么调用方对判垃圾应整页丢弃而不是过滤后返回幸存者:前 5 条全无关说明
22
+ 这一页整体答非所问(缓存页/错路由),幸存的一两条是同一页的噪声,过滤
23
+ 返回等于给垃圾留门,宁缺勿假。调用方(engines_base)判垃圾后返回 [] 且
24
+ 不记 note_failure——结果无关不是引擎故障,不该进熔断归因。
25
+
26
+ ARGO_SERP_GUARD=0 整体关闭守卫(逃生门:规则误杀时第一反应是关闸复现,
27
+ 而不是删代码)。
28
+
29
+ 不 import search_rank:本模块处在 engines_base 的冷启动热路径上,search_rank
30
+ 较重且有环风险;分词口径按其 _tokens/_bigrams 的语义在本模块最小复刻
31
+ (CJK 二元组 + 小写拉丁词),不共享代码。
32
+ """
33
+
34
+ from __future__ import annotations
35
+
36
+ import json
37
+ import os
38
+ import re
39
+ from typing import Any
40
+ from urllib.parse import urlparse
41
+
42
+ # CJK 连续段 / 小写拉丁数字词(findall 前文本已 lower())
43
+ _RUN_RE = re.compile(r"[\u4e00-\u9fff]+|[a-z0-9]+")
44
+ _GUARD_WINDOW = 5 # 只看前 5 条:SERP 的「答非所问」看头部即可下结论
45
+ _MIN_QUERY_TOKENS = 2 # query token 少于此数 → 无裁决权,恒不拦
46
+ _SWITCH_ENV = "ARGO_SERP_GUARD"
47
+
48
+
49
+ def guard_enabled() -> bool:
50
+ """逃生门:仅 ARGO_SERP_GUARD=0 关闭,其余取值(含未设置)均开启。"""
51
+ return os.environ.get(_SWITCH_ENV, "") != "0"
52
+
53
+
54
+ def _extract_tokens(text: str) -> set[str]:
55
+ """提取 token:CJK 相邻二元组 + 小写拉丁/数字词(与 search_rank 同口径)。"""
56
+ tokens: set[str] = set()
57
+ for run in _RUN_RE.findall((text or "").lower()):
58
+ if "\u4e00" <= run[0] <= "\u9fff":
59
+ tokens.update(run[i:i + 2] for i in range(len(run) - 1))
60
+ else:
61
+ tokens.add(run)
62
+ return tokens
63
+
64
+
65
+ def _result_text(item: object) -> str:
66
+ """单条结果的相关性文本 = title + snippet;非 dict 或字段缺失按空处理。"""
67
+ if not isinstance(item, dict):
68
+ return ""
69
+ return f"{item.get('title') or ''} {item.get('snippet') or ''}"
70
+
71
+
72
+ def is_junk_serp(query: str, results: list[dict]) -> bool:
73
+ """判一页 SERP 结果是否整体答非所问(口径见模块 docstring)。
74
+
75
+ 返回 True 表示整页判垃圾,调用方应丢弃全部条目;返回 False 表示不拦
76
+ (含 query 无裁决权、results 为空、守卫被关闭——三种都不拦)。
77
+ """
78
+ if not guard_enabled():
79
+ return False
80
+ q_tokens = _extract_tokens(query if isinstance(query, str) else "")
81
+ if len(q_tokens) < _MIN_QUERY_TOKENS:
82
+ return False
83
+ if not isinstance(results, list):
84
+ return False
85
+ head = [r for r in results[:_GUARD_WINDOW] if r is not None]
86
+ if not head:
87
+ return False
88
+ for item in head:
89
+ if _extract_tokens(_result_text(item)) & q_tokens:
90
+ return False
91
+ return True
92
+
93
+
94
+ # ── SERP/跳转 URL 判定(自 evidence.py 迁入,2026-10-01)────────────────────
95
+ # 迁入原因:这组判定是纯 URL 规则,但原宿主 evidence.py 会把 content_signals
96
+ # 等证据质量栈一起拖进 import(实测 ~9 ms/进程),而它的调用方
97
+ # (evidence_loop.gate_results、search_pipeline 的 SERP 过滤)在**每次搜索
98
+ # 必经**的路径上——缓存命中也照付。本模块只依赖 os/re/json/urllib.parse,
99
+ # 处在 engines_base 冷启动热路径上,轻量有既有纪律保证(见模块 docstring)。
100
+ # evidence.py 保留同名转出,既有调用方(ab_eval_p0p1 等)不受影响。
101
+ _BACKENDS_DIR = os.path.normpath(os.path.join(
102
+ os.path.dirname(os.path.abspath(__file__)), "..", "backends"))
103
+ _CN_OVERRIDES: "dict[str, Any] | None" = None
104
+
105
+
106
+ def _load_cn_source_types() -> dict[str, Any]:
107
+ global _CN_OVERRIDES
108
+ if _CN_OVERRIDES is not None:
109
+ return _CN_OVERRIDES
110
+ path = os.path.join(_BACKENDS_DIR, "source_types_cn.json")
111
+ try:
112
+ with open(path, "r", encoding="utf-8") as f:
113
+ _CN_OVERRIDES = json.load(f)
114
+ except Exception:
115
+ _CN_OVERRIDES = {}
116
+ return _CN_OVERRIDES
117
+
118
+
119
+ def _normalize_domain(url: str) -> str:
120
+ if not url:
121
+ return ""
122
+ try:
123
+ host = urlparse(url).netloc.lower()
124
+ except Exception:
125
+ return ""
126
+ if "@" in host:
127
+ host = host.rsplit("@", 1)[1]
128
+ if ":" in host:
129
+ host = host.split(":", 1)[0]
130
+ if host.startswith("www."):
131
+ host = host[4:]
132
+ return host
133
+
134
+
135
+ def _query_has_param(query: str, name: str) -> bool:
136
+ """query 参数名完整匹配:只在开头或 & 后出现才算。
137
+
138
+ 子串匹配会把百度网盘分享的 ?pwd=x 连坐进 wd=(pwd 恰以 wd= 结尾,
139
+ 实测误杀)——参数名只可能出现在 query 开头或 & 之后。
140
+ """
141
+ return query.startswith(name + "=") or ("&" + name + "=") in query
142
+
143
+
144
+ def is_serp_or_jump_url(url: str) -> bool:
145
+ """是否为搜索引擎结果页 / 跳转壳(不可当信源正文)。"""
146
+ if not url:
147
+ return True
148
+ low = url.lower()
149
+ host = _normalize_domain(url)
150
+
151
+ # 白名单:百度子域(百家号、网盘等)不是 SERP,即使 path 为 /s 也不应误杀。
152
+ # 必须最先检查,否则 serp_url_patterns 中的 baidu.com/s? 会误匹配
153
+ # baijiahao.baidu.com/s?id= 百家号文章页。
154
+ _serp_whitelist = ("baijiahao.baidu.com", "pan.baidu.com", "tieba.baidu.com",
155
+ "zhidao.baidu.com", "wenku.baidu.com")
156
+ host_bare = host[4:] if host.startswith("www.") else host
157
+ if host in _serp_whitelist or host_bare in _serp_whitelist:
158
+ return False
159
+
160
+ cfg = _load_cn_source_types()
161
+ for pat in cfg.get("serp_url_patterns") or []:
162
+ if pat.lower() in low:
163
+ return True
164
+ # 显式搜索/跳转模式
165
+ if re.search(r"(?:^|\.)baidu\.com/s\?", low):
166
+ return True
167
+ if "baidu.com/link" in low or "baidu.com/baidu.php" in low:
168
+ return True
169
+ if "sogou.com/link" in low:
170
+ return True
171
+ if "google.com/url?" in low or "google.com.hk/url?" in low:
172
+ return True
173
+ if re.search(r"(bing|google|google\.com\.hk)\.com/search\?", low):
174
+ return True
175
+ if "weixin.sogou.com/weixin" in low:
176
+ return True
177
+ try:
178
+ path = urlparse(url).path or ""
179
+ query = urlparse(url).query or ""
180
+ except Exception:
181
+ path, query = "", ""
182
+ # serp 域名只从 source_types_cn.json 的 serp_host_markers 读。此前这里
183
+ # 还并列写了一份字面量集合,两处各写一份正是漏网的成因:google.co.jp /
184
+ # yahoo.co.jp / duckduckgo.com 的结果页会被当成正文信源(authority 拿
185
+ # 到正常分、并进入最终输出),而 google.com 被拦——同一个概念两种判定。
186
+ #
187
+ # 判定用**后缀匹配**而非等值:同一搜索引擎的入口域名很多(yahoo.co.jp /
188
+ # search.yahoo.co.jp、duckduckgo.com / lite.duckduckgo.com、brave.com /
189
+ # search.brave.com),而 _normalize_domain 只去 www.,子域会原样保留。
190
+ # 等值比较下每上一个新入口就得往表里再补一条——那正是这份表原本在漏的。
191
+ serp_hosts = tuple(cfg.get("serp_host_markers") or ())
192
+ _hit = any(
193
+ host_bare == m or host == m
194
+ or host_bare.endswith("." + m) or host.endswith("." + m)
195
+ for m in serp_hosts
196
+ )
197
+ if _hit:
198
+ # path 只精确比较:此前 `path.startswith("/s")` 会把 pan.baidu.com/s/<id>
199
+ # 网盘分享正文链连坐误杀(实测;中文资源类查询的常见结果)。SERP 的
200
+ # path 恰为 /s,上方精确元组已覆盖,不需要前缀匹配。query 参数走
201
+ # _query_has_param 完整匹配(?pwd= 不再喂进 wd=)。
202
+ if path in ("", "/", "/s", "/web", "/search") \
203
+ or "link" in path or "search" in path \
204
+ or _query_has_param(query, "q") or _query_has_param(query, "wd"):
205
+ return True
206
+ return False
207
+
208
+
209
+ def drop_shell_urls(results: list) -> list:
210
+ """过滤结果列表里的 SERP/跳转壳条目——壳判定的唯一过滤出口。
211
+
212
+ 为什么必须是本模块的共用函数而不是各处内联:壳判定有多个消费点
213
+ (充分性判定 query_signals——壳不算「够数」;恢复链 L0——壳不算
214
+ 「交付」,否则误判恢复成功;L3 恢复结果收口——主链的壳过滤跑在
215
+ 恢复链之前,恢复产出是唯一可能把壳带回最终输出的入口)。判定规则
216
+ 演进过多次(百度子域白名单、网盘 /s/<id> 误杀),每处内联一份就多
217
+ 一处漏改点;2026-10-11 热榜壳链事故的根因之一正是充分性与漏斗
218
+ 两处口径漂移。无 url 的结构化快照(行情/天气)不受影响。
219
+ """
220
+ kept = []
221
+ for r in results:
222
+ if not isinstance(r, dict):
223
+ continue
224
+ u = r.get("url")
225
+ if isinstance(u, str) and u and is_serp_or_jump_url(u):
226
+ continue
227
+ kept.append(r)
228
+ return kept
@@ -0,0 +1,51 @@
1
+ #!/usr/bin/env python3
2
+ """serp_kill_log — serp_guard 整页误杀的运行内账本(2026-10-10 方案甲)。
3
+
4
+ 为什么单开一个模块:engines_base 是祖父文件、行数顶在登记上限,尺寸门的
5
+ 哲学是「要加东西先拆模块」。_serp_guard_apply 的判定逻辑随本账本一起搬来,
6
+ engines_base 留薄委托保持原导入面不变。
7
+
8
+ 背景:_serp_guard_apply 对整页垃圾返回 [] 是「诚实空」(结果无关不进熔断
9
+ 归因、不伪装成成功),但对调用方等价于「引擎没搜到」——误杀类缺陷(搜狗
10
+ 跳转壳 10-06、百家号 10-08)都在这里静默发生过。击杀先入账本,search 层
11
+ 执行完 drain 进 errors:可见、可归因、可用 ARGO_SERP_GUARD=0 复核。
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import threading
16
+ from typing import Any
17
+
18
+ # 判定规则与逃生门(ARGO_SERP_GUARD=0)在 serp_guard 单点;这里只登记
19
+ # 哪些引擎接了守卫(解析上游 HTML 页最易被缓存 SERP 静默命中)。
20
+ SERP_GUARD_ENGINES = frozenset({
21
+ "local_bing", "local_baidu", "local_sogou",
22
+ })
23
+
24
+ # 进程级 + 锁。并发搜索下 drain 可能拿到彼此的条目——条目只含引擎名与
25
+ # 条数、无查询内容,跨运行误挂是无害的信息性噪音。
26
+ _LOCK = threading.Lock()
27
+ _KILLS: list[dict[str, Any]] = []
28
+
29
+
30
+ def serp_guard_apply(engine_name: str, query: str,
31
+ results: list[dict[str, Any]]) -> list[dict[str, Any]]:
32
+ """守卫接线:命中引擎且整页判垃圾 → 记账后返回 [](诚实空语义不变)。"""
33
+ if not results or engine_name not in SERP_GUARD_ENGINES:
34
+ return results
35
+ try:
36
+ from serp_guard import is_junk_serp
37
+ except ImportError:
38
+ return results # fail-open:守卫自身故障不放大成搜索故障
39
+ if is_junk_serp(query, results):
40
+ with _LOCK:
41
+ _KILLS.append({"engine": engine_name, "results": len(results)})
42
+ return []
43
+ return results
44
+
45
+
46
+ def drain_serp_guard_kills() -> list[dict[str, Any]]:
47
+ """取走并清空本进程累计的整页击杀记录(search 层专用)。"""
48
+ with _LOCK:
49
+ kills = list(_KILLS)
50
+ _KILLS.clear()
51
+ return kills