argo-search 1.0.1 → 2.9.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (463) hide show
  1. package/LICENSE +201 -21
  2. package/README.en.md +822 -0
  3. package/README.es.md +864 -0
  4. package/README.ja.md +864 -0
  5. package/README.ko.md +864 -0
  6. package/README.md +867 -320
  7. package/SKILL.md +135 -0
  8. package/assets/readme/hero.svg +121 -0
  9. package/assets/readme/made-with-beautify.svg +10 -0
  10. package/assets/readme/proof-routes.svg +73 -0
  11. package/assets/readme/settings-keys.png +0 -0
  12. package/assets/readme/settings-tools.png +0 -0
  13. package/assets/readme/why-better.svg +104 -0
  14. package/assets/readme/workflow.svg +83 -0
  15. package/backends/domain_profiles.json +2308 -96
  16. package/backends/engine_registry.yaml +3314 -503
  17. package/backends/query_synonyms_cn.json +35 -0
  18. package/backends/quota_profiles.json +2409 -161
  19. package/backends/source_types_cn.json +83 -0
  20. package/backends/text_lexicon.json +44 -0
  21. package/bin/argo +703 -0
  22. package/bin/argo.js +71 -39
  23. package/config.yaml +4976 -363
  24. package/cordis.patch.yml +21 -0
  25. package/data/countries.json +40 -0
  26. package/data/lang_matrix/lang_capability.json +842 -0
  27. package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
  28. package/data/regions_cn.json +1 -0
  29. package/docs/ADDING_NEW_ENGINE.md +184 -0
  30. package/docs/ARGO_INTRO.md +124 -0
  31. package/docs/DATA_SOURCE_GAPS.md +81 -0
  32. package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
  33. package/docs/ENGINE_CATALOG.md +564 -0
  34. package/docs/MCP_SETUP.md +106 -0
  35. package/docs/PLATFORM_PATHS.md +111 -0
  36. package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
  37. package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
  38. package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
  39. package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
  40. package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
  41. package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
  42. package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
  43. package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
  44. package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
  45. package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
  46. package/docs/SEARCH_ARCHIVE.md +115 -0
  47. package/docs/assets/architecture.svg +104 -0
  48. package/docs/assets/hero.svg +98 -0
  49. package/docs/engines/bangumi.md +27 -0
  50. package/docs/engines/biorxiv.md +27 -0
  51. package/docs/engines/clawhub.md +27 -0
  52. package/docs/engines/crt_sh.md +27 -0
  53. package/docs/engines/datacite.md +27 -0
  54. package/docs/engines/deps_dev.md +27 -0
  55. package/docs/engines/douban_movie.md +27 -0
  56. package/docs/engines/endoflife.md +27 -0
  57. package/docs/engines/flk_law.md +27 -0
  58. package/docs/engines/fxtwitter.md +27 -0
  59. package/docs/engines/google_news.md +27 -0
  60. package/docs/engines/gov_regulations.md +27 -0
  61. package/docs/engines/guqian.md +27 -0
  62. package/docs/engines/hackernews.md +27 -0
  63. package/docs/engines/iplant.md +27 -0
  64. package/docs/engines/keenable.md +27 -0
  65. package/docs/engines/met_museum.md +27 -0
  66. package/docs/engines/nasa_images.md +27 -0
  67. package/docs/engines/netease_music.md +27 -0
  68. package/docs/engines/npm.md +27 -0
  69. package/docs/engines/nvd.md +27 -0
  70. package/docs/engines/openreview.md +27 -0
  71. package/docs/engines/openstd.md +27 -0
  72. package/docs/engines/parallel_free.md +27 -0
  73. package/docs/engines/people_daily.md +27 -0
  74. package/docs/engines/realtime_index.md +27 -0
  75. package/docs/engines/sec_edgar.md +27 -0
  76. package/docs/engines/seltz.md +27 -0
  77. package/docs/engines/skillsmp.md +27 -0
  78. package/docs/engines/sspai.md +27 -0
  79. package/docs/engines/std_samr.md +27 -0
  80. package/docs/engines/train.md +27 -0
  81. package/docs/engines/un_comtrade.md +27 -0
  82. package/docs/engines/wikisource.md +27 -0
  83. package/docs/engines/zdic.md +27 -0
  84. package/docs/engines/zenodo.md +27 -0
  85. package/docs/engines/zhihu_hot_app.md +27 -0
  86. package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
  87. package/engines/README.md +10 -0
  88. package/engines/_template_cli.yaml +39 -0
  89. package/engines/_template_http.yaml +37 -0
  90. package/engines/plugins/_template_plugin.py +50 -0
  91. package/engines/specs/_cisa_kev.yaml +54 -0
  92. package/engines/specs/_osv.yaml +63 -0
  93. package/engines/specs/aljazeera.yaml +43 -0
  94. package/engines/specs/antara.yaml +43 -0
  95. package/engines/specs/artic.yaml +20 -0
  96. package/engines/specs/aviation_weather.yaml +39 -0
  97. package/engines/specs/bangumi.yaml +21 -0
  98. package/engines/specs/biorxiv.yaml +24 -0
  99. package/engines/specs/carbon_intensity.yaml +32 -0
  100. package/engines/specs/cinii.yaml +44 -0
  101. package/engines/specs/clawhub.yaml +49 -0
  102. package/engines/specs/cleveland.yaml +19 -0
  103. package/engines/specs/cn_ai_news.yaml +43 -0
  104. package/engines/specs/core.yaml +35 -0
  105. package/engines/specs/crt_sh.yaml +38 -0
  106. package/engines/specs/ct24.yaml +43 -0
  107. package/engines/specs/datacite.yaml +27 -0
  108. package/engines/specs/datagov_uk.yaml +41 -0
  109. package/engines/specs/ddgs_images.yaml +45 -0
  110. package/engines/specs/deepseek_search.yaml +74 -0
  111. package/engines/specs/deezer.yaml +20 -0
  112. package/engines/specs/deps_dev.yaml +21 -0
  113. package/engines/specs/digup.yaml +36 -0
  114. package/engines/specs/doi.yaml +39 -0
  115. package/engines/specs/douban_movie.yaml +22 -0
  116. package/engines/specs/dw_news.yaml +47 -0
  117. package/engines/specs/egov_law.yaml +21 -0
  118. package/engines/specs/elpais.yaml +43 -0
  119. package/engines/specs/endoflife.yaml +21 -0
  120. package/engines/specs/energy_charts.yaml +21 -0
  121. package/engines/specs/factcheck_org.yaml +52 -0
  122. package/engines/specs/faz.yaml +43 -0
  123. package/engines/specs/federal_register.yaml +57 -0
  124. package/engines/specs/firecrawl.yaml +53 -0
  125. package/engines/specs/flk_law.yaml +24 -0
  126. package/engines/specs/folha.yaml +43 -0
  127. package/engines/specs/france24.yaml +44 -0
  128. package/engines/specs/full_fact.yaml +47 -0
  129. package/engines/specs/fxtwitter.yaml +31 -0
  130. package/engines/specs/gbfs_nyc.yaml +32 -0
  131. package/engines/specs/gdacs.yaml +21 -0
  132. package/engines/specs/gdelt.yaml +64 -0
  133. package/engines/specs/google_news.yaml +22 -0
  134. package/engines/specs/gov_regulations.yaml +38 -0
  135. package/engines/specs/guardian_rss.yaml +59 -0
  136. package/engines/specs/hurriyet.yaml +43 -0
  137. package/engines/specs/iplant.yaml +25 -0
  138. package/engines/specs/jikan.yaml +24 -0
  139. package/engines/specs/jolpica.yaml +20 -0
  140. package/engines/specs/k10plus.yaml +20 -0
  141. package/engines/specs/keenable.yaml +44 -0
  142. package/engines/specs/lefigaro.yaml +43 -0
  143. package/engines/specs/listenbrainz.yaml +19 -0
  144. package/engines/specs/matichon.yaml +43 -0
  145. package/engines/specs/met_museum.yaml +21 -0
  146. package/engines/specs/nasa_images.yaml +45 -0
  147. package/engines/specs/netease_music.yaml +38 -0
  148. package/engines/specs/nhk.yaml +43 -0
  149. package/engines/specs/nhtsa_vpic.yaml +20 -0
  150. package/engines/specs/noaa_swpc.yaml +20 -0
  151. package/engines/specs/nvd.yaml +38 -0
  152. package/engines/specs/obis.yaml +20 -0
  153. package/engines/specs/opencitations.yaml +52 -0
  154. package/engines/specs/openf1.yaml +21 -0
  155. package/engines/specs/openfoodfacts.yaml +45 -0
  156. package/engines/specs/openligadb.yaml +20 -0
  157. package/engines/specs/openreview.yaml +37 -0
  158. package/engines/specs/openstd.yaml +21 -0
  159. package/engines/specs/people_daily.yaml +22 -0
  160. package/engines/specs/realtime_index.yaml +34 -0
  161. package/engines/specs/ror.yaml +20 -0
  162. package/engines/specs/satnogs.yaml +20 -0
  163. package/engines/specs/sec_edgar.yaml +46 -0
  164. package/engines/specs/skillsmp.yaml +50 -0
  165. package/engines/specs/sogou_weixin_browser.yaml +30 -0
  166. package/engines/specs/soilgrids.yaml +25 -0
  167. package/engines/specs/sspai.yaml +36 -0
  168. package/engines/specs/stackexchange.yaml +46 -0
  169. package/engines/specs/std_samr.yaml +21 -0
  170. package/engines/specs/tass.yaml +43 -0
  171. package/engines/specs/tle_mirror.yaml +21 -0
  172. package/engines/specs/train.yaml +37 -0
  173. package/engines/specs/tvmaze.yaml +20 -0
  174. package/engines/specs/twitter_syndication.yaml +31 -0
  175. package/engines/specs/un_comtrade.yaml +23 -0
  176. package/engines/specs/unpaywall.yaml +54 -0
  177. package/engines/specs/vnpress.yaml +43 -0
  178. package/engines/specs/weather.yaml +32 -0
  179. package/engines/specs/who_don.yaml +21 -0
  180. package/engines/specs/who_gho.yaml +20 -0
  181. package/engines/specs/wikimedia_commons.yaml +34 -0
  182. package/engines/specs/wikisource.yaml +22 -0
  183. package/engines/specs/worms.yaml +19 -0
  184. package/engines/specs/x_trending.yaml +34 -0
  185. package/engines/specs/yna.yaml +43 -0
  186. package/engines/specs/ynet.yaml +43 -0
  187. package/engines/specs/zbmath.yaml +40 -0
  188. package/engines/specs/zdic.yaml +22 -0
  189. package/engines/specs/zenodo.yaml +27 -0
  190. package/engines/specs/zhihu_hot_app.yaml +35 -0
  191. package/mcp/clients.yaml +74 -0
  192. package/package.json +51 -12
  193. package/packages/dsh-plugin/README.md +134 -0
  194. package/packages/dsh-plugin/cordis.patch.yml +48 -0
  195. package/packages/dsh-plugin/dsh/client.js +1244 -0
  196. package/packages/dsh-plugin/dsh/index.js +2290 -0
  197. package/packages/dsh-plugin/dsh/native-tools.mjs +743 -0
  198. package/packages/dsh-plugin/package.json +38 -0
  199. package/packages/dsh-plugin/test/client.test.js +232 -0
  200. package/packages/dsh-plugin/test/config.test.js +30 -0
  201. package/packages/dsh-plugin/test/index.test.js +1338 -0
  202. package/references/query-templates.md +96 -0
  203. package/references/query-templates.yaml +118 -0
  204. package/scripts/ab_eval.py +219 -0
  205. package/scripts/ab_eval_p0p1.py +266 -0
  206. package/scripts/adaptive.py +271 -70
  207. package/scripts/answer.py +154 -0
  208. package/scripts/archive_run.py +591 -0
  209. package/scripts/argo_engine_registry.py +172 -0
  210. package/scripts/argo_paths.py +728 -0
  211. package/scripts/article.py +136 -0
  212. package/scripts/arxiv_source.py +201 -0
  213. package/scripts/badcase_ledger.py +252 -0
  214. package/scripts/batch_probe.py +241 -0
  215. package/scripts/benchmark.py +19 -0
  216. package/scripts/bounded_run.py +144 -0
  217. package/scripts/browser_auth.py +323 -0
  218. package/scripts/burstiness_detector.py +147 -0
  219. package/scripts/cache.py +1078 -67
  220. package/scripts/cache_guard.py +271 -0
  221. package/scripts/cache_key_vdom.py +56 -0
  222. package/scripts/cache_similarity.py +170 -0
  223. package/scripts/caliber_isolation.py +195 -0
  224. package/scripts/calibrate_lowquality.py +210 -0
  225. package/scripts/candidate_envelope.py +366 -0
  226. package/scripts/chrome_cdp.py +871 -0
  227. package/scripts/circuit_breaker.py +389 -0
  228. package/scripts/citations.py +334 -0
  229. package/scripts/cite_cli.py +59 -0
  230. package/scripts/cjk_tokens.py +95 -0
  231. package/scripts/clarify.py +47 -24
  232. package/scripts/cli_io.py +240 -0
  233. package/scripts/config.py +683 -45
  234. package/scripts/config_cache.py +402 -0
  235. package/scripts/conn_pool.py +220 -0
  236. package/scripts/content_security.py +685 -0
  237. package/scripts/content_signals.py +730 -0
  238. package/scripts/crawl.py +74 -35
  239. package/scripts/daemon.py +866 -0
  240. package/scripts/digup_bridge.py +214 -0
  241. package/scripts/engine_admission.py +366 -0
  242. package/scripts/engine_dispatch.py +1019 -0
  243. package/scripts/engine_env.py +534 -0
  244. package/scripts/engine_failure.py +411 -0
  245. package/scripts/engine_families.py +732 -0
  246. package/scripts/engine_policy.py +218 -0
  247. package/scripts/engine_requires.py +190 -0
  248. package/scripts/engine_status.py +322 -0
  249. package/scripts/engine_validate.py +638 -0
  250. package/scripts/engines.py +778 -294
  251. package/scripts/engines_base.py +1521 -0
  252. package/scripts/engines_builders.py +298 -0
  253. package/scripts/engines_builders_batch10.py +159 -0
  254. package/scripts/engines_builders_batch11.py +174 -0
  255. package/scripts/engines_builders_batch12.py +103 -0
  256. package/scripts/engines_builders_batch13.py +340 -0
  257. package/scripts/engines_builders_batch9.py +1468 -0
  258. package/scripts/engines_builders_cn.py +2283 -0
  259. package/scripts/engines_builders_data.py +2923 -0
  260. package/scripts/engines_builders_data_macro.py +775 -0
  261. package/scripts/engines_builders_feeds.py +255 -0
  262. package/scripts/engines_builders_intl.py +1239 -0
  263. package/scripts/engines_builders_search.py +440 -0
  264. package/scripts/engines_builders_tech.py +1360 -0
  265. package/scripts/evidence.py +588 -162
  266. package/scripts/evidence_loop.py +592 -0
  267. package/scripts/evidence_tier.py +298 -0
  268. package/scripts/except_sets.py +25 -0
  269. package/scripts/extract.py +55 -16
  270. package/scripts/fact_align.py +207 -0
  271. package/scripts/fact_crosscheck.py +230 -0
  272. package/scripts/fetch.py +25 -12
  273. package/scripts/fetch_browser.py +87 -0
  274. package/scripts/fetch_jsrun.py +122 -0
  275. package/scripts/fetch_quality.py +225 -0
  276. package/scripts/fetch_render_tinyfish.py +117 -0
  277. package/scripts/fetch_v3.py +1779 -0
  278. package/scripts/focus_extract.py +350 -0
  279. package/scripts/fulltext_store.py +169 -0
  280. package/scripts/fxtwitter_api.py +325 -0
  281. package/scripts/gen_engine_catalog.py +364 -0
  282. package/scripts/gen_native_tools.py +91 -0
  283. package/scripts/gen_routing_snapshot.py +184 -0
  284. package/scripts/health_check.py +140 -402
  285. package/scripts/health_probe.py +23 -8
  286. package/scripts/hot_state.py +134 -0
  287. package/scripts/http_client.py +1053 -0
  288. package/scripts/image_ops.py +430 -0
  289. package/scripts/image_vision/vision_probe +0 -0
  290. package/scripts/image_vision/vision_probe.swift +126 -0
  291. package/scripts/install.ps1 +148 -0
  292. package/scripts/install.sh +206 -0
  293. package/scripts/job.py +1186 -0
  294. package/scripts/jump_resolver.py +235 -0
  295. package/scripts/lang_capability.py +150 -0
  296. package/scripts/lang_detect.py +289 -0
  297. package/scripts/lang_matrix_probe.py +329 -0
  298. package/scripts/lang_pref.py +311 -0
  299. package/scripts/link_source.py +340 -0
  300. package/scripts/local_engines.py +321 -0
  301. package/scripts/local_image.py +970 -0
  302. package/scripts/local_seek.py +366 -0
  303. package/scripts/macro_countries.py +82 -0
  304. package/scripts/matrix_search_eval.py +1105 -0
  305. package/scripts/mcp_diag.py +8 -3
  306. package/scripts/mcp_handlers.py +1074 -0
  307. package/scripts/mcp_handlers_surface.py +157 -0
  308. package/scripts/mcp_launch.sh +122 -0
  309. package/scripts/mcp_server.py +118 -417
  310. package/scripts/mcp_setup.py +407 -0
  311. package/scripts/mcp_tools.py +330 -0
  312. package/scripts/mcp_transport.py +191 -0
  313. package/scripts/min_version_scan.py +335 -0
  314. package/scripts/net_proxy.py +258 -0
  315. package/scripts/network_aware.py +152 -0
  316. package/scripts/paper.py +274 -0
  317. package/scripts/pdf_extract.py +518 -0
  318. package/scripts/phrase_filter.py +66 -0
  319. package/scripts/plan.py +422 -0
  320. package/scripts/postprocess_signals.py +150 -0
  321. package/scripts/pre-commit +48 -0
  322. package/scripts/query_enhance.py +157 -0
  323. package/scripts/query_operators.py +196 -0
  324. package/scripts/query_rewriter.py +116 -7
  325. package/scripts/query_signals.py +300 -0
  326. package/scripts/query_templates.py +167 -0
  327. package/scripts/query_understanding.py +411 -0
  328. package/scripts/query_variants.py +298 -0
  329. package/scripts/quota.py +488 -76
  330. package/scripts/rank_signals.py +775 -0
  331. package/scripts/ranking_eval.py +319 -0
  332. package/scripts/readability_extract.py +433 -0
  333. package/scripts/recompute.py +415 -0
  334. package/scripts/recovery.py +616 -0
  335. package/scripts/recovery_l0.py +160 -0
  336. package/scripts/redskill/redskill_engine.py +332 -0
  337. package/scripts/regression_p0p1.py +318 -0
  338. package/scripts/replay_eval.py +640 -0
  339. package/scripts/rerank_tfidf.py +225 -0
  340. package/scripts/research.py +591 -403
  341. package/scripts/research_cli.py +361 -0
  342. package/scripts/research_dossier.py +492 -0
  343. package/scripts/research_expand.py +146 -0
  344. package/scripts/research_gates.py +272 -0
  345. package/scripts/research_report.py +209 -0
  346. package/scripts/research_strategy.py +37 -0
  347. package/scripts/research_work_packages.py +195 -0
  348. package/scripts/result_lang.py +311 -0
  349. package/scripts/robots_guard.py +259 -0
  350. package/scripts/route.py +921 -254
  351. package/scripts/route_bangs.py +40 -0
  352. package/scripts/route_cache.py +225 -0
  353. package/scripts/route_combo.py +910 -0
  354. package/scripts/route_domains.py +328 -0
  355. package/scripts/route_lang.py +605 -0
  356. package/scripts/route_log.py +63 -0
  357. package/scripts/route_policy.py +250 -0
  358. package/scripts/rrf_classic.py +66 -0
  359. package/scripts/screenshot.py +73 -0
  360. package/scripts/search.py +897 -405
  361. package/scripts/search_attribution.py +88 -0
  362. package/scripts/search_benchmark.py +407 -0
  363. package/scripts/search_cli.py +627 -0
  364. package/scripts/search_entry.py +389 -0
  365. package/scripts/search_output.py +606 -0
  366. package/scripts/search_pipeline.py +1077 -0
  367. package/scripts/search_rank.py +991 -0
  368. package/scripts/search_types.py +27 -12
  369. package/scripts/seek_locator.py +68 -0
  370. package/scripts/semantic_evidence.py +215 -0
  371. package/scripts/serp_guard.py +228 -0
  372. package/scripts/serp_kill_log.py +51 -0
  373. package/scripts/single_flight.py +120 -0
  374. package/scripts/social_engines/__init__.py +10 -0
  375. package/scripts/social_engines/bilibili_engine.py +131 -0
  376. package/scripts/social_engines/hackernews_engine.py +113 -0
  377. package/scripts/social_engines/reddit_engine.py +172 -0
  378. package/scripts/social_engines/twitter_engine.py +397 -0
  379. package/scripts/social_engines/v2ex_engine.py +101 -0
  380. package/scripts/social_engines/weibo_engine.py +128 -0
  381. package/scripts/social_engines/xiaohongshu_engine.py +137 -0
  382. package/scripts/social_engines/zhihu_engine.py +139 -0
  383. package/scripts/social_research.py +201 -0
  384. package/scripts/sogou_weixin_browser.py +318 -0
  385. package/scripts/source_ledger.py +249 -0
  386. package/scripts/stage_timing.py +58 -0
  387. package/scripts/stats_cli.py +95 -0
  388. package/scripts/straggler_policy.py +60 -0
  389. package/scripts/stylometry_detector.py +231 -0
  390. package/scripts/sync_backends.py +353 -0
  391. package/scripts/sync_installs.py +29 -0
  392. package/scripts/template_cli.py +127 -0
  393. package/scripts/text_lexicon.py +72 -0
  394. package/scripts/tfidf_router.py +109 -37
  395. package/scripts/time_utils.py +233 -0
  396. package/scripts/topic_research_profiles.py +643 -0
  397. package/scripts/train.py +737 -0
  398. package/scripts/tweet.py +596 -0
  399. package/scripts/url_canon.py +217 -0
  400. package/scripts/url_optimize.py +56 -0
  401. package/scripts/url_safety.py +236 -0
  402. package/scripts/usage_log.py +116 -0
  403. package/scripts/usage_log_clean.py +113 -0
  404. package/scripts/v2ex_nodes.py +311 -0
  405. package/scripts/variant_recall.py +188 -0
  406. package/scripts/watch.py +205 -0
  407. package/scripts/wechat_article.py +258 -0
  408. package/scripts/wide_research.py +402 -0
  409. package/scripts/wx.py +349 -0
  410. package/scripts/x_article_fetch.py +141 -0
  411. package/scripts/x_search.py +234 -0
  412. package/scripts/x_trending.py +189 -0
  413. package/scripts/yaml_load.py +67 -0
  414. package/sub-skills/ego-search/SKILL.md +253 -0
  415. package/sub-skills/ego-search/references/browser-runtime.md +43 -0
  416. package/sub-skills/ego-search/references/install.md +67 -0
  417. package/sub-skills/ego-search/references/original-ego-upgrade.md +51 -0
  418. package/sub-skills/ego-search/scripts/ego_search.py +800 -0
  419. package/sub-skills/ego-search/scripts/merge.py +226 -0
  420. package/sub-skills/ego-search/scripts/quality.py +64 -0
  421. package/sub-skills/ego-search/scripts/runtime.py +172 -0
  422. package/sub-skills/ego-search/scripts/safety.py +109 -0
  423. package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
  424. package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
  425. package/sub-skills/ego-search/tests/test_query_normalize.py +68 -0
  426. package/sub-skills/js-run/SKILL.md +67 -0
  427. package/sub-skills/js-run/scripts/jsrun.py +112 -0
  428. package/sub-skills/js-run/scripts/shim.js +188 -0
  429. package/sub-skills/js-run/tests/fixtures/challenge_like.js +27 -0
  430. package/sub-skills/js-run/tests/test_jsrun.py +165 -0
  431. package/sub-skills/local-search/SKILL.md +50 -53
  432. package/sub-skills/local-search/bm25.py +162 -0
  433. package/sub-skills/local-search/config.yaml +152 -55
  434. package/sub-skills/local-search/engine_registry.py +91 -26
  435. package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
  436. package/sub-skills/local-search/local_search_adapter.py +13 -2
  437. package/sub-skills/local-search/parse_maps.yaml +40 -42
  438. package/sub-skills/local-search/search_v3.py +1182 -65
  439. package/sub-skills/local-search/smart_router.py +47 -7
  440. package/sub-skills/local-seek/SKILL.md +110 -0
  441. package/sub-skills/local-seek/config/domains.yaml +20 -0
  442. package/sub-skills/local-seek/references/auto-behaviors.md +24 -0
  443. package/sub-skills/local-seek/references/fd-mdfind-recipes.md +62 -0
  444. package/sub-skills/local-seek/references/git-integration.md +54 -0
  445. package/sub-skills/local-seek/references/remote-handoff.md +46 -0
  446. package/sub-skills/local-seek/references/rg-recipes.md +96 -0
  447. package/sub-skills/local-seek/references/strategies.md +52 -0
  448. package/sub-skills/local-seek/references/structural-search.md +59 -0
  449. package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
  450. package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
  451. package/sub-skills/local-seek/scripts/seek.py +1465 -0
  452. package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
  453. package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
  454. package/sub-skills/local-seek/scripts/seek_rank.py +193 -0
  455. package/sub-skills/local-seek/tests/test_content_rank.py +222 -0
  456. package/sub-skills/local-seek/tests/test_determinism_20261006.py +145 -0
  457. package/sub-skills/local-seek/tests/test_noise_tier.py +101 -0
  458. package/sub-skills/local-seek/tests/test_pcre2_probe.py +136 -0
  459. package/sub-skills/local-seek/tests/test_rank_pinyin.py +77 -0
  460. package/sub-skills/local-seek/tests/test_regression_20261003.py +169 -0
  461. package/sub-skills/local-seek/tests/test_regression_20261006.py +146 -0
  462. package/sub-skills/local-seek/tests/test_seek_degradation.py +111 -0
  463. package/sub-skills/local-seek/tests/test_seek_fts.py +313 -0
@@ -0,0 +1,271 @@
1
+ #!/usr/bin/env python3
2
+ """cache_guard.py — 公共缓存的写入守卫(登录态 + 上游退化)。
3
+
4
+ 从 cache.py 拆出(2026-09-27,方案 C1)。拆的理由是纪律性的:cache.py 已在
5
+ tests/test_module_size_gate.py 的 GRANDFATHERED 里登记(1362 行上限),
6
+ 任何净增长都会被门禁拦下——而「写缓存前该不该写」这条判断不属于缓存的存储
7
+ 实现,属于「缓存的准入策略」,两者混在一起只会让 cache.py 继续变大。
8
+
9
+ ## 两条守卫
10
+
11
+ **登录态守卫**(本体与异常类都在 cache.py):登录态检索的结果不得进入
12
+ 公共缓存,避免一个账号的私有结果被所有人共享。
13
+
14
+ **退化守卫**:上游抖动返回的残次品不得写入。
15
+ 实锤事故见 is_degraded_results 的 docstring——一次上游把整句当单词查、
16
+ 返回 8 条词典释义,相关性全为 0.1429,却照常缓存并从此固化。
17
+
18
+ 两个守卫的异常类**不在同一个模块**:cache.py 模块级 `from cache_guard
19
+ import ...`,所以这里只能用函数级 import 去取 LoginCacheRejected,模块级
20
+ 会成环。统一入口是 attempt_cache_write,调用方不必自己认这两个类。
21
+
22
+ ## 为什么退化守卫在「写入」这一层而不是「返回」那一层
23
+
24
+ 退化结果**当场返回**是有价值的:它至少让调用方看到「这次没搜到」,
25
+ 且恢复路径是重打一次(无缓存命中,自然重新请求上游)。
26
+ 但它**不该被缓存**:一旦缓存,退化就从「一次抖动」变成「永久结果」——
27
+ 后续每次命中缓存都复现,且没有任何信号提示它曾经退化过。
28
+ 写入层拦截是唯一能切断这个固化的位置。
29
+
30
+ ## 调用方必须走 attempt_cache_write,不要裸调 cache.set
31
+
32
+ 「不写缓存」和「检索失败」是两件事。守卫抛出的异常如果穿透到调用主路径,
33
+ 代价是一次检索白跑——而且比白跑更糟,见 attempt_cache_write 的 docstring。
34
+ """
35
+ from __future__ import annotations
36
+
37
+ from collections.abc import Callable
38
+ from typing import Any
39
+
40
+ from except_sets import OPT_IMPORT
41
+
42
+
43
+ class DegradedCacheRejected(ValueError):
44
+ """退化结果(低相关/查询词回声)禁止写入公共 SearchCache。"""
45
+
46
+
47
+ class FailedStateCacheRejected(ValueError):
48
+ """配置/状态类失败(缺密钥/熔断/鉴权/配额)且无有效结果:不写负缓存。
49
+
50
+ 与退化守卫的分工:退化拦「搜到了但内容是垃圾」;本守卫拦「根本没搜成、
51
+ 且换个人/等一会/改个配置就能好」。把后者缓存下来,等于把「argo 此刻
52
+ 没配对」固化成「这个查询没有答案」——实测形态(2026-09-28):
53
+ `--engine parallel` 缺 key → L2 落 {"results": [], "ttl": 45},用户按
54
+ 文档配好 PARALLEL_API_KEY 后同一查询仍 cached=true 并回放
55
+ 「PARALLEL_API_KEY 未设置」,与 issue #12 报告人的体验逐字同构。
56
+ """
57
+
58
+
59
+ # 配置/状态类失败的状态名(engine_dispatch.classify_engine_outcome 词汇表)。
60
+ # 网络类失败(timeout/blocked/rate-limited/no-results)刻意不收——它们中
61
+ # 确实混有「真的没有」,负缓存是既有设计(EMPTY_RESULT_TTL),等一会重试
62
+ # 由 TTL 自然兜底。本类的共同点:**换环境就能好**,缓存它没有信息量。
63
+ _STATE_FAIL_STATUSES = frozenset({
64
+ "skipped-missing-env", "skipped-circuit-open", "auth-failed",
65
+ "quota-exhausted", "skipped-quota-exhausted",
66
+ })
67
+
68
+ # 配额耗尽错误关键词(唯一来源):classify_engine_outcome 的 quota-exhausted
69
+ # 分类与自适应学习跳过逻辑共用。新增配额错误码(如新的 API 业务码)只改这里。
70
+ _QUOTA_ERROR_KEYWORDS = ("quota", "10406", "insufficient balance",
71
+ "insufficient funds", "not enough money",
72
+ "余额不足", "额度不足", "配额耗尽")
73
+
74
+ # builder 路径漏到 status="error" 的缺密钥形态:路由层 env 拦截覆盖不到
75
+ # 自定义 required_env 的引擎(search.py:192-209 docstring 自认的缺口),
76
+ # issue #12 的 parallel/seltz/you 就读裸名——状态层 env_ready=True、执行层
77
+ # 取不到值。按 detail 文本兜底识别。
78
+ _MISSING_ENV_DETAIL_KEYWORDS = (
79
+ "未设置", "api_key", "api key", "apikey", "环境变量",
80
+ )
81
+
82
+
83
+ def is_state_failure_outcome(outcome: Any) -> bool:
84
+ """单引擎结局是否「配置/状态类失败」(可修复,不该负缓存或记入学习器失败)。"""
85
+ if not isinstance(outcome, dict):
86
+ return False
87
+ status = str(outcome.get("status") or "")
88
+ if status in _STATE_FAIL_STATUSES:
89
+ return True
90
+ if status == "error":
91
+ detail = str(outcome.get("detail") or "").lower()
92
+ return any(k in detail for k in _MISSING_ENV_DETAIL_KEYWORDS)
93
+ return False
94
+
95
+
96
+ def should_record_adaptive(outcome: dict[str, Any] | None,
97
+ res: list[dict[str, Any]]) -> bool:
98
+ """单引擎结局是否应记入 adaptive 学习器(跳过早停弃置与配额/鉴权/缺密钥等状态类故障)。"""
99
+ st = str((outcome or {}).get("status") or "")
100
+ if st.startswith(("cancelled", "skipped-")) or is_state_failure_outcome(outcome):
101
+ return False
102
+ errors = [str(r.get("error", "")) for r in (res or ())
103
+ if isinstance(r, dict) and "error" in r]
104
+ if errors and all(
105
+ any(k in msg.lower() for k in (
106
+ *_QUOTA_ERROR_KEYWORDS, "unauthorized", "api key",
107
+ "forbidden", "401", "403",
108
+ ))
109
+ for msg in errors
110
+ ):
111
+ return False
112
+ return True
113
+
114
+
115
+ def failed_state_reason(payload: object) -> str | None:
116
+ """载荷是否「整体没搜成且原因可修复」——是则返回理由(否则 None)。
117
+
118
+ 判据:**没有任何有效结果**、且**至少一个**引擎结局是配置/状态类失败。
119
+ 两条边界的理由:
120
+
121
+ - 有有效结果时不拦:个别引擎的配置失败不毒化整批(照常缓存);
122
+ - 「至少一个」而不是「全部」:缓存命中会**原样回放 engine_outcomes**
123
+ 并由 _collect_errors 重建 errors——混合场景里那条「未设置」会在
124
+ 用户配好 key 后继续重放 45s,与单引擎形态同构。连带的代价只是
125
+ 别的引擎少一份 45s 负缓存(重试一次超时引擎本就无妨)。
126
+
127
+ 网络类失败单独出现时不拦(见 _STATE_FAIL_STATUSES 注释)。
128
+ """
129
+ if not isinstance(payload, dict):
130
+ return None
131
+ results = payload.get("results")
132
+ if isinstance(results, list) and any(
133
+ isinstance(r, dict) and "error" not in r for r in results):
134
+ return None
135
+ outcomes = payload.get("engine_outcomes")
136
+ if isinstance(outcomes, dict):
137
+ outcomes = [outcomes] # 单引擎直写形态(CLI --engine 路径)
138
+ if not isinstance(outcomes, list) or not outcomes:
139
+ return None
140
+ failed = [o for o in outcomes if is_state_failure_outcome(o)]
141
+ if failed:
142
+ kinds = sorted({str(o.get("status") or "?") for o in failed
143
+ if isinstance(o, dict)})
144
+ return (f"{len(failed)}/{len(outcomes)} 个引擎为配置/状态类失败"
145
+ f"({'/'.join(kinds)})且无有效结果,不写负缓存")
146
+ return None
147
+
148
+
149
+ def assert_not_failed_state(payload: object, *, context: str) -> None:
150
+ """配置/状态类失败硬拒绝:宁可下次重打,也不把「没配对」固化成「没答案」。"""
151
+ reason = failed_state_reason(payload)
152
+ if reason:
153
+ raise FailedStateCacheRejected(f"{context}: {reason}")
154
+
155
+
156
+ def cache_write_rejections() -> tuple[type[Exception], ...]:
157
+ """写入守卫会抛的全部异常类:退化 + 失败态(本模块)+ 登录态(cache.py)。
158
+
159
+ 登录态那一个只能函数级取:cache.py 模块级 import 本模块,反向 import
160
+ 会成环。取不到(cache 被换掉/裁掉)时只留本模块守卫——少认一种异常最多
161
+ 让那一路退回旧行为,不该反过来让调用方炸掉。
162
+ """
163
+ try:
164
+ from cache import LoginCacheRejected
165
+ except OPT_IMPORT:
166
+ return (DegradedCacheRejected, FailedStateCacheRejected)
167
+ return (DegradedCacheRejected, FailedStateCacheRejected, LoginCacheRejected)
168
+
169
+
170
+ def attempt_cache_write(write: Callable[[], Any], *,
171
+ context: str = "cache") -> str | None:
172
+ """执行一次缓存写入;被写入守卫拒绝时返回异常类别名,而不是抛出。
173
+
174
+ 守卫的职责是「不固化」,不是「不返回」——两者在调用主路径上必须分开:
175
+
176
+ - **combo 写入点**(search_pipeline.finalize):异常穿透会取消整个
177
+ 查询。2026-09-27 实锤——`Apple Inc 10-K annual report 2025` 的 5 条
178
+ 结果里 4 条 relevance=0 触发退化守卫,CLI 直接 traceback 退出,
179
+ **已检索到的结果全部丢弃**(这批结果本身是有用的:第 1 条正是
180
+ Apple 的 CIK 归档页)。
181
+ - **per-engine 写入点**(engine_dispatch):异常被 daemon 线程的兜底
182
+ except 接住,后果更隐蔽——那次**成功的**引擎调用被改写成
183
+ `status=error` 且结果清空,还按 kind=error 记账进熔断器,把一个
184
+ 健康引擎推向 auto-disable。
185
+
186
+ 返回值为 None 表示写入成功(或本就没有可写内容);返回字符串表示被
187
+ 拒绝,值即异常类别名(如 `"DegradedCacheRejected"`),供调用方写进
188
+ 响应字段或日志。*context* 只用于日志措辞。
189
+ """
190
+ try:
191
+ write()
192
+ except cache_write_rejections() as exc:
193
+ import logging
194
+ logging.getLogger("unified_search").debug(
195
+ f"{context}: 缓存写入跳过({type(exc).__name__}): {exc}")
196
+ return type(exc).__name__
197
+ return None
198
+
199
+
200
+ # 退化判据的阈值。与 rank_signals 的相关性算子配套:正常结果的 rel 普遍
201
+ # ≥0.35,查询词回声(上游把整句当单词查、返回词典释义)稳定在 0.05-0.20。
202
+ # 0.25 落在两者之间,且远高于噪声、又低于任何真实命中。
203
+ DEGRADED_RELEVANCE_FLOOR = 0.25
204
+
205
+ # 「多数条目低于地板」的比例门槛。不用 1.0:上游降级时通常仍混有 1-2 条
206
+ # 正常结果(降级不是全有全无)。0.7 是「压倒性低分」与「正常长尾」的分界。
207
+ DEGRADED_BATCH_RATIO = 0.7
208
+
209
+ # 可判定条目数下限。低于此不判:样本太少时比例不可信,且宁可漏拦不误伤。
210
+ DEGRADED_MIN_SCORED = 3
211
+
212
+ # 恢复批降级写入的 TTL(秒)。恢复链成功产出的批次若整批低相关,不硬拒
213
+ # (硬拒会把「难查询的恢复成果」挡在缓存外,重复查询每次重付 dispatch+恢复
214
+ # 链全程——2026-10-17 实测同查询 21s 内 3 次恢复链),降级为 10 分钟短缓存:
215
+ # 给重复查询留命中,又不长期固化低相关结果。
216
+ RECOVERED_DEGRADED_TTL = 600
217
+
218
+
219
+ def _entry_relevance(item: Any) -> float | None:
220
+ """取单条结果的相关性;取不到返回 None。
221
+
222
+ 相关性通常不在顶层,而在 `rerank_dims` 里(本地五维 rerank 写入)。
223
+ 顶层也看一眼是为了兼容未跑 rerank 的引擎直写路径。
224
+ 踩过的坑:只读顶层会得到「全部取不到 → 一律不判」,守卫静默失效。
225
+ """
226
+ if not isinstance(item, dict):
227
+ return None
228
+ rel = item.get("relevance")
229
+ if not isinstance(rel, (int, float)) or isinstance(rel, bool):
230
+ dims = item.get("rerank_dims")
231
+ rel = dims.get("relevance") if isinstance(dims, dict) else None
232
+ if isinstance(rel, (int, float)) and not isinstance(rel, bool):
233
+ return float(rel)
234
+ return None
235
+
236
+
237
+ def is_degraded_results(results: object) -> bool:
238
+ """这批结果是否是「上游退化」——不完整命中查询词的回声。
239
+
240
+ 2026-09-27 实锤(`detecting LLM generated content farm SEO spam`):
241
+ 路由正确(anysearch + local_bing),但上游某一刻把整句当成单词去查,
242
+ 返回 8 条「detecting 这个词的词典释义」。它们的 rerank_dims.relevance
243
+ 全部是 **0.1429**(只命中查询里的 detecting 一词),却照常返回、照常
244
+ 进入排序、照常写入缓存——于是这次上游抖动被**固化**下来,之后每次命中
245
+ 缓存都复现同样的垃圾结果。funnel 看着完全正常(returned 10 → kept 8),
246
+ 所以极难察觉。
247
+
248
+ 这与「结果少」是两件事:`funnel` 只数数量,不看每条像不像答案。
249
+
250
+ 判据刻意保守——**只拦整批退化**,不拦单条低分:
251
+ - 单条低相关完全正常(长尾查询本就有弱命中),拦它会误伤真结果;
252
+ - 整批都低于地板才是信号:要么上游确实降级了,要么这个查询真的没有
253
+ 答案(此时缓存一份垃圾不如重打一次)。
254
+ """
255
+ if not isinstance(results, list) or not results:
256
+ return False
257
+ scored = [v for v in (_entry_relevance(it) for it in results) if v is not None]
258
+ if len(scored) < DEGRADED_MIN_SCORED:
259
+ # 拿不到足够的相关性数据就不判——不猜,也不用缺字段去拒绝正常写入。
260
+ return False
261
+ low = sum(1 for v in scored if v < DEGRADED_RELEVANCE_FLOOR)
262
+ return (low / len(scored)) >= DEGRADED_BATCH_RATIO
263
+
264
+
265
+ def assert_not_degraded(results: object, *, context: str) -> None:
266
+ """退化结果硬拒绝:宁可下次重打,也不要把上游抖动固化进缓存。"""
267
+ if is_degraded_results(results):
268
+ raise DegradedCacheRejected(
269
+ f"{context}: 多数条目 relevance < {DEGRADED_RELEVANCE_FLOOR},"
270
+ f"判定为上游退化,不写入公共 SearchCache"
271
+ )
@@ -0,0 +1,56 @@
1
+ """cache_key_vdom — 缓存键里的「引擎级垂直域」维度。
2
+
3
+ **为什么单独成文件**:`cache.py` 已到模块体积门禁上限(`tests/test_module_size_gate.py`
4
+ 登记 1375 行),而这层语义又必须与缓存键的其余维度待在一起。
5
+
6
+ **为什么它必须在键里**:`--domain` / `--sub_domain` 改变的是**发给引擎的请求**,
7
+ 同 query 在不同垂直域下结果集不同。此前两个开关被 argparse 接下、写进 SKILL.md
8
+ 参数表,却没有任何一层读取——请求照发、结果照回,`domain` 从未到达引擎。
9
+ 接上之后若不进键,「不限域」的结果会被当成「限定金融域」的答案发回去:
10
+ 一个静默失效的开关会变成一个静默给错答案的开关。
11
+
12
+ **两种 domain 同名不同义,刻意不合并**:
13
+
14
+ - `domain`:**路由域**(general / stock / weather…),决定选哪些源;
15
+ - `engine_domain` / `engine_sub_domain`:**引擎入参**,改变单个引擎的请求体。
16
+
17
+ 前者早就在键里;这里补的是后者。`short` 短键只用于拼 `raw`,不进 DB 列。
18
+ """
19
+ from __future__ import annotations
20
+
21
+ from typing import Any, Iterable
22
+
23
+ # kwargs 名 → 键内短名。顺序固定,保证键的构造是确定性的。
24
+ _VDOM_TAGS: tuple[tuple[str, str], ...] = (
25
+ ("engine_domain", "ed"),
26
+ ("engine_sub_domain", "esd"),
27
+ )
28
+
29
+
30
+ def cache_key_vdom(vdom: dict[str, Any] | None) -> list[tuple[str, str]]:
31
+ """把垂直域 kwargs 摊成 `[(短名, 值)]`,供缓存键拼装使用。
32
+
33
+ 未知键一律忽略(而不是透传):`_key` 的 `**vdom` 是开放签名,
34
+ 拼键只认登记过的维度,避免调用方拼错名字时静默丢维度——那正是本模块
35
+ 要消灭的那类 bug。空值 / 非字符串 / None 都被滤掉,保证「没给」
36
+ 与「给了空串」在键上等价,不会凭空分裂出两份缓存。
37
+ """
38
+ if not vdom:
39
+ return []
40
+ out: list[tuple[str, str]] = []
41
+ for name, tag in _VDOM_TAGS:
42
+ val = vdom.get(name)
43
+ if isinstance(val, str) and val:
44
+ out.append((tag, val))
45
+ return out
46
+
47
+
48
+ def has_vdom(vdom: dict[str, Any] | None) -> bool:
49
+ """是否指定了任意垂直域维度(供需要早退的调用方省一次拼键)。"""
50
+ return bool(cache_key_vdom(vdom))
51
+
52
+
53
+ def vdom_of(iterable: Iterable[tuple[str, str]]) -> dict[str, str]:
54
+ """`cache_key_vdom` 的逆(按登记顺序还原成长名 → 值)。"""
55
+ rev = {tag: name for name, tag in _VDOM_TAGS}
56
+ return {rev[tag]: val for tag, val in iterable if tag in rev}
@@ -0,0 +1,170 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ cache_similarity.py — 缓存侧的查询归一化与近重复判定(2026-10-16 从 cache.py 拆出)
4
+
5
+ 拆出原因:拆分前 cache.py 1382 行(祖父上限),是全仓仅次于源声明构建器的
6
+ 大文件,module size 门禁的既定结论是「按键 / 存储 / 命中策略三段拆,并把
7
+ 上限调低」。本模块承载三段里的「命中策略」判据层——**纯函数、零 IO、零
8
+ sqlite 依赖**:
9
+
10
+ - normalize_query 缓存键归一化(空白/全半角/大小写)
11
+ - query_similarity 字符 n-gram minhash 近似 Jaccard(软命中的判据)
12
+ - split_qualifiers 限定符(site: / keywords:)与载荷拆分
13
+ - is_freshness_sensitive_query 时效敏感查询检测(TTL 帽的输入)
14
+
15
+ 为什么整段搬而不是只搬一半:这四件事共用同一套「查询字符串 → 可比签名」
16
+ 的变换,`find_similar` 的每一次软命中判定都要依序走完它们;留一半在
17
+ cache.py 只会让「判据在哪」重新变成两处。搬走后 cache.py 只保留存储实现
18
+ (L1/L2)与 SearchCache 门面。
19
+
20
+ 兼容约定(拆模块的两条老规矩,与 cache_guard / cache_key_vdom 一致):
21
+ - cache.py 显式 re-export 本模块全部公开名,`from cache import
22
+ query_similarity` 等既有 import 点零改动;
23
+ - 测试 monkeypatch 的面仍在 cache.py(`cache.is_freshness_sensitive_query`),
24
+ 调用方经模块全局名查找,patch 照常生效。
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import functools
30
+ import re
31
+
32
+ __all__ = [
33
+ "normalize_query",
34
+ "query_similarity",
35
+ "split_qualifiers",
36
+ "is_freshness_sensitive_query",
37
+ "FRESHNESS_QUERY_RE",
38
+ ]
39
+
40
+
41
+ @functools.lru_cache(maxsize=4096)
42
+ def normalize_query(query: str) -> str:
43
+ """缓存键用查询归一化:折叠空白、全半角空格、两端 trim、小写英文字母。
44
+
45
+ 不改变语义实体大小写敏感场景时仍用 lower;中文不受影响。
46
+ 目标:同一问句不同空白/大小写命中同一 key。
47
+ """
48
+ if not query:
49
+ return ""
50
+ # 全角空格 → 半角;连续空白折叠
51
+ q = query.replace("\u3000", " ").strip()
52
+ q = re.sub(r"\s+", " ", q)
53
+ return q.casefold()
54
+
55
+
56
+ # ── 近重复查询检测(minhash 字符 n-gram)────────────────────────────────────
57
+
58
+ _NGRAM_N = 3
59
+ # 置换数。**看起来偏低,但不要单独调它**——它与下游软命中阈值 0.7 是一套
60
+ # 标定,动一个必须同时重标另一个,否则会把「检索上一年的缓存」放进来。
61
+ #
62
+ # 2026-09-19 实测记录(起因是「提到 32 更准」,结论是**保持 8 不改**):
63
+ # K=8 的估计确实有偏且方差大——四条中文对的真实 Jaccard 是
64
+ # 0.275/0.444/0.500/0.375,K=8 估成 0.250/0.750/0.375/0.375(最大误差 0.306,
65
+ # 双向发散);换 32 后为 0.281/0.438/0.562/0.375,明显贴住真值。
66
+ # 成本也不是障碍:`_signature` 带 lru_cache,比对是逐位比较(实测 0.41us/次,
67
+ # 不随 K 变),提高 K 只增加一次性签名成本(K=8:0.171ms、K=32:0.832ms 每条
68
+ # 查询,相对 1000–3000ms 的单次搜索可忽略)。
69
+ #
70
+ # 拦路石在**阈值**:0.7 正是照着 K=8 的偏高估计定的(下方 find_similar 的
71
+ # docstring 写「单字差异约 0.75」,而真值只有 0.444)。把 K 提到 32 后,要维持
72
+ # 原有软命中范围就得把阈值降到 ~0.35,而实测该阈值下
73
+ # 苹果 2025 营收 ↔ 苹果 2024 营收 = 0.469 ≥ 0.35 → 命中
74
+ # 即「今年的查询会命中去年的缓存」。这比「估计偏一点」严重得多——软命中的
75
+ # 前提是结果集可互换,年份不同则不可互换。两侧一起保持原状。
76
+ #
77
+ # 要走这条路,正确顺序是先换能区分数字/年份的判据(词级或数字感知 shingle),
78
+ # 再重标阈值;单独提 K 只是把偏差从「估计层」搬到「阈值层」。
79
+ _MINHASH_PERM = 8 # 置换数(越多越准,越少越快;8 对查询级足够)
80
+
81
+
82
+ def _ngrams(s: str, n: int = _NGRAM_N) -> set[str]:
83
+ """字符级 n-gram(中文按字,英文按字符,无需分词)。"""
84
+ s = re.sub(r"\s+", "", s)
85
+ return {s[i:i + n] for i in range(max(len(s) - n + 1, 1))}
86
+
87
+
88
+ def _hash_token(t: str, seed: int) -> int:
89
+ """带种子的简单哈希(minhash 置换模拟)。"""
90
+ h = seed * 1315423911
91
+ for c in t:
92
+ h = (h ^ ord(c)) * 1099511628211 & 0xFFFFFFFFFFFFFFFF
93
+ return h
94
+
95
+
96
+ @functools.lru_cache(maxsize=8192)
97
+ def _signature(s: str) -> tuple[int, ...]:
98
+ """查询的 minhash 签名:每个置换下全部 n-gram 的最小哈希。
99
+
100
+ 这是「查询 → 签名」的纯函数,按查询串记忆化。提出来的意义在于:
101
+ `query_similarity(q1, q2)` 会被拿去和缓存里**多条**候选比对,旧写法每次
102
+ 都重算两边的签名——一次搜索实测 28,552 次 `_hash_token` 调用,绝大多数
103
+ 是同一批 token 的重复置换。签名化后每条查询的签名全进程只算一次,
104
+ 且比对本身从 O(K × |tokens|) 降到 O(K)(K=置换数=8)。
105
+ 长驻进程(MCP server)里同一查询反复比对时收益为常数级。
106
+ """
107
+ grams = _ngrams(s)
108
+ if not grams:
109
+ return ()
110
+ return tuple(min(_hash_token(t, seed) for t in grams)
111
+ for seed in range(_MINHASH_PERM))
112
+
113
+
114
+ def query_similarity(q1: str, q2: str) -> float:
115
+ """两查询的字符 n-gram minhash 近似 Jaccard 相似度(0-1)。
116
+
117
+ 中文「苹果 2025 营收」vs「苹果 2025 年营收」这类近重复查询
118
+ 会得到高相似度(>0.7),用于语义缓存软命中。
119
+
120
+ 实现 = 两个签名逐位置相等的比例(minhash 估计 Jaccard 的标准做法)。
121
+ """
122
+ a = _signature(q1)
123
+ b = _signature(q2)
124
+ if not a or not b:
125
+ return 0.0
126
+ hits = sum(1 for x, y in zip(a, b) if x == y)
127
+ return hits / _MINHASH_PERM
128
+
129
+
130
+ # 结构化限定符(keywords:pi-package / site:zhihu.com / author:bcoe)。
131
+ # 值里不含 `/` 是刻意的:URL 的 `https:` 因此不被当成限定符——fetch/evidence
132
+ # 的 URL 近重复软命中(`.../x` 与 `.../x.md`,相似度 0.875)是既有的正确
133
+ # 行为,不能被这次修复误伤。键以字母开头则排除 `10:30` 这类时间写法。
134
+ _QUALIFIER_RE = re.compile(r"\b[A-Za-z][A-Za-z0-9_.-]*:[A-Za-z0-9_*.+-]+")
135
+
136
+
137
+ def split_qualifiers(query: str) -> tuple[frozenset[str], str]:
138
+ """把查询拆成(限定符集合,载荷)。
139
+
140
+ 限定符是**过滤器**而不是内容:两条查询的限定符集合不同,结果全集就不同,
141
+ 再高的词面相似度也不构成可互换的理由。
142
+
143
+ 为什么要拆:字符 n-gram 相似度会被长公共前缀主导。实测
144
+ `keywords:pi-package mcp` 与 `keywords:pi-package memory` 整串相似度
145
+ 0.875(远超 0.7 阈值),而判别词只占几个字符——两条不同的检索需求被判成
146
+ 近重复,软命中把 memory 的结果当成 mcp 的结果交了出去(2026-09-18 修复)。
147
+ 拆开后比的是载荷(`mcp` vs `memory`,相似度 0.000),判据恢复有效。
148
+ """
149
+ text = query or ""
150
+ quals = frozenset(q.lower() for q in _QUALIFIER_RE.findall(text))
151
+ payload = re.sub(r"\s+", " ", _QUALIFIER_RE.sub(" ", text)).strip()
152
+ return quals, payload
153
+
154
+
155
+ # 时效敏感查询正则:延迟编译(import 期编译正则对冷启动是纯税,见函数内判断)。
156
+ FRESHNESS_QUERY_RE = None
157
+
158
+
159
+ def is_freshness_sensitive_query(query: str) -> bool:
160
+ """检测查询是否时效敏感(今日/实时/盘中/快讯等)。"""
161
+ global FRESHNESS_QUERY_RE
162
+ if FRESHNESS_QUERY_RE is None:
163
+ FRESHNESS_QUERY_RE = re.compile(
164
+ r"(今日|今天|昨晚|昨夜|本周|本月|实时|即时|最新|刚刚|"
165
+ r"盘中|盘前|盘后|快讯|直播|热点新闻|头条|"
166
+ r"today|tonight|breaking|live\s*update|just\s*now|"
167
+ r"right\s*now|this\s*(morning|week|month))",
168
+ re.I,
169
+ )
170
+ return bool(FRESHNESS_QUERY_RE.search(query or ""))