argo-search 1.0.1 → 2.9.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (463) hide show
  1. package/LICENSE +201 -21
  2. package/README.en.md +822 -0
  3. package/README.es.md +864 -0
  4. package/README.ja.md +864 -0
  5. package/README.ko.md +864 -0
  6. package/README.md +867 -320
  7. package/SKILL.md +135 -0
  8. package/assets/readme/hero.svg +121 -0
  9. package/assets/readme/made-with-beautify.svg +10 -0
  10. package/assets/readme/proof-routes.svg +73 -0
  11. package/assets/readme/settings-keys.png +0 -0
  12. package/assets/readme/settings-tools.png +0 -0
  13. package/assets/readme/why-better.svg +104 -0
  14. package/assets/readme/workflow.svg +83 -0
  15. package/backends/domain_profiles.json +2308 -96
  16. package/backends/engine_registry.yaml +3314 -503
  17. package/backends/query_synonyms_cn.json +35 -0
  18. package/backends/quota_profiles.json +2409 -161
  19. package/backends/source_types_cn.json +83 -0
  20. package/backends/text_lexicon.json +44 -0
  21. package/bin/argo +703 -0
  22. package/bin/argo.js +71 -39
  23. package/config.yaml +4976 -363
  24. package/cordis.patch.yml +21 -0
  25. package/data/countries.json +40 -0
  26. package/data/lang_matrix/lang_capability.json +842 -0
  27. package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
  28. package/data/regions_cn.json +1 -0
  29. package/docs/ADDING_NEW_ENGINE.md +184 -0
  30. package/docs/ARGO_INTRO.md +124 -0
  31. package/docs/DATA_SOURCE_GAPS.md +81 -0
  32. package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
  33. package/docs/ENGINE_CATALOG.md +564 -0
  34. package/docs/MCP_SETUP.md +106 -0
  35. package/docs/PLATFORM_PATHS.md +111 -0
  36. package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
  37. package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
  38. package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
  39. package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
  40. package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
  41. package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
  42. package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
  43. package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
  44. package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
  45. package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
  46. package/docs/SEARCH_ARCHIVE.md +115 -0
  47. package/docs/assets/architecture.svg +104 -0
  48. package/docs/assets/hero.svg +98 -0
  49. package/docs/engines/bangumi.md +27 -0
  50. package/docs/engines/biorxiv.md +27 -0
  51. package/docs/engines/clawhub.md +27 -0
  52. package/docs/engines/crt_sh.md +27 -0
  53. package/docs/engines/datacite.md +27 -0
  54. package/docs/engines/deps_dev.md +27 -0
  55. package/docs/engines/douban_movie.md +27 -0
  56. package/docs/engines/endoflife.md +27 -0
  57. package/docs/engines/flk_law.md +27 -0
  58. package/docs/engines/fxtwitter.md +27 -0
  59. package/docs/engines/google_news.md +27 -0
  60. package/docs/engines/gov_regulations.md +27 -0
  61. package/docs/engines/guqian.md +27 -0
  62. package/docs/engines/hackernews.md +27 -0
  63. package/docs/engines/iplant.md +27 -0
  64. package/docs/engines/keenable.md +27 -0
  65. package/docs/engines/met_museum.md +27 -0
  66. package/docs/engines/nasa_images.md +27 -0
  67. package/docs/engines/netease_music.md +27 -0
  68. package/docs/engines/npm.md +27 -0
  69. package/docs/engines/nvd.md +27 -0
  70. package/docs/engines/openreview.md +27 -0
  71. package/docs/engines/openstd.md +27 -0
  72. package/docs/engines/parallel_free.md +27 -0
  73. package/docs/engines/people_daily.md +27 -0
  74. package/docs/engines/realtime_index.md +27 -0
  75. package/docs/engines/sec_edgar.md +27 -0
  76. package/docs/engines/seltz.md +27 -0
  77. package/docs/engines/skillsmp.md +27 -0
  78. package/docs/engines/sspai.md +27 -0
  79. package/docs/engines/std_samr.md +27 -0
  80. package/docs/engines/train.md +27 -0
  81. package/docs/engines/un_comtrade.md +27 -0
  82. package/docs/engines/wikisource.md +27 -0
  83. package/docs/engines/zdic.md +27 -0
  84. package/docs/engines/zenodo.md +27 -0
  85. package/docs/engines/zhihu_hot_app.md +27 -0
  86. package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
  87. package/engines/README.md +10 -0
  88. package/engines/_template_cli.yaml +39 -0
  89. package/engines/_template_http.yaml +37 -0
  90. package/engines/plugins/_template_plugin.py +50 -0
  91. package/engines/specs/_cisa_kev.yaml +54 -0
  92. package/engines/specs/_osv.yaml +63 -0
  93. package/engines/specs/aljazeera.yaml +43 -0
  94. package/engines/specs/antara.yaml +43 -0
  95. package/engines/specs/artic.yaml +20 -0
  96. package/engines/specs/aviation_weather.yaml +39 -0
  97. package/engines/specs/bangumi.yaml +21 -0
  98. package/engines/specs/biorxiv.yaml +24 -0
  99. package/engines/specs/carbon_intensity.yaml +32 -0
  100. package/engines/specs/cinii.yaml +44 -0
  101. package/engines/specs/clawhub.yaml +49 -0
  102. package/engines/specs/cleveland.yaml +19 -0
  103. package/engines/specs/cn_ai_news.yaml +43 -0
  104. package/engines/specs/core.yaml +35 -0
  105. package/engines/specs/crt_sh.yaml +38 -0
  106. package/engines/specs/ct24.yaml +43 -0
  107. package/engines/specs/datacite.yaml +27 -0
  108. package/engines/specs/datagov_uk.yaml +41 -0
  109. package/engines/specs/ddgs_images.yaml +45 -0
  110. package/engines/specs/deepseek_search.yaml +74 -0
  111. package/engines/specs/deezer.yaml +20 -0
  112. package/engines/specs/deps_dev.yaml +21 -0
  113. package/engines/specs/digup.yaml +36 -0
  114. package/engines/specs/doi.yaml +39 -0
  115. package/engines/specs/douban_movie.yaml +22 -0
  116. package/engines/specs/dw_news.yaml +47 -0
  117. package/engines/specs/egov_law.yaml +21 -0
  118. package/engines/specs/elpais.yaml +43 -0
  119. package/engines/specs/endoflife.yaml +21 -0
  120. package/engines/specs/energy_charts.yaml +21 -0
  121. package/engines/specs/factcheck_org.yaml +52 -0
  122. package/engines/specs/faz.yaml +43 -0
  123. package/engines/specs/federal_register.yaml +57 -0
  124. package/engines/specs/firecrawl.yaml +53 -0
  125. package/engines/specs/flk_law.yaml +24 -0
  126. package/engines/specs/folha.yaml +43 -0
  127. package/engines/specs/france24.yaml +44 -0
  128. package/engines/specs/full_fact.yaml +47 -0
  129. package/engines/specs/fxtwitter.yaml +31 -0
  130. package/engines/specs/gbfs_nyc.yaml +32 -0
  131. package/engines/specs/gdacs.yaml +21 -0
  132. package/engines/specs/gdelt.yaml +64 -0
  133. package/engines/specs/google_news.yaml +22 -0
  134. package/engines/specs/gov_regulations.yaml +38 -0
  135. package/engines/specs/guardian_rss.yaml +59 -0
  136. package/engines/specs/hurriyet.yaml +43 -0
  137. package/engines/specs/iplant.yaml +25 -0
  138. package/engines/specs/jikan.yaml +24 -0
  139. package/engines/specs/jolpica.yaml +20 -0
  140. package/engines/specs/k10plus.yaml +20 -0
  141. package/engines/specs/keenable.yaml +44 -0
  142. package/engines/specs/lefigaro.yaml +43 -0
  143. package/engines/specs/listenbrainz.yaml +19 -0
  144. package/engines/specs/matichon.yaml +43 -0
  145. package/engines/specs/met_museum.yaml +21 -0
  146. package/engines/specs/nasa_images.yaml +45 -0
  147. package/engines/specs/netease_music.yaml +38 -0
  148. package/engines/specs/nhk.yaml +43 -0
  149. package/engines/specs/nhtsa_vpic.yaml +20 -0
  150. package/engines/specs/noaa_swpc.yaml +20 -0
  151. package/engines/specs/nvd.yaml +38 -0
  152. package/engines/specs/obis.yaml +20 -0
  153. package/engines/specs/opencitations.yaml +52 -0
  154. package/engines/specs/openf1.yaml +21 -0
  155. package/engines/specs/openfoodfacts.yaml +45 -0
  156. package/engines/specs/openligadb.yaml +20 -0
  157. package/engines/specs/openreview.yaml +37 -0
  158. package/engines/specs/openstd.yaml +21 -0
  159. package/engines/specs/people_daily.yaml +22 -0
  160. package/engines/specs/realtime_index.yaml +34 -0
  161. package/engines/specs/ror.yaml +20 -0
  162. package/engines/specs/satnogs.yaml +20 -0
  163. package/engines/specs/sec_edgar.yaml +46 -0
  164. package/engines/specs/skillsmp.yaml +50 -0
  165. package/engines/specs/sogou_weixin_browser.yaml +30 -0
  166. package/engines/specs/soilgrids.yaml +25 -0
  167. package/engines/specs/sspai.yaml +36 -0
  168. package/engines/specs/stackexchange.yaml +46 -0
  169. package/engines/specs/std_samr.yaml +21 -0
  170. package/engines/specs/tass.yaml +43 -0
  171. package/engines/specs/tle_mirror.yaml +21 -0
  172. package/engines/specs/train.yaml +37 -0
  173. package/engines/specs/tvmaze.yaml +20 -0
  174. package/engines/specs/twitter_syndication.yaml +31 -0
  175. package/engines/specs/un_comtrade.yaml +23 -0
  176. package/engines/specs/unpaywall.yaml +54 -0
  177. package/engines/specs/vnpress.yaml +43 -0
  178. package/engines/specs/weather.yaml +32 -0
  179. package/engines/specs/who_don.yaml +21 -0
  180. package/engines/specs/who_gho.yaml +20 -0
  181. package/engines/specs/wikimedia_commons.yaml +34 -0
  182. package/engines/specs/wikisource.yaml +22 -0
  183. package/engines/specs/worms.yaml +19 -0
  184. package/engines/specs/x_trending.yaml +34 -0
  185. package/engines/specs/yna.yaml +43 -0
  186. package/engines/specs/ynet.yaml +43 -0
  187. package/engines/specs/zbmath.yaml +40 -0
  188. package/engines/specs/zdic.yaml +22 -0
  189. package/engines/specs/zenodo.yaml +27 -0
  190. package/engines/specs/zhihu_hot_app.yaml +35 -0
  191. package/mcp/clients.yaml +74 -0
  192. package/package.json +51 -12
  193. package/packages/dsh-plugin/README.md +134 -0
  194. package/packages/dsh-plugin/cordis.patch.yml +48 -0
  195. package/packages/dsh-plugin/dsh/client.js +1244 -0
  196. package/packages/dsh-plugin/dsh/index.js +2290 -0
  197. package/packages/dsh-plugin/dsh/native-tools.mjs +743 -0
  198. package/packages/dsh-plugin/package.json +38 -0
  199. package/packages/dsh-plugin/test/client.test.js +232 -0
  200. package/packages/dsh-plugin/test/config.test.js +30 -0
  201. package/packages/dsh-plugin/test/index.test.js +1338 -0
  202. package/references/query-templates.md +96 -0
  203. package/references/query-templates.yaml +118 -0
  204. package/scripts/ab_eval.py +219 -0
  205. package/scripts/ab_eval_p0p1.py +266 -0
  206. package/scripts/adaptive.py +271 -70
  207. package/scripts/answer.py +154 -0
  208. package/scripts/archive_run.py +591 -0
  209. package/scripts/argo_engine_registry.py +172 -0
  210. package/scripts/argo_paths.py +728 -0
  211. package/scripts/article.py +136 -0
  212. package/scripts/arxiv_source.py +201 -0
  213. package/scripts/badcase_ledger.py +252 -0
  214. package/scripts/batch_probe.py +241 -0
  215. package/scripts/benchmark.py +19 -0
  216. package/scripts/bounded_run.py +144 -0
  217. package/scripts/browser_auth.py +323 -0
  218. package/scripts/burstiness_detector.py +147 -0
  219. package/scripts/cache.py +1078 -67
  220. package/scripts/cache_guard.py +271 -0
  221. package/scripts/cache_key_vdom.py +56 -0
  222. package/scripts/cache_similarity.py +170 -0
  223. package/scripts/caliber_isolation.py +195 -0
  224. package/scripts/calibrate_lowquality.py +210 -0
  225. package/scripts/candidate_envelope.py +366 -0
  226. package/scripts/chrome_cdp.py +871 -0
  227. package/scripts/circuit_breaker.py +389 -0
  228. package/scripts/citations.py +334 -0
  229. package/scripts/cite_cli.py +59 -0
  230. package/scripts/cjk_tokens.py +95 -0
  231. package/scripts/clarify.py +47 -24
  232. package/scripts/cli_io.py +240 -0
  233. package/scripts/config.py +683 -45
  234. package/scripts/config_cache.py +402 -0
  235. package/scripts/conn_pool.py +220 -0
  236. package/scripts/content_security.py +685 -0
  237. package/scripts/content_signals.py +730 -0
  238. package/scripts/crawl.py +74 -35
  239. package/scripts/daemon.py +866 -0
  240. package/scripts/digup_bridge.py +214 -0
  241. package/scripts/engine_admission.py +366 -0
  242. package/scripts/engine_dispatch.py +1019 -0
  243. package/scripts/engine_env.py +534 -0
  244. package/scripts/engine_failure.py +411 -0
  245. package/scripts/engine_families.py +732 -0
  246. package/scripts/engine_policy.py +218 -0
  247. package/scripts/engine_requires.py +190 -0
  248. package/scripts/engine_status.py +322 -0
  249. package/scripts/engine_validate.py +638 -0
  250. package/scripts/engines.py +778 -294
  251. package/scripts/engines_base.py +1521 -0
  252. package/scripts/engines_builders.py +298 -0
  253. package/scripts/engines_builders_batch10.py +159 -0
  254. package/scripts/engines_builders_batch11.py +174 -0
  255. package/scripts/engines_builders_batch12.py +103 -0
  256. package/scripts/engines_builders_batch13.py +340 -0
  257. package/scripts/engines_builders_batch9.py +1468 -0
  258. package/scripts/engines_builders_cn.py +2283 -0
  259. package/scripts/engines_builders_data.py +2923 -0
  260. package/scripts/engines_builders_data_macro.py +775 -0
  261. package/scripts/engines_builders_feeds.py +255 -0
  262. package/scripts/engines_builders_intl.py +1239 -0
  263. package/scripts/engines_builders_search.py +440 -0
  264. package/scripts/engines_builders_tech.py +1360 -0
  265. package/scripts/evidence.py +588 -162
  266. package/scripts/evidence_loop.py +592 -0
  267. package/scripts/evidence_tier.py +298 -0
  268. package/scripts/except_sets.py +25 -0
  269. package/scripts/extract.py +55 -16
  270. package/scripts/fact_align.py +207 -0
  271. package/scripts/fact_crosscheck.py +230 -0
  272. package/scripts/fetch.py +25 -12
  273. package/scripts/fetch_browser.py +87 -0
  274. package/scripts/fetch_jsrun.py +122 -0
  275. package/scripts/fetch_quality.py +225 -0
  276. package/scripts/fetch_render_tinyfish.py +117 -0
  277. package/scripts/fetch_v3.py +1779 -0
  278. package/scripts/focus_extract.py +350 -0
  279. package/scripts/fulltext_store.py +169 -0
  280. package/scripts/fxtwitter_api.py +325 -0
  281. package/scripts/gen_engine_catalog.py +364 -0
  282. package/scripts/gen_native_tools.py +91 -0
  283. package/scripts/gen_routing_snapshot.py +184 -0
  284. package/scripts/health_check.py +140 -402
  285. package/scripts/health_probe.py +23 -8
  286. package/scripts/hot_state.py +134 -0
  287. package/scripts/http_client.py +1053 -0
  288. package/scripts/image_ops.py +430 -0
  289. package/scripts/image_vision/vision_probe +0 -0
  290. package/scripts/image_vision/vision_probe.swift +126 -0
  291. package/scripts/install.ps1 +148 -0
  292. package/scripts/install.sh +206 -0
  293. package/scripts/job.py +1186 -0
  294. package/scripts/jump_resolver.py +235 -0
  295. package/scripts/lang_capability.py +150 -0
  296. package/scripts/lang_detect.py +289 -0
  297. package/scripts/lang_matrix_probe.py +329 -0
  298. package/scripts/lang_pref.py +311 -0
  299. package/scripts/link_source.py +340 -0
  300. package/scripts/local_engines.py +321 -0
  301. package/scripts/local_image.py +970 -0
  302. package/scripts/local_seek.py +366 -0
  303. package/scripts/macro_countries.py +82 -0
  304. package/scripts/matrix_search_eval.py +1105 -0
  305. package/scripts/mcp_diag.py +8 -3
  306. package/scripts/mcp_handlers.py +1074 -0
  307. package/scripts/mcp_handlers_surface.py +157 -0
  308. package/scripts/mcp_launch.sh +122 -0
  309. package/scripts/mcp_server.py +118 -417
  310. package/scripts/mcp_setup.py +407 -0
  311. package/scripts/mcp_tools.py +330 -0
  312. package/scripts/mcp_transport.py +191 -0
  313. package/scripts/min_version_scan.py +335 -0
  314. package/scripts/net_proxy.py +258 -0
  315. package/scripts/network_aware.py +152 -0
  316. package/scripts/paper.py +274 -0
  317. package/scripts/pdf_extract.py +518 -0
  318. package/scripts/phrase_filter.py +66 -0
  319. package/scripts/plan.py +422 -0
  320. package/scripts/postprocess_signals.py +150 -0
  321. package/scripts/pre-commit +48 -0
  322. package/scripts/query_enhance.py +157 -0
  323. package/scripts/query_operators.py +196 -0
  324. package/scripts/query_rewriter.py +116 -7
  325. package/scripts/query_signals.py +300 -0
  326. package/scripts/query_templates.py +167 -0
  327. package/scripts/query_understanding.py +411 -0
  328. package/scripts/query_variants.py +298 -0
  329. package/scripts/quota.py +488 -76
  330. package/scripts/rank_signals.py +775 -0
  331. package/scripts/ranking_eval.py +319 -0
  332. package/scripts/readability_extract.py +433 -0
  333. package/scripts/recompute.py +415 -0
  334. package/scripts/recovery.py +616 -0
  335. package/scripts/recovery_l0.py +160 -0
  336. package/scripts/redskill/redskill_engine.py +332 -0
  337. package/scripts/regression_p0p1.py +318 -0
  338. package/scripts/replay_eval.py +640 -0
  339. package/scripts/rerank_tfidf.py +225 -0
  340. package/scripts/research.py +591 -403
  341. package/scripts/research_cli.py +361 -0
  342. package/scripts/research_dossier.py +492 -0
  343. package/scripts/research_expand.py +146 -0
  344. package/scripts/research_gates.py +272 -0
  345. package/scripts/research_report.py +209 -0
  346. package/scripts/research_strategy.py +37 -0
  347. package/scripts/research_work_packages.py +195 -0
  348. package/scripts/result_lang.py +311 -0
  349. package/scripts/robots_guard.py +259 -0
  350. package/scripts/route.py +921 -254
  351. package/scripts/route_bangs.py +40 -0
  352. package/scripts/route_cache.py +225 -0
  353. package/scripts/route_combo.py +910 -0
  354. package/scripts/route_domains.py +328 -0
  355. package/scripts/route_lang.py +605 -0
  356. package/scripts/route_log.py +63 -0
  357. package/scripts/route_policy.py +250 -0
  358. package/scripts/rrf_classic.py +66 -0
  359. package/scripts/screenshot.py +73 -0
  360. package/scripts/search.py +897 -405
  361. package/scripts/search_attribution.py +88 -0
  362. package/scripts/search_benchmark.py +407 -0
  363. package/scripts/search_cli.py +627 -0
  364. package/scripts/search_entry.py +389 -0
  365. package/scripts/search_output.py +606 -0
  366. package/scripts/search_pipeline.py +1077 -0
  367. package/scripts/search_rank.py +991 -0
  368. package/scripts/search_types.py +27 -12
  369. package/scripts/seek_locator.py +68 -0
  370. package/scripts/semantic_evidence.py +215 -0
  371. package/scripts/serp_guard.py +228 -0
  372. package/scripts/serp_kill_log.py +51 -0
  373. package/scripts/single_flight.py +120 -0
  374. package/scripts/social_engines/__init__.py +10 -0
  375. package/scripts/social_engines/bilibili_engine.py +131 -0
  376. package/scripts/social_engines/hackernews_engine.py +113 -0
  377. package/scripts/social_engines/reddit_engine.py +172 -0
  378. package/scripts/social_engines/twitter_engine.py +397 -0
  379. package/scripts/social_engines/v2ex_engine.py +101 -0
  380. package/scripts/social_engines/weibo_engine.py +128 -0
  381. package/scripts/social_engines/xiaohongshu_engine.py +137 -0
  382. package/scripts/social_engines/zhihu_engine.py +139 -0
  383. package/scripts/social_research.py +201 -0
  384. package/scripts/sogou_weixin_browser.py +318 -0
  385. package/scripts/source_ledger.py +249 -0
  386. package/scripts/stage_timing.py +58 -0
  387. package/scripts/stats_cli.py +95 -0
  388. package/scripts/straggler_policy.py +60 -0
  389. package/scripts/stylometry_detector.py +231 -0
  390. package/scripts/sync_backends.py +353 -0
  391. package/scripts/sync_installs.py +29 -0
  392. package/scripts/template_cli.py +127 -0
  393. package/scripts/text_lexicon.py +72 -0
  394. package/scripts/tfidf_router.py +109 -37
  395. package/scripts/time_utils.py +233 -0
  396. package/scripts/topic_research_profiles.py +643 -0
  397. package/scripts/train.py +737 -0
  398. package/scripts/tweet.py +596 -0
  399. package/scripts/url_canon.py +217 -0
  400. package/scripts/url_optimize.py +56 -0
  401. package/scripts/url_safety.py +236 -0
  402. package/scripts/usage_log.py +116 -0
  403. package/scripts/usage_log_clean.py +113 -0
  404. package/scripts/v2ex_nodes.py +311 -0
  405. package/scripts/variant_recall.py +188 -0
  406. package/scripts/watch.py +205 -0
  407. package/scripts/wechat_article.py +258 -0
  408. package/scripts/wide_research.py +402 -0
  409. package/scripts/wx.py +349 -0
  410. package/scripts/x_article_fetch.py +141 -0
  411. package/scripts/x_search.py +234 -0
  412. package/scripts/x_trending.py +189 -0
  413. package/scripts/yaml_load.py +67 -0
  414. package/sub-skills/ego-search/SKILL.md +253 -0
  415. package/sub-skills/ego-search/references/browser-runtime.md +43 -0
  416. package/sub-skills/ego-search/references/install.md +67 -0
  417. package/sub-skills/ego-search/references/original-ego-upgrade.md +51 -0
  418. package/sub-skills/ego-search/scripts/ego_search.py +800 -0
  419. package/sub-skills/ego-search/scripts/merge.py +226 -0
  420. package/sub-skills/ego-search/scripts/quality.py +64 -0
  421. package/sub-skills/ego-search/scripts/runtime.py +172 -0
  422. package/sub-skills/ego-search/scripts/safety.py +109 -0
  423. package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
  424. package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
  425. package/sub-skills/ego-search/tests/test_query_normalize.py +68 -0
  426. package/sub-skills/js-run/SKILL.md +67 -0
  427. package/sub-skills/js-run/scripts/jsrun.py +112 -0
  428. package/sub-skills/js-run/scripts/shim.js +188 -0
  429. package/sub-skills/js-run/tests/fixtures/challenge_like.js +27 -0
  430. package/sub-skills/js-run/tests/test_jsrun.py +165 -0
  431. package/sub-skills/local-search/SKILL.md +50 -53
  432. package/sub-skills/local-search/bm25.py +162 -0
  433. package/sub-skills/local-search/config.yaml +152 -55
  434. package/sub-skills/local-search/engine_registry.py +91 -26
  435. package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
  436. package/sub-skills/local-search/local_search_adapter.py +13 -2
  437. package/sub-skills/local-search/parse_maps.yaml +40 -42
  438. package/sub-skills/local-search/search_v3.py +1182 -65
  439. package/sub-skills/local-search/smart_router.py +47 -7
  440. package/sub-skills/local-seek/SKILL.md +110 -0
  441. package/sub-skills/local-seek/config/domains.yaml +20 -0
  442. package/sub-skills/local-seek/references/auto-behaviors.md +24 -0
  443. package/sub-skills/local-seek/references/fd-mdfind-recipes.md +62 -0
  444. package/sub-skills/local-seek/references/git-integration.md +54 -0
  445. package/sub-skills/local-seek/references/remote-handoff.md +46 -0
  446. package/sub-skills/local-seek/references/rg-recipes.md +96 -0
  447. package/sub-skills/local-seek/references/strategies.md +52 -0
  448. package/sub-skills/local-seek/references/structural-search.md +59 -0
  449. package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
  450. package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
  451. package/sub-skills/local-seek/scripts/seek.py +1465 -0
  452. package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
  453. package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
  454. package/sub-skills/local-seek/scripts/seek_rank.py +193 -0
  455. package/sub-skills/local-seek/tests/test_content_rank.py +222 -0
  456. package/sub-skills/local-seek/tests/test_determinism_20261006.py +145 -0
  457. package/sub-skills/local-seek/tests/test_noise_tier.py +101 -0
  458. package/sub-skills/local-seek/tests/test_pcre2_probe.py +136 -0
  459. package/sub-skills/local-seek/tests/test_rank_pinyin.py +77 -0
  460. package/sub-skills/local-seek/tests/test_regression_20261003.py +169 -0
  461. package/sub-skills/local-seek/tests/test_regression_20261006.py +146 -0
  462. package/sub-skills/local-seek/tests/test_seek_degradation.py +111 -0
  463. package/sub-skills/local-seek/tests/test_seek_fts.py +313 -0
@@ -0,0 +1,318 @@
1
+ #!/usr/bin/env python3
2
+ """搜狗微信搜索·浏览器兜底通道(ego-browser,纯标准库)。
3
+
4
+ 定位:HTTP 快车道 wechat_sogou(engines_builders_tech._build_wechat_sogou_engine)
5
+ 被搜狗反爬墙拦截(302 antispiderwall,实测 raw curl 第 2 次查询即触发)时的
6
+ 域内兜底车道——借 ego-browser 驱动用户默认浏览器(Ego Lite)里的真实会话拉取
7
+ weixin.sogou.com 结果页:指纹与 cookie 连续,2026-09-30 实测连查无风控。
8
+
9
+ 与 HTTP 车道输出同构(字段语义对齐):
10
+ - published_at:timeConvert('unix') → 本时区 ISO(同 HTTP 车道格式)
11
+ - url_resolved:中间链解析成功 true / 失败回落中间链 false(同语义)
12
+ - 解析封顶:单轮 ≤5 条(同 _SOGOU_RESOLVE_MAX_PER_CALL),带 cookie 的
13
+ 浏览器内 fetch 拉反爬 JS 页,`url += '片段'` 拼回真实链接
14
+ - account 字段 CLI 解析器(_parse_yaml_output)不保留,折进 snippet 头部
15
+ 「公众号「X」 」前缀,信息不丢
16
+
17
+ 会话复用:task space 与 page 持久化在状态文件 _state_path(),跨调用复用同一
18
+ 浏览器页——保持指纹与 cookie 连续性正是本通道的降反爬手段,故刻意不走
19
+ task.finish()(ego-browser「默认关闭空间」之例外:结果必须留在浏览器里供
20
+ 下一轮复用)。Ego Lite 重启后空间失效则自动新建。文件锁串行化并发调用;
21
+ 调用间最小间隔默认 6 秒(状态文件记最近调用时刻),贴近 HTTP 车道的
22
+ 域族节流量级。
23
+
24
+ 依赖:ego-browser CLI。缺失 / Ego Lite 未运行 / 反爬墙拦截 → stderr 报因、
25
+ 退出码 1(argo _run 对 rc!=0 只取 stderr 留痕,引擎返回空,路由轮到下一候选)。
26
+
27
+ 用法:
28
+ python3 scripts/sogou_weixin_browser.py "<query>" --n 5
29
+ python3 scripts/sogou_weixin_browser.py "<query>" --resolve-limit 3 --min-interval 8
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ import argparse
35
+ import fcntl
36
+ import json
37
+ import os
38
+ import re
39
+ import shutil
40
+ import subprocess
41
+ import sys
42
+ import tempfile
43
+ import time
44
+ from datetime import datetime
45
+ from urllib.parse import quote
46
+
47
+ def _state_path() -> str:
48
+ """登录态文件路径:ARGO_STATE_DIR 显式隔离时跟随,否则维持历史位置。
49
+
50
+ 存量优先:默认仍用 ~/.config/argo/(老用户登录态原地复用,不静默搬家);
51
+ 测试/多实例隔离设 ARGO_STATE_DIR 后登录态随状态根走,互不共享。
52
+ """
53
+ override = os.environ.get("ARGO_STATE_DIR", "").strip()
54
+ if override:
55
+ return os.path.join(os.path.expanduser(override),
56
+ "sogou_weixin_browser.json")
57
+ return os.path.expanduser("~/.config/argo/sogou_weixin_browser.json")
58
+ DEFAULT_MIN_INTERVAL = 6.0 # 调用间最小间隔(秒),贴近域族节流量级
59
+ MIN_INTERVAL_CAP = 20.0 # 补睡封顶:状态漂移时不至于睡死
60
+ JS_RESULT_WAIT_S = 15 # 浏览器内等结果/判反爬的墙钟
61
+ RESOLVE_FETCH_TIMEOUT_MS = 6000
62
+ SUBPROC_TIMEOUT = 70 # ego-browser 子进程墙钟封顶(argo spec timeout=75 先杀)
63
+ SOGOU_SEARCH_URL = "https://weixin.sogou.com/weixin?type={type}&query={query}&ie=utf8"
64
+ JSON_MARKER = "@@JSON@@"
65
+
66
+ # 反爬信号:与 HTTP 车道同一目标页,命中即宣告车道被墙
67
+ _ANTISPIDER_RE = re.compile(r"antispider|验证码|异常访问|请输入")
68
+
69
+ # JS 侧:结果抽取 + 中间链片段抓取。CFG 为 JSON 字面量(ensure_ascii 注入,
70
+ # 不存在引号/换行逃逸问题)。查询 URL 由 Python 拼好后注入,JS 不再做编码。
71
+ # 抽原始片段不在浏览器里拼接——拼接与校验放 Python 侧(_assemble_link),可离线单测。
72
+ _JS_TEMPLATE = """
73
+ const CFG = %(cfg)s;
74
+ let task = null;
75
+ if (CFG.spaceId) {
76
+ try { task = await taskSpace(CFG.spaceId); } catch (e) { task = null; }
77
+ }
78
+ if (!task) task = await taskSpace("argo 搜狗微信兜底");
79
+ let page = null;
80
+ try { page = task.page("p1"); } catch (e) { page = null; }
81
+ if (!page) page = await task.newPage();
82
+ await page.goto(%(search_url)s);
83
+ let blocked = false, got = false;
84
+ for (let i = 0; i < %(wait_s)d; i++) {
85
+ await page.waitForTimeout(1000);
86
+ const st = await page.evaluate(() => ({
87
+ n: document.querySelectorAll('li[id^="sogou_vr_11002601_box_"]').length,
88
+ text: document.body.innerText.slice(0, 2500),
89
+ }));
90
+ if (/(antispider|验证码|异常访问|请输入)/.test(st.text)) { blocked = true; break; }
91
+ if (st.n > 0) { got = true; break; }
92
+ }
93
+ let items = [];
94
+ if (!blocked) {
95
+ items = await page.evaluate(() => {
96
+ const out = [];
97
+ for (const li of document.querySelectorAll('li[id^="sogou_vr_11002601_box_"]')) {
98
+ const a = li.querySelector("h3 a");
99
+ if (!a) continue;
100
+ const info = li.querySelector(".txt-info");
101
+ const acc = li.querySelector(".all-time-y2");
102
+ const tm = li.innerHTML.match(/timeConvert\\('?([0-9]{10})'?\\)/);
103
+ out.push({
104
+ title: a.innerText.trim(),
105
+ url: a.href,
106
+ snippet: info ? info.innerText.trim() : "",
107
+ account: acc ? acc.innerText.trim() : "",
108
+ epoch: tm ? tm[1] : "",
109
+ frag: [],
110
+ fallback_url: "",
111
+ });
112
+ }
113
+ return out;
114
+ });
115
+ let resolvedCount = 0;
116
+ for (const it of items) {
117
+ if (resolvedCount >= CFG.resolveLimit) break;
118
+ if (!/weixin\\.sogou\\.com.*(\\/link\\?|\\/weixin\\?)/.test(it.url)) continue;
119
+ try {
120
+ const resp = await page.fetch(it.url, { timeout: %(resolve_to_ms)d });
121
+ const body = String(resp.body || "");
122
+ const frags = [];
123
+ for (const m of body.matchAll(/url\\s*\\+=\\s*(['"])(.*?)\\1/g)) frags.push(m[2]);
124
+ const fb = body.match(/https?:\\/\\/mp\\.weixin\\.qq\\.com[^'"<>\\s]+/);
125
+ it.frag = frags;
126
+ it.fallback_url = fb ? fb[0] : "";
127
+ resolvedCount++;
128
+ } catch (e) { /* 解析失败保留中间链,Python 侧标 url_resolved=false */ }
129
+ }
130
+ }
131
+ console.log("%(marker)s" + JSON.stringify({
132
+ ok: !blocked,
133
+ space_id: task.spaceId,
134
+ blocked: blocked,
135
+ got: got,
136
+ items: items.slice(0, CFG.n),
137
+ }));
138
+ """
139
+
140
+
141
+ def _build_js(query: str, n: int, stype: int, resolve_limit: int,
142
+ space_id: object) -> str:
143
+ """把配置注入 JS 模板。ensure_ascii 保证任意查询词的引号/换行/行分隔符全转义。"""
144
+ final_url = SOGOU_SEARCH_URL.format(type=stype, query=quote(query))
145
+ js = _JS_TEMPLATE % {
146
+ "cfg": json.dumps({"n": n, "resolveLimit": resolve_limit,
147
+ "spaceId": space_id}, ensure_ascii=True),
148
+ "search_url": json.dumps(final_url),
149
+ "wait_s": JS_RESULT_WAIT_S,
150
+ "resolve_to_ms": RESOLVE_FETCH_TIMEOUT_MS,
151
+ "marker": JSON_MARKER,
152
+ }
153
+ return js
154
+
155
+
156
+ def _assemble_link(frag: list | None, fallback_url: str | None,
157
+ original: str) -> tuple[str, bool]:
158
+ """搜狗反爬 JS 页 → 真实链接。与 HTTP 车道 _resolve_sogou_link 同判据:
159
+ url += 片段拼接优先,拼不出 http(s) 时回落响应体内的 mp.weixin 直链,
160
+ 再不行保留中间链并标 false(降级不丢结果)。入参容忍 None(JS 侧缺键)。"""
161
+ real = "".join(frag or [])
162
+ if real.startswith(("http://", "https://")):
163
+ return real, True
164
+ if (fallback_url or "").startswith(("http://", "https://")):
165
+ return fallback_url, True
166
+ return original, False
167
+
168
+
169
+ def _norm_published_at(epoch: str) -> str:
170
+ """10 位 unix 秒 → 本时区 ISO(与 HTTP 车道 datetime.fromtimestamp().astimezone() 一致)。"""
171
+ if not epoch or not str(epoch).isdigit():
172
+ return ""
173
+ try:
174
+ return datetime.fromtimestamp(int(epoch)).astimezone().isoformat(timespec="seconds")
175
+ except (ValueError, OSError):
176
+ return ""
177
+
178
+
179
+ def _extract_json(stdout: str) -> dict | None:
180
+ """从 ego-browser 输出里取标记行 JSON(CLI 可能夹杂 notice 行)。"""
181
+ for line in reversed((stdout or "").splitlines()):
182
+ if line.startswith(JSON_MARKER):
183
+ try:
184
+ return json.loads(line[len(JSON_MARKER):])
185
+ except json.JSONDecodeError:
186
+ continue
187
+ return None
188
+
189
+
190
+ def _safe_state_path(p: str) -> str:
191
+ """状态文件路径硬化:白名单限定在 ~/.config/argo/ 或系统临时目录(测试)
192
+ 之内,显式拒绝 .. 段。路径是模块常量面,不随命令行开放。"""
193
+ norm = os.path.normpath(os.path.abspath(os.path.expanduser(p)))
194
+ if ".." in norm.split(os.sep):
195
+ raise ValueError(f"state 路径不允许包含 ..:{p}")
196
+ allowed_roots = [
197
+ os.path.realpath(os.path.expanduser("~/.config/argo")),
198
+ os.path.realpath(tempfile.gettempdir()),
199
+ ]
200
+ # ARGO_STATE_DIR 隔离态:状态根也是合法落点(与 _state_path 的解析对齐)
201
+ _override = os.environ.get("ARGO_STATE_DIR", "").strip()
202
+ if _override:
203
+ allowed_roots.append(os.path.realpath(os.path.expanduser(_override)))
204
+ rp = os.path.realpath(norm)
205
+ if not any(rp == root or rp.startswith(root + os.sep) for root in allowed_roots):
206
+ raise ValueError(f"state 路径必须在 {' 或 '.join(allowed_roots)} 之内:{p}")
207
+ return rp
208
+
209
+
210
+ def _load_state(path: str) -> dict:
211
+ try:
212
+ with open(path, encoding="utf-8") as f:
213
+ return json.load(f)
214
+ except (OSError, json.JSONDecodeError):
215
+ return {}
216
+
217
+
218
+ def _save_state(path: str, state: dict) -> None:
219
+ os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
220
+ with open(path, "w", encoding="utf-8") as f:
221
+ json.dump(state, f, ensure_ascii=False)
222
+
223
+
224
+ def main(argv: list | None = None) -> int:
225
+ ap = argparse.ArgumentParser(description="搜狗微信搜索·浏览器兜底通道")
226
+ ap.add_argument("query")
227
+ ap.add_argument("--n", type=int, default=5)
228
+ ap.add_argument("--type", type=int, choices=(1, 2), default=2,
229
+ help="2=文章(默认)1=公众号")
230
+ ap.add_argument("--resolve-limit", type=int, default=5)
231
+ ap.add_argument("--min-interval", type=float, default=DEFAULT_MIN_INTERVAL)
232
+ args = ap.parse_args(argv)
233
+ try:
234
+ state_path = _safe_state_path(_state_path())
235
+ except ValueError as e:
236
+ print(f"sogou_weixin_browser: {e}", file=sys.stderr)
237
+ return 1
238
+
239
+ if not shutil.which("ego-browser"):
240
+ print("sogou_weixin_browser: ego-browser CLI 不存在(Ego Lite 未安装?)",
241
+ file=sys.stderr)
242
+ return 1
243
+
244
+ # 文件锁串行化:并发调用会争抢同一浏览器页(goto 竞态)与节流窗口
245
+ lock_path = state_path + ".lock"
246
+ os.makedirs(os.path.dirname(lock_path) or ".", exist_ok=True)
247
+ with open(lock_path, "w", encoding="utf-8") as lock_f:
248
+ fcntl.flock(lock_f, fcntl.LOCK_EX)
249
+ state = _load_state(state_path)
250
+ last_ts = float(state.get("last_ts") or 0)
251
+ wait = args.min_interval - (time.time() - last_ts)
252
+ if 0 < wait <= MIN_INTERVAL_CAP:
253
+ time.sleep(wait)
254
+
255
+ js = _build_js(args.query, args.n, args.type, args.resolve_limit,
256
+ state.get("space_id"))
257
+ try:
258
+ proc = subprocess.run(
259
+ ["ego-browser", "nodejs", "-e", js],
260
+ capture_output=True, text=True, encoding="utf-8",
261
+ errors="replace", timeout=SUBPROC_TIMEOUT)
262
+ except subprocess.TimeoutExpired:
263
+ print(f"sogou_weixin_browser: ego-browser 超时(>{SUBPROC_TIMEOUT}s)",
264
+ file=sys.stderr)
265
+ return 1
266
+ except FileNotFoundError:
267
+ print("sogou_weixin_browser: ego-browser 调用失败", file=sys.stderr)
268
+ return 1
269
+
270
+ if proc.returncode != 0:
271
+ tail = (proc.stderr or "").strip()[:200]
272
+ print(f"sogou_weixin_browser: ego-browser rc={proc.returncode}: {tail}",
273
+ file=sys.stderr)
274
+ return 1
275
+ data = _extract_json(proc.stdout) or _extract_json(proc.stderr)
276
+ if not data:
277
+ print("sogou_weixin_browser: 未能解析 ego-browser 输出", file=sys.stderr)
278
+ return 1
279
+
280
+ # 空间句柄持久化(Ego Lite 重启后 taskSpace(id) 会抛错 → 下轮自动新建)
281
+ if data.get("space_id"):
282
+ state["space_id"] = data["space_id"]
283
+ state["last_ts"] = time.time()
284
+ _save_state(state_path, state)
285
+
286
+ if data.get("blocked"):
287
+ print("sogou_weixin_browser: 搜狗反爬墙拦截——请在浏览器打开 "
288
+ "weixin.sogou.com 手动通过验证后重试", file=sys.stderr)
289
+ return 1
290
+ if not data.get("ok"):
291
+ print("sogou_weixin_browser: 浏览器通道未取到结果页", file=sys.stderr)
292
+ return 1
293
+
294
+ rows = []
295
+ for it in data.get("items", []):
296
+ title = str(it.get("title") or "").strip()
297
+ url = str(it.get("url") or "").strip()
298
+ if not title and not url:
299
+ continue
300
+ real, resolved = _assemble_link(it.get("frag"), it.get("fallback_url"), url)
301
+ snippet = str(it.get("snippet") or "").strip()[:200]
302
+ account = str(it.get("account") or "").strip()
303
+ if account:
304
+ snippet = f"公众号「{account}」 {snippet}".strip()
305
+ row = {"title": title[:80], "url": real, "snippet": snippet}
306
+ published = _norm_published_at(it.get("epoch", ""))
307
+ if published:
308
+ row["published_at"] = published
309
+ row["url_resolved"] = resolved
310
+ rows.append(row)
311
+
312
+ import yaml
313
+ print(yaml.safe_dump(rows[: max(1, args.n)], allow_unicode=True, sort_keys=False))
314
+ return 0
315
+
316
+
317
+ if __name__ == "__main__":
318
+ sys.exit(main())
@@ -0,0 +1,249 @@
1
+ #!/usr/bin/env python3
2
+ """source_ledger.py — 来源账本
3
+
4
+ 每个来源必须记录访问状态、发布者、URL、验证日期、提取说明、新鲜度节奏。
5
+ 用于自动识别过期/不可验证来源,对抗内容农场。
6
+
7
+ 设计原则:
8
+ - 纯 stdlib,零依赖
9
+ - 与 evidence_tier 互补:后者判证据等级,本模块判来源健康度
10
+ - 输出可直接用于内容质量评分和 GEO 监测
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import re
15
+ from dataclasses import dataclass, field
16
+ from datetime import datetime, timezone, timedelta
17
+ from typing import Any, Optional
18
+ from urllib.parse import urlparse
19
+
20
+
21
+ # ── 数据结构 ────────────────────────────────────────────────────────
22
+
23
+ @dataclass
24
+ class SourceRecord:
25
+ """来源记录"""
26
+ url: str
27
+ publisher: str = ""
28
+ verified_at: Optional[datetime] = None
29
+ access_status: str = "unknown" # accessible / restricted / unavailable / unknown
30
+ freshness_cadence: str = "unknown" # realtime / daily / weekly / monthly / static / unknown
31
+ confidence_tier: str = "D" # A / B / C / D
32
+ extraction_note: str = ""
33
+ title: str = ""
34
+ content_hash: str = "" # 内容指纹,用于检测变更
35
+
36
+ def to_dict(self) -> dict:
37
+ return {
38
+ "url": self.url,
39
+ "publisher": self.publisher,
40
+ "verified_at": self.verified_at.isoformat() if self.verified_at else None,
41
+ "access_status": self.access_status,
42
+ "freshness_cadence": self.freshness_cadence,
43
+ "confidence_tier": self.confidence_tier,
44
+ "extraction_note": self.extraction_note,
45
+ "title": self.title,
46
+ "content_hash": self.content_hash,
47
+ }
48
+
49
+
50
+ # ── 新鲜度节奏 ──────────────────────────────────────────────────────
51
+
52
+ FRESHNESS_CADENCE_DAYS = {
53
+ "realtime": 0, # 实时(分钟级)
54
+ "daily": 1, # 每日
55
+ "weekly": 7, # 每周
56
+ "monthly": 30, # 每月
57
+ "quarterly": 90, # 每季度
58
+ "yearly": 365, # 每年
59
+ "static": 3650, # 静态(10 年)
60
+ "unknown": 30, # 未知,默认按月
61
+ }
62
+
63
+
64
+ class SourceLedger:
65
+ """来源账本"""
66
+
67
+ def __init__(self):
68
+ self.records: dict[str, SourceRecord] = {}
69
+
70
+ def add_source(self, record: SourceRecord) -> None:
71
+ """添加来源记录"""
72
+ self.records[record.url] = record
73
+
74
+ def get_source(self, url: str) -> Optional[SourceRecord]:
75
+ """获取来源记录"""
76
+ return self.records.get(url)
77
+
78
+ def get_freshness_risk(self, url: str) -> float:
79
+ """计算来源新鲜度风险(0-1,越高越危险)
80
+
81
+ 风险 = 实际年龄 / 期望新鲜度周期
82
+ """
83
+ record = self.records.get(url)
84
+ if not record:
85
+ return 1.0 # 未知来源,最高风险
86
+
87
+ if not record.verified_at:
88
+ return 0.8 # 无验证日期,高风险
89
+
90
+ age = datetime.now(timezone.utc) - record.verified_at
91
+ cadence_days = FRESHNESS_CADENCE_DAYS.get(record.freshness_cadence, 30)
92
+
93
+ if cadence_days == 0:
94
+ return 0.0 # 实时内容,无风险
95
+
96
+ risk = age.days / cadence_days
97
+ return min(risk, 1.0)
98
+
99
+ def get_access_risk(self, url: str) -> float:
100
+ """计算访问风险(0-1,越高越危险)"""
101
+ record = self.records.get(url)
102
+ if not record:
103
+ return 1.0
104
+
105
+ risk_map = {
106
+ "accessible": 0.0,
107
+ "restricted": 0.5,
108
+ "unavailable": 1.0,
109
+ "unknown": 0.7,
110
+ }
111
+ return risk_map.get(record.access_status, 0.7)
112
+
113
+ def get_overall_risk(self, url: str) -> float:
114
+ """计算综合风险(0-1,越高越危险)
115
+
116
+ 综合风险 = 0.4 * 新鲜度风险 + 0.3 * 访问风险 + 0.3 * 证据等级风险
117
+ """
118
+ freshness_risk = self.get_freshness_risk(url)
119
+ access_risk = self.get_access_risk(url)
120
+
121
+ record = self.records.get(url)
122
+ if record:
123
+ tier_risk = 1.0 - {"A": 1.0, "B": 0.7, "C": 0.3, "D": 0.0}.get(record.confidence_tier, 0.0)
124
+ else:
125
+ tier_risk = 1.0
126
+
127
+ return round(0.4 * freshness_risk + 0.3 * access_risk + 0.3 * tier_risk, 3)
128
+
129
+ def get_stale_sources(self, threshold: float = 0.5) -> list[str]:
130
+ """获取过期来源列表(风险超过阈值)"""
131
+ return [url for url in self.records if self.get_overall_risk(url) > threshold]
132
+
133
+ def get_summary(self) -> dict:
134
+ """获取账本摘要"""
135
+ if not self.records:
136
+ return {
137
+ "total_sources": 0,
138
+ "accessible_count": 0,
139
+ "restricted_count": 0,
140
+ "unavailable_count": 0,
141
+ "high_risk_count": 0,
142
+ "tier_distribution": {},
143
+ }
144
+
145
+ accessible = sum(1 for r in self.records.values() if r.access_status == "accessible")
146
+ restricted = sum(1 for r in self.records.values() if r.access_status == "restricted")
147
+ unavailable = sum(1 for r in self.records.values() if r.access_status == "unavailable")
148
+ high_risk = sum(1 for r in self.records.values() if self.get_overall_risk(r.url) > 0.5)
149
+
150
+ tier_dist = {}
151
+ for r in self.records.values():
152
+ tier_dist[r.confidence_tier] = tier_dist.get(r.confidence_tier, 0) + 1
153
+
154
+ return {
155
+ "total_sources": len(self.records),
156
+ "accessible_count": accessible,
157
+ "restricted_count": restricted,
158
+ "unavailable_count": unavailable,
159
+ "high_risk_count": high_risk,
160
+ "tier_distribution": tier_dist,
161
+ }
162
+
163
+
164
+ # ── 自动推断 ────────────────────────────────────────────────────────
165
+
166
+ def infer_freshness_cadence(url: str, title: str = "", content: str = "") -> str:
167
+ """推断内容的新鲜度节奏
168
+
169
+ 基于 URL 模式、标题关键词和内容特征推断。
170
+ """
171
+ host = _extract_host(url)
172
+ combined = f"{url} {title} {content[:500]}".lower()
173
+
174
+ # 实时信号
175
+ if any(kw in combined for kw in ["实时", "快讯", "直播", "breaking", "live", "realtime"]):
176
+ return "realtime"
177
+
178
+ # 每日信号
179
+ if any(kw in combined for kw in ["日报", "每日", "daily", "today", "今日"]):
180
+ return "daily"
181
+
182
+ # 每周信号
183
+ if any(kw in combined for kw in ["周报", "每周", "weekly", "week"]):
184
+ return "weekly"
185
+
186
+ # 每月信号
187
+ if any(kw in combined for kw in ["月报", "每月", "monthly", "month"]):
188
+ return "monthly"
189
+
190
+ # 静态内容
191
+ if any(kw in combined for kw in ["文档", "指南", "教程", "docs", "guide", "tutorial", "reference"]):
192
+ return "static"
193
+
194
+ # 默认
195
+ return "unknown"
196
+
197
+
198
+ def infer_access_status(url: str, fetch_success: bool = True, http_status: int = 200) -> str:
199
+ """推断访问状态"""
200
+ if not fetch_success:
201
+ return "unavailable"
202
+ if http_status == 200:
203
+ return "accessible"
204
+ if http_status in (401, 403, 429):
205
+ return "restricted"
206
+ if http_status >= 400:
207
+ return "unavailable"
208
+ return "unknown"
209
+
210
+
211
+ def _extract_host(url: str) -> str:
212
+ """提取主机名"""
213
+ if not url:
214
+ return ""
215
+ try:
216
+ host = urlparse(url).netloc.lower()
217
+ if "@" in host:
218
+ host = host.rsplit("@", 1)[1]
219
+ if ":" in host:
220
+ host = host.split(":", 1)[0]
221
+ return host
222
+ except Exception:
223
+ return ""
224
+
225
+
226
+ def create_source_record(
227
+ url: str,
228
+ publisher: str = "",
229
+ fetch_success: bool = True,
230
+ http_status: int = 200,
231
+ title: str = "",
232
+ content: str = "",
233
+ verified_at: Optional[datetime] = None,
234
+ confidence_tier: str = "D",
235
+ ) -> SourceRecord:
236
+ """创建来源记录(自动推断缺失字段)
237
+
238
+ 注意:verified_at 为 None 时,来源会被标记为高风险(无验证日期)。
239
+ 只有明确知道验证时间时才传入。
240
+ """
241
+ return SourceRecord(
242
+ url=url,
243
+ publisher=publisher,
244
+ verified_at=verified_at, # None 表示未知,会被标记为高风险
245
+ access_status=infer_access_status(url, fetch_success, http_status),
246
+ freshness_cadence=infer_freshness_cadence(url, title, content),
247
+ confidence_tier=confidence_tier,
248
+ title=title,
249
+ )
@@ -0,0 +1,58 @@
1
+ #!/usr/bin/env python3
2
+ """stage_timing.py — 阶段耗时计时器(--explain-timing 的载体)。
3
+
4
+ 从 search.py 中外提。此前「最大瓶颈在哪」只能靠外挂计时得出(importtime
5
+ 看冷启动、cProfile 看 CPU、临时包装模块级函数看阶段),换个人、换台机器
6
+ 就得重做一遍;这个类让工具自己报出钱花在哪。
7
+
8
+ 零依赖纯数据结构,可被搜索之外的层(抓取、研究、评测)复用。
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import time
14
+ from typing import Any
15
+
16
+
17
+ class StageTiming:
18
+ """各阶段墙钟耗时(毫秒)。
19
+
20
+ **默认开启**(`--no-timing` 可关)。默认关的话,想知道瓶颈就得会外部
21
+ 计时和临时代码,等于把「自己动手优化」的门槛抬到只有维护者过得去。
22
+ 代价是每次多几百字节输出——相对它省下的排查成本可以忽略。
23
+ """
24
+
25
+ __slots__ = ("marks",)
26
+
27
+ def __init__(self) -> None:
28
+ self.marks: dict[str, float] = {}
29
+
30
+ def add(self, stage: str, ms: float) -> None:
31
+ # 保留到微秒:1 位小数会把亚毫秒阶段(本地融合/去重在热进程里常 <0.1ms)
32
+ # 全压成 0.0,看起来像「没测到」。多出的两位数字对输出体积可忽略。
33
+ self.marks[stage] = round(self.marks.get(stage, 0.0) + ms, 3)
34
+
35
+ def summary(self, total_ms: int | None = None) -> dict[str, Any]:
36
+ """按耗时降序返回,并附占比。
37
+
38
+ 占比才是行动依据:「rerank 213 ms」本身说明不了什么,「占 59%」才说明
39
+ 该不该动它。默认以**各阶段之和**为分母(自洽,纯 CPU 计算方式,不含进程
40
+ 启动);调用方给出 total_ms 时以它为准。
41
+ """
42
+ # 不取整:热进程里各阶段合计常不足 1 ms,`int()` 会把 0.9 截成 0,
43
+ # 占比随即全部变成 0%——看着像「没测到」,实际是被自己的取整吃掉了。
44
+ total = float(total_ms) if total_ms else sum(self.marks.values())
45
+ rows = [{"stage": k, "ms": v,
46
+ "pct": (round(v * 100.0 / total, 1) if total else 0.0)}
47
+ for k, v in sorted(self.marks.items(), key=lambda kv: -kv[1])]
48
+ return {"stages_ms": round(total, 3), "stages": rows}
49
+
50
+
51
+ def tick(timing: StageTiming | None) -> float | None:
52
+ """取时点;未开计时返回 None——关闭时连 perf_counter 都不调。"""
53
+ return time.perf_counter() if timing is not None else None
54
+
55
+
56
+ def tock(timing: StageTiming | None, stage: str, t0: float | None) -> None:
57
+ if timing is not None and t0 is not None:
58
+ timing.add(stage, (time.perf_counter() - t0) * 1000.0)