argo-search 1.0.1 → 2.9.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (472) hide show
  1. package/LICENSE +201 -21
  2. package/README.en.md +735 -0
  3. package/README.es.md +777 -0
  4. package/README.ja.md +777 -0
  5. package/README.ko.md +777 -0
  6. package/README.md +778 -320
  7. package/SKILL.md +135 -0
  8. package/assets/readme/hero.svg +121 -0
  9. package/assets/readme/made-with-beautify.svg +10 -0
  10. package/assets/readme/proof-routes.svg +73 -0
  11. package/assets/readme/why-better.svg +104 -0
  12. package/assets/readme/workflow.svg +83 -0
  13. package/backends/domain_profiles.json +2356 -96
  14. package/backends/engine_registry.yaml +3428 -503
  15. package/backends/query_synonyms_cn.json +35 -0
  16. package/backends/quota_profiles.json +2488 -160
  17. package/backends/source_types_cn.json +109 -0
  18. package/backends/text_lexicon.json +44 -0
  19. package/bin/argo +772 -0
  20. package/bin/argo.js +102 -39
  21. package/config.yaml +5139 -367
  22. package/cordis.patch.yml +21 -0
  23. package/data/countries.json +40 -0
  24. package/data/lang_matrix/lang_capability.json +842 -0
  25. package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
  26. package/data/regions_cn.json +1 -0
  27. package/docs/ADDING_NEW_ENGINE.md +184 -0
  28. package/docs/ARGO_INTRO.md +124 -0
  29. package/docs/DATA_SOURCE_GAPS.md +81 -0
  30. package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
  31. package/docs/ENGINE_CATALOG.md +575 -0
  32. package/docs/MCP_SETUP.md +108 -0
  33. package/docs/PLATFORM_PATHS.md +111 -0
  34. package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
  35. package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
  36. package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
  37. package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
  38. package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
  39. package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
  40. package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
  41. package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
  42. package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
  43. package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
  44. package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
  45. package/docs/SEARCH_ARCHIVE.md +115 -0
  46. package/docs/assets/architecture.svg +104 -0
  47. package/docs/assets/hero.svg +98 -0
  48. package/docs/engines/bangumi.md +27 -0
  49. package/docs/engines/biorxiv.md +27 -0
  50. package/docs/engines/clawhub.md +27 -0
  51. package/docs/engines/crt_sh.md +27 -0
  52. package/docs/engines/datacite.md +27 -0
  53. package/docs/engines/deps_dev.md +27 -0
  54. package/docs/engines/douban_movie.md +27 -0
  55. package/docs/engines/endoflife.md +27 -0
  56. package/docs/engines/flk_law.md +27 -0
  57. package/docs/engines/fxtwitter.md +27 -0
  58. package/docs/engines/google_news.md +27 -0
  59. package/docs/engines/gov_regulations.md +27 -0
  60. package/docs/engines/guqian.md +27 -0
  61. package/docs/engines/hackernews.md +27 -0
  62. package/docs/engines/iplant.md +27 -0
  63. package/docs/engines/keenable.md +27 -0
  64. package/docs/engines/met_museum.md +27 -0
  65. package/docs/engines/nasa_images.md +27 -0
  66. package/docs/engines/netease_music.md +27 -0
  67. package/docs/engines/npm.md +27 -0
  68. package/docs/engines/nvd.md +27 -0
  69. package/docs/engines/openreview.md +27 -0
  70. package/docs/engines/openstd.md +27 -0
  71. package/docs/engines/parallel_free.md +27 -0
  72. package/docs/engines/people_daily.md +27 -0
  73. package/docs/engines/realtime_index.md +27 -0
  74. package/docs/engines/sec_edgar.md +27 -0
  75. package/docs/engines/seltz.md +27 -0
  76. package/docs/engines/skillsmp.md +27 -0
  77. package/docs/engines/sspai.md +27 -0
  78. package/docs/engines/std_samr.md +27 -0
  79. package/docs/engines/train.md +27 -0
  80. package/docs/engines/un_comtrade.md +27 -0
  81. package/docs/engines/wikisource.md +27 -0
  82. package/docs/engines/zdic.md +27 -0
  83. package/docs/engines/zenodo.md +27 -0
  84. package/docs/engines/zhihu_hot_app.md +27 -0
  85. package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
  86. package/engines/README.md +10 -0
  87. package/engines/_template_cli.yaml +39 -0
  88. package/engines/_template_http.yaml +37 -0
  89. package/engines/plugins/_template_plugin.py +50 -0
  90. package/engines/specs/_cisa_kev.yaml +54 -0
  91. package/engines/specs/_osv.yaml +63 -0
  92. package/engines/specs/aljazeera.yaml +43 -0
  93. package/engines/specs/antara.yaml +43 -0
  94. package/engines/specs/artic.yaml +20 -0
  95. package/engines/specs/aviation_weather.yaml +39 -0
  96. package/engines/specs/bangumi.yaml +21 -0
  97. package/engines/specs/biorxiv.yaml +24 -0
  98. package/engines/specs/carbon_intensity.yaml +32 -0
  99. package/engines/specs/cfets.yaml +41 -0
  100. package/engines/specs/cinii.yaml +44 -0
  101. package/engines/specs/clawhub.yaml +49 -0
  102. package/engines/specs/cleveland.yaml +19 -0
  103. package/engines/specs/cn_ai_news.yaml +43 -0
  104. package/engines/specs/core.yaml +35 -0
  105. package/engines/specs/crt_sh.yaml +38 -0
  106. package/engines/specs/ct24.yaml +43 -0
  107. package/engines/specs/datacite.yaml +27 -0
  108. package/engines/specs/datagov_uk.yaml +41 -0
  109. package/engines/specs/ddgs_images.yaml +45 -0
  110. package/engines/specs/deepseek_search.yaml +74 -0
  111. package/engines/specs/deezer.yaml +20 -0
  112. package/engines/specs/defillama.yaml +38 -0
  113. package/engines/specs/deps_dev.yaml +21 -0
  114. package/engines/specs/digup.yaml +36 -0
  115. package/engines/specs/doi.yaml +39 -0
  116. package/engines/specs/douban_movie.yaml +22 -0
  117. package/engines/specs/dw_news.yaml +47 -0
  118. package/engines/specs/egov_law.yaml +21 -0
  119. package/engines/specs/elpais.yaml +43 -0
  120. package/engines/specs/endoflife.yaml +21 -0
  121. package/engines/specs/energy_charts.yaml +21 -0
  122. package/engines/specs/factcheck_org.yaml +52 -0
  123. package/engines/specs/faz.yaml +43 -0
  124. package/engines/specs/federal_register.yaml +57 -0
  125. package/engines/specs/firecrawl.yaml +53 -0
  126. package/engines/specs/flk_law.yaml +24 -0
  127. package/engines/specs/folha.yaml +43 -0
  128. package/engines/specs/france24.yaml +44 -0
  129. package/engines/specs/full_fact.yaml +47 -0
  130. package/engines/specs/fxtwitter.yaml +36 -0
  131. package/engines/specs/gbfs_nyc.yaml +32 -0
  132. package/engines/specs/gdacs.yaml +21 -0
  133. package/engines/specs/gdelt.yaml +64 -0
  134. package/engines/specs/gleif.yaml +55 -0
  135. package/engines/specs/google_news.yaml +22 -0
  136. package/engines/specs/gov_regulations.yaml +38 -0
  137. package/engines/specs/guardian_rss.yaml +59 -0
  138. package/engines/specs/hkex.yaml +40 -0
  139. package/engines/specs/hurriyet.yaml +43 -0
  140. package/engines/specs/iplant.yaml +25 -0
  141. package/engines/specs/jikan.yaml +24 -0
  142. package/engines/specs/jolpica.yaml +20 -0
  143. package/engines/specs/k10plus.yaml +20 -0
  144. package/engines/specs/keenable.yaml +44 -0
  145. package/engines/specs/keenable_free.yaml +36 -0
  146. package/engines/specs/lefigaro.yaml +43 -0
  147. package/engines/specs/listenbrainz.yaml +19 -0
  148. package/engines/specs/matichon.yaml +43 -0
  149. package/engines/specs/met_museum.yaml +21 -0
  150. package/engines/specs/nasa_images.yaml +45 -0
  151. package/engines/specs/netease_music.yaml +38 -0
  152. package/engines/specs/nhk.yaml +43 -0
  153. package/engines/specs/nhtsa_vpic.yaml +20 -0
  154. package/engines/specs/noaa_swpc.yaml +20 -0
  155. package/engines/specs/nvd.yaml +38 -0
  156. package/engines/specs/obis.yaml +20 -0
  157. package/engines/specs/opencitations.yaml +52 -0
  158. package/engines/specs/openf1.yaml +21 -0
  159. package/engines/specs/openfoodfacts.yaml +45 -0
  160. package/engines/specs/openligadb.yaml +20 -0
  161. package/engines/specs/openreview.yaml +37 -0
  162. package/engines/specs/openstd.yaml +21 -0
  163. package/engines/specs/people_daily.yaml +22 -0
  164. package/engines/specs/realtime_index.yaml +34 -0
  165. package/engines/specs/ror.yaml +20 -0
  166. package/engines/specs/satnogs.yaml +20 -0
  167. package/engines/specs/sec_edgar.yaml +46 -0
  168. package/engines/specs/shfe.yaml +39 -0
  169. package/engines/specs/skillsmp.yaml +50 -0
  170. package/engines/specs/sogou_weixin_browser.yaml +30 -0
  171. package/engines/specs/soilgrids.yaml +25 -0
  172. package/engines/specs/sspai.yaml +36 -0
  173. package/engines/specs/stackexchange.yaml +46 -0
  174. package/engines/specs/std_samr.yaml +21 -0
  175. package/engines/specs/tass.yaml +43 -0
  176. package/engines/specs/tavily_free.yaml +56 -0
  177. package/engines/specs/tle_mirror.yaml +21 -0
  178. package/engines/specs/train.yaml +37 -0
  179. package/engines/specs/tvmaze.yaml +20 -0
  180. package/engines/specs/twitter_syndication.yaml +31 -0
  181. package/engines/specs/un_comtrade.yaml +23 -0
  182. package/engines/specs/unpaywall.yaml +54 -0
  183. package/engines/specs/vnpress.yaml +43 -0
  184. package/engines/specs/weather.yaml +32 -0
  185. package/engines/specs/who_don.yaml +21 -0
  186. package/engines/specs/who_gho.yaml +20 -0
  187. package/engines/specs/wikimedia_commons.yaml +34 -0
  188. package/engines/specs/wikisource.yaml +22 -0
  189. package/engines/specs/worms.yaml +19 -0
  190. package/engines/specs/x_trending.yaml +34 -0
  191. package/engines/specs/yna.yaml +43 -0
  192. package/engines/specs/ynet.yaml +43 -0
  193. package/engines/specs/yuanbao.yaml +93 -0
  194. package/engines/specs/zbmath.yaml +40 -0
  195. package/engines/specs/zdic.yaml +22 -0
  196. package/engines/specs/zenodo.yaml +27 -0
  197. package/engines/specs/zhihu_hot_app.yaml +35 -0
  198. package/mcp/clients.yaml +106 -0
  199. package/package.json +55 -12
  200. package/packages/dsh-plugin/README.md +152 -0
  201. package/packages/dsh-plugin/cordis.patch.yml +48 -0
  202. package/packages/dsh-plugin/dsh/client.js +1533 -0
  203. package/packages/dsh-plugin/dsh/diagnostics.js +54 -0
  204. package/packages/dsh-plugin/dsh/index.js +527 -0
  205. package/packages/dsh-plugin/dsh/native-tools.mjs +749 -0
  206. package/packages/dsh-plugin/dsh/runner.js +666 -0
  207. package/packages/dsh-plugin/dsh/wide-research.js +878 -0
  208. package/packages/dsh-plugin/package.json +38 -0
  209. package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
  210. package/packages/dsh-plugin/test/client.test.js +406 -0
  211. package/packages/dsh-plugin/test/config.test.js +30 -0
  212. package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
  213. package/packages/dsh-plugin/test/index.test.js +1382 -0
  214. package/references/query-templates.md +96 -0
  215. package/references/query-templates.yaml +118 -0
  216. package/scripts/ab_eval.py +219 -0
  217. package/scripts/ab_eval_p0p1.py +266 -0
  218. package/scripts/adaptive.py +294 -70
  219. package/scripts/answer.py +154 -0
  220. package/scripts/archive_run.py +591 -0
  221. package/scripts/argo_engine_registry.py +172 -0
  222. package/scripts/argo_paths.py +728 -0
  223. package/scripts/article.py +136 -0
  224. package/scripts/article_meta.py +131 -0
  225. package/scripts/arxiv_source.py +201 -0
  226. package/scripts/badcase_ledger.py +252 -0
  227. package/scripts/batch_probe.py +241 -0
  228. package/scripts/benchmark.py +19 -0
  229. package/scripts/bounded_run.py +144 -0
  230. package/scripts/browser_auth.py +323 -0
  231. package/scripts/budget.py +247 -0
  232. package/scripts/budget_cli.py +93 -0
  233. package/scripts/build_release_assets.sh +65 -0
  234. package/scripts/burstiness_detector.py +147 -0
  235. package/scripts/cache.py +1066 -67
  236. package/scripts/cache_guard.py +271 -0
  237. package/scripts/cache_key_vdom.py +56 -0
  238. package/scripts/cache_similarity.py +170 -0
  239. package/scripts/caliber_isolation.py +195 -0
  240. package/scripts/calibrate_lowquality.py +210 -0
  241. package/scripts/candidate_envelope.py +366 -0
  242. package/scripts/chrome_cdp.py +871 -0
  243. package/scripts/ci_drift_gate.py +90 -0
  244. package/scripts/circuit_breaker.py +389 -0
  245. package/scripts/citations.py +334 -0
  246. package/scripts/cite_cli.py +59 -0
  247. package/scripts/cjk_tokens.py +95 -0
  248. package/scripts/clarify.py +47 -24
  249. package/scripts/cli_io.py +240 -0
  250. package/scripts/config.py +683 -45
  251. package/scripts/config_cache.py +402 -0
  252. package/scripts/conn_pool.py +220 -0
  253. package/scripts/content_boundary.py +46 -0
  254. package/scripts/content_security.py +685 -0
  255. package/scripts/content_signals.py +751 -0
  256. package/scripts/crawl.py +74 -35
  257. package/scripts/cross_lingual_trace.py +293 -0
  258. package/scripts/daemon.py +866 -0
  259. package/scripts/digup_bridge.py +214 -0
  260. package/scripts/dispatch_result.py +98 -0
  261. package/scripts/engine_admission.py +366 -0
  262. package/scripts/engine_dispatch.py +982 -0
  263. package/scripts/engine_env.py +539 -0
  264. package/scripts/engine_failure.py +411 -0
  265. package/scripts/engine_families.py +738 -0
  266. package/scripts/engine_policy.py +218 -0
  267. package/scripts/engine_requires.py +190 -0
  268. package/scripts/engine_status.py +322 -0
  269. package/scripts/engine_validate.py +638 -0
  270. package/scripts/engines.py +799 -294
  271. package/scripts/engines_base.py +1521 -0
  272. package/scripts/engines_builders.py +310 -0
  273. package/scripts/engines_builders_batch10.py +204 -0
  274. package/scripts/engines_builders_batch11.py +174 -0
  275. package/scripts/engines_builders_batch12.py +103 -0
  276. package/scripts/engines_builders_batch13.py +340 -0
  277. package/scripts/engines_builders_batch14.py +656 -0
  278. package/scripts/engines_builders_batch15.py +249 -0
  279. package/scripts/engines_builders_batch9.py +1468 -0
  280. package/scripts/engines_builders_cn.py +2283 -0
  281. package/scripts/engines_builders_data.py +2923 -0
  282. package/scripts/engines_builders_data_macro.py +775 -0
  283. package/scripts/engines_builders_feeds.py +255 -0
  284. package/scripts/engines_builders_intl.py +1239 -0
  285. package/scripts/engines_builders_keyless.py +111 -0
  286. package/scripts/engines_builders_search.py +440 -0
  287. package/scripts/engines_builders_tech.py +1359 -0
  288. package/scripts/evidence.py +620 -166
  289. package/scripts/evidence_loop.py +624 -0
  290. package/scripts/evidence_tier.py +298 -0
  291. package/scripts/except_sets.py +25 -0
  292. package/scripts/extract.py +55 -16
  293. package/scripts/fact_align.py +207 -0
  294. package/scripts/fact_crosscheck.py +230 -0
  295. package/scripts/fetch.py +25 -12
  296. package/scripts/fetch_browser.py +87 -0
  297. package/scripts/fetch_jsrun.py +122 -0
  298. package/scripts/fetch_quality.py +225 -0
  299. package/scripts/fetch_render_tinyfish.py +117 -0
  300. package/scripts/fetch_v3.py +1804 -0
  301. package/scripts/focus_extract.py +350 -0
  302. package/scripts/fulltext_store.py +169 -0
  303. package/scripts/fxtwitter_api.py +325 -0
  304. package/scripts/gen_engine_catalog.py +364 -0
  305. package/scripts/gen_native_tools.py +91 -0
  306. package/scripts/gen_routing_snapshot.py +195 -0
  307. package/scripts/health_check.py +140 -402
  308. package/scripts/health_probe.py +23 -8
  309. package/scripts/hot_state.py +134 -0
  310. package/scripts/http_client.py +1053 -0
  311. package/scripts/image_ops.py +430 -0
  312. package/scripts/image_vision/vision_probe.swift +126 -0
  313. package/scripts/install.ps1 +166 -0
  314. package/scripts/install.sh +221 -0
  315. package/scripts/job.py +1186 -0
  316. package/scripts/jump_resolver.py +235 -0
  317. package/scripts/lang_capability.py +150 -0
  318. package/scripts/lang_detect.py +289 -0
  319. package/scripts/lang_matrix_probe.py +329 -0
  320. package/scripts/lang_pref.py +311 -0
  321. package/scripts/link_source.py +340 -0
  322. package/scripts/local_image.py +970 -0
  323. package/scripts/local_seek.py +402 -0
  324. package/scripts/macro_countries.py +97 -0
  325. package/scripts/matrix_search_eval.py +1187 -0
  326. package/scripts/mcp_compact.py +234 -0
  327. package/scripts/mcp_diag.py +8 -3
  328. package/scripts/mcp_handlers.py +492 -0
  329. package/scripts/mcp_handlers_surface.py +157 -0
  330. package/scripts/mcp_launch.sh +124 -0
  331. package/scripts/mcp_local.py +241 -0
  332. package/scripts/mcp_server.py +125 -419
  333. package/scripts/mcp_setup.py +431 -0
  334. package/scripts/mcp_tools.py +331 -0
  335. package/scripts/mcp_transport.py +191 -0
  336. package/scripts/mcp_warm.py +127 -0
  337. package/scripts/min_version_scan.py +335 -0
  338. package/scripts/net_proxy.py +258 -0
  339. package/scripts/network_aware.py +152 -0
  340. package/scripts/output_transparency.py +200 -0
  341. package/scripts/paid_policy.py +98 -0
  342. package/scripts/paper.py +274 -0
  343. package/scripts/pdf_extract.py +518 -0
  344. package/scripts/phrase_filter.py +66 -0
  345. package/scripts/plan.py +422 -0
  346. package/scripts/postprocess_signals.py +150 -0
  347. package/scripts/pre-commit +48 -0
  348. package/scripts/query_enhance.py +157 -0
  349. package/scripts/query_operators.py +196 -0
  350. package/scripts/query_rewriter.py +116 -7
  351. package/scripts/query_signals.py +300 -0
  352. package/scripts/query_templates.py +167 -0
  353. package/scripts/query_understanding.py +411 -0
  354. package/scripts/query_variants.py +298 -0
  355. package/scripts/quota.py +633 -76
  356. package/scripts/rank_signals.py +775 -0
  357. package/scripts/ranking_eval.py +319 -0
  358. package/scripts/readability_extract.py +433 -0
  359. package/scripts/recompute.py +415 -0
  360. package/scripts/recovery.py +616 -0
  361. package/scripts/recovery_l0.py +160 -0
  362. package/scripts/recovery_runner.py +150 -0
  363. package/scripts/redskill/redskill_engine.py +332 -0
  364. package/scripts/regression_p0p1.py +318 -0
  365. package/scripts/replay_eval.py +658 -0
  366. package/scripts/rerank_tfidf.py +225 -0
  367. package/scripts/research.py +607 -403
  368. package/scripts/research_cli.py +361 -0
  369. package/scripts/research_dossier.py +492 -0
  370. package/scripts/research_expand.py +169 -0
  371. package/scripts/research_gates.py +272 -0
  372. package/scripts/research_report.py +209 -0
  373. package/scripts/research_strategy.py +37 -0
  374. package/scripts/research_work_packages.py +195 -0
  375. package/scripts/result_lang.py +311 -0
  376. package/scripts/retry_policy.py +76 -0
  377. package/scripts/robots_guard.py +259 -0
  378. package/scripts/route.py +921 -254
  379. package/scripts/route_bangs.py +40 -0
  380. package/scripts/route_cache.py +243 -0
  381. package/scripts/route_combo.py +917 -0
  382. package/scripts/route_domains.py +341 -0
  383. package/scripts/route_lang.py +605 -0
  384. package/scripts/route_log.py +63 -0
  385. package/scripts/route_policy.py +250 -0
  386. package/scripts/rrf_classic.py +66 -0
  387. package/scripts/screenshot.py +73 -0
  388. package/scripts/search.py +886 -406
  389. package/scripts/search_attribution.py +185 -0
  390. package/scripts/search_benchmark.py +407 -0
  391. package/scripts/search_cli.py +640 -0
  392. package/scripts/search_entry.py +395 -0
  393. package/scripts/search_output.py +690 -0
  394. package/scripts/search_pipeline.py +926 -0
  395. package/scripts/search_rank.py +991 -0
  396. package/scripts/search_types.py +129 -16
  397. package/scripts/seek_locator.py +68 -0
  398. package/scripts/semantic_evidence.py +391 -0
  399. package/scripts/serp_guard.py +336 -0
  400. package/scripts/serp_kill_log.py +74 -0
  401. package/scripts/single_flight.py +120 -0
  402. package/scripts/social_engines/__init__.py +10 -0
  403. package/scripts/social_engines/bilibili_engine.py +131 -0
  404. package/scripts/social_engines/hackernews_engine.py +113 -0
  405. package/scripts/social_engines/reddit_engine.py +172 -0
  406. package/scripts/social_engines/twitter_engine.py +397 -0
  407. package/scripts/social_engines/v2ex_engine.py +101 -0
  408. package/scripts/social_engines/weibo_engine.py +128 -0
  409. package/scripts/social_engines/xiaohongshu_engine.py +137 -0
  410. package/scripts/social_engines/zhihu_engine.py +139 -0
  411. package/scripts/social_research.py +201 -0
  412. package/scripts/sogou_weixin_browser.py +318 -0
  413. package/scripts/source_ledger.py +249 -0
  414. package/scripts/stage_timing.py +58 -0
  415. package/scripts/stats_cli.py +107 -0
  416. package/scripts/straggler_policy.py +60 -0
  417. package/scripts/stylometry_detector.py +231 -0
  418. package/scripts/sync_backends.py +353 -0
  419. package/scripts/sync_installs.py +29 -0
  420. package/scripts/template_cli.py +127 -0
  421. package/scripts/text_lexicon.py +72 -0
  422. package/scripts/tfidf_router.py +109 -37
  423. package/scripts/time_utils.py +233 -0
  424. package/scripts/topic_research_profiles.py +643 -0
  425. package/scripts/train.py +737 -0
  426. package/scripts/tweet.py +596 -0
  427. package/scripts/url_canon.py +217 -0
  428. package/scripts/url_optimize.py +56 -0
  429. package/scripts/url_safety.py +236 -0
  430. package/scripts/usage_log.py +116 -0
  431. package/scripts/usage_log_clean.py +113 -0
  432. package/scripts/v2ex_nodes.py +311 -0
  433. package/scripts/variant_recall.py +188 -0
  434. package/scripts/watch.py +205 -0
  435. package/scripts/wechat_article.py +258 -0
  436. package/scripts/wide_research.py +447 -0
  437. package/scripts/wx.py +349 -0
  438. package/scripts/x_article_fetch.py +141 -0
  439. package/scripts/x_search.py +234 -0
  440. package/scripts/x_trending.py +189 -0
  441. package/scripts/yaml_load.py +67 -0
  442. package/scripts/zhihu_quota.py +139 -0
  443. package/sub-skills/ego-search/SKILL.md +253 -0
  444. package/sub-skills/ego-search/scripts/ego_search.py +800 -0
  445. package/sub-skills/ego-search/scripts/merge.py +226 -0
  446. package/sub-skills/ego-search/scripts/quality.py +64 -0
  447. package/sub-skills/ego-search/scripts/runtime.py +172 -0
  448. package/sub-skills/ego-search/scripts/safety.py +109 -0
  449. package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
  450. package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
  451. package/sub-skills/js-run/SKILL.md +67 -0
  452. package/sub-skills/js-run/scripts/jsrun.py +112 -0
  453. package/sub-skills/js-run/scripts/shim.js +188 -0
  454. package/sub-skills/local-search/SKILL.md +50 -53
  455. package/sub-skills/local-search/bm25.py +186 -0
  456. package/sub-skills/local-search/cli_runner.py +331 -0
  457. package/sub-skills/local-search/config.yaml +152 -55
  458. package/sub-skills/local-search/engine_registry.py +91 -26
  459. package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
  460. package/sub-skills/local-search/local_search_adapter.py +13 -2
  461. package/sub-skills/local-search/parse_maps.yaml +40 -42
  462. package/sub-skills/local-search/parsers.py +328 -0
  463. package/sub-skills/local-search/search_v3.py +716 -242
  464. package/sub-skills/local-search/smart_router.py +47 -7
  465. package/sub-skills/local-seek/SKILL.md +110 -0
  466. package/sub-skills/local-seek/config/domains.yaml +20 -0
  467. package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
  468. package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
  469. package/sub-skills/local-seek/scripts/seek.py +1465 -0
  470. package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
  471. package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
  472. package/sub-skills/local-seek/scripts/seek_rank.py +197 -0
@@ -0,0 +1,113 @@
1
+ #!/usr/bin/env python3
2
+ """Hacker News 搜索引擎(HN Algolia 公开 API,零密钥)
3
+
4
+ 保持一致 social_engines 统一 schema(title/url/snippet/source/score/social_meta),
5
+ sentiment 聚合(aggregate_social_sentiment)按 social_meta 互动字段直接可用。
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import urllib.parse
12
+ import urllib.request
13
+ from typing import Any
14
+
15
+ # 出口调度唯一入口(issue #13 同类修复):urlopen 不认 config.yaml 的
16
+ # network.proxy,裸用会在「必须经代理才能出网」的环境里整源连不上。
17
+ import os
18
+ import sys
19
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
20
+ from net_proxy import open_url # noqa: E402
21
+
22
+ HN_API = "https://hn.algolia.com/api/v1/search"
23
+
24
+
25
+ def _http_get(url: str, timeout: int = 10) -> bytes:
26
+ req = urllib.request.Request(
27
+ url,
28
+ headers={"User-Agent": "argo-search/1.0 (+https://github.com/taxueseek/argo)", "Accept": "application/json"},
29
+ )
30
+ with open_url(req, timeout=timeout) as resp:
31
+ return resp.read()
32
+
33
+
34
+ def search(query: str, n: int = 5) -> list[dict[str, Any]]:
35
+ """通过 HN Algolia API 搜索帖子(story 类型)。"""
36
+ if not query or not query.strip():
37
+ return []
38
+ params = urllib.parse.urlencode({
39
+ "query": query.strip(),
40
+ "tags": "story",
41
+ "hitsPerPage": max(1, min(int(n), 20)),
42
+ })
43
+ url = f"{HN_API}?{params}"
44
+ try:
45
+ data = json.loads(_http_get(url).decode("utf-8"))
46
+ except Exception as e:
47
+ # 吞异常返空会让 MCP 社交搜索把网络故障静默报成「平台成功、0 结果」
48
+ # (mcp_handlers 的 err=None 设计依赖引擎抛异常或返 error 占位,
49
+ # 见 tests/test_regression_p0p1.py「零结果应以 errors 提示」契约)。
50
+ # 对齐 zhihu_engine 范式:失败必须带着 error 字段浮上来。
51
+ return [{"error": f"hackernews {type(e).__name__}: {e}", "source": "hackernews"}]
52
+
53
+ results: list[dict[str, Any]] = []
54
+ for h in (data.get("hits") or []):
55
+ if not isinstance(h, dict) or not h.get("title"):
56
+ continue
57
+ object_id = str(h.get("objectID") or "")
58
+ item_url = h.get("url") or f"https://news.ycombinator.com/item?id={object_id}"
59
+ points = h.get("points") or 0
60
+ comments = h.get("num_comments") or 0
61
+ author = h.get("author") or ""
62
+ created = h.get("created_at") or None
63
+ text = h.get("story_text") or (h.get("title") or "")
64
+ snippet = (text[:300] if isinstance(text, str) else "")
65
+ results.append({
66
+ "title": h["title"],
67
+ "url": item_url,
68
+ "snippet": snippet,
69
+ "source": "hackernews",
70
+ "score": max(1.0 - len(results) * 0.1, 0.1),
71
+ "published_at": created,
72
+ "social_meta": {
73
+ "platform": "hackernews",
74
+ "content_type": "story",
75
+ "id": object_id,
76
+ "author": author,
77
+ "likes": points,
78
+ "comments": comments,
79
+ "provider": "hn_algolia",
80
+ },
81
+ })
82
+ if len(results) >= n:
83
+ break
84
+ return results
85
+
86
+
87
+ def main():
88
+ import argparse
89
+ parser = argparse.ArgumentParser(description="Hacker News search engine (HN Algolia)")
90
+ parser.add_argument("action", nargs="?", default="search")
91
+ parser.add_argument("query", nargs="?")
92
+ parser.add_argument("-n", type=int, default=5)
93
+ parser.add_argument("--json", action="store_true")
94
+ args = parser.parse_args()
95
+
96
+ if not args.query:
97
+ print("[]")
98
+ return
99
+ results = search(args.query, args.n)
100
+ if args.json:
101
+ print(json.dumps(results, ensure_ascii=False))
102
+ else:
103
+ for i, r in enumerate(results, 1):
104
+ meta = r.get("social_meta") or {}
105
+ print(f"### {i}. {r['title']}")
106
+ print(f"- **URL**: {r['url']}")
107
+ print(f"- {r['snippet'][:200]}")
108
+ print(f"- points={meta.get('likes')} comments={meta.get('comments')} @{meta.get('author', '')}")
109
+ print()
110
+
111
+
112
+ if __name__ == "__main__":
113
+ main()
@@ -0,0 +1,172 @@
1
+ #!/usr/bin/env python3
2
+ """Reddit 搜索引擎
3
+
4
+ 使用 Reddit JSON API(无需认证即可搜索公开内容)。
5
+ 也可通过 rdt CLI 或 praw 扩展。
6
+ """
7
+
8
+ import json
9
+ import subprocess
10
+ import time
11
+ import urllib.error
12
+ import urllib.parse
13
+ import urllib.request
14
+
15
+ # 出口调度唯一入口(issue #13 同类修复):urlopen 原生只认标准 HTTP(S)_PROXY
16
+ # 环境变量,**不认** config.yaml 的 network.proxy —— 裸用会在「必须经代理才能
17
+ # 出网」的环境里整源连不上。本文件在 scripts/ 子目录,父目录不在 sys.path 上。
18
+ import os
19
+ import sys
20
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
21
+ from net_proxy import open_url # noqa: E402
22
+
23
+
24
+ def _http_get_with_retry(url: str, headers: dict, timeout: int = 10, max_retries: int = 2):
25
+ """带重试的 HTTP GET,尊重 429 + Retry-After。
26
+
27
+ 仅使用 stdlib,不引入第三方依赖。
28
+ 返回 (body_bytes, status_code)。失败时抛出最后一次异常。
29
+ """
30
+ for attempt in range(max_retries + 1):
31
+ try:
32
+ req = urllib.request.Request(url, headers=headers)
33
+ with open_url(req, timeout=timeout) as resp:
34
+ return resp.read(), resp.status
35
+ except urllib.error.HTTPError as e:
36
+ if e.code == 429 and attempt < max_retries:
37
+ retry_after = int(e.headers.get("Retry-After", "5"))
38
+ time.sleep(min(retry_after, 30))
39
+ continue
40
+ raise
41
+ except (urllib.error.URLError, OSError):
42
+ if attempt < max_retries:
43
+ time.sleep(2 ** attempt + 0.5)
44
+ continue
45
+ raise
46
+ return b"", 0
47
+
48
+
49
+ def search_reddit_api(query: str, n: int = 5) -> list[dict]:
50
+ """通过 Reddit JSON API 搜索"""
51
+ encoded = urllib.parse.quote(query)
52
+ url = f"https://www.reddit.com/search.json?q={encoded}&limit={n}&sort=relevance"
53
+ headers = {"User-Agent": "argo-search/1.0 (by taxueseek)"}
54
+ try:
55
+ body, _ = _http_get_with_retry(url, headers, timeout=10, max_retries=2)
56
+ data = json.loads(body.decode("utf-8"))
57
+ return _parse_reddit_response(data, n)
58
+ except Exception as e:
59
+ # 吞异常返空会让 MCP 社交搜索把网络故障静默报成「平台成功、0 结果」
60
+ # (mcp_handlers 的 err=None 设计依赖引擎抛异常或返 error 占位,
61
+ # 见 tests/test_regression_p0p1.py「零结果应以 errors 提示」契约)。
62
+ # 对齐 zhihu_engine 范式:失败必须带着 error 字段浮上来。
63
+ return [{"error": f"reddit {type(e).__name__}: {e}", "source": "reddit"}]
64
+
65
+
66
+ def _parse_reddit_response(data: dict, n: int) -> list[dict]:
67
+ """解析 Reddit API 响应"""
68
+ results = []
69
+ for child in data.get("data", {}).get("children", [])[:n]:
70
+ post = child.get("data", {})
71
+ if not post:
72
+ continue
73
+ title = post.get("title", "")
74
+ selftext = post.get("selftext", "")
75
+ results.append({
76
+ "title": title[:100] + ("..." if len(title) > 100 else ""),
77
+ "url": f"https://reddit.com{post.get('permalink', '')}",
78
+ "snippet": (selftext or title)[:300],
79
+ "source": "reddit",
80
+ "score": max(1.0 - len(results) * 0.1, 0.1),
81
+ "social_meta": {
82
+ "platform": "reddit",
83
+ "content_type": "post",
84
+ "subreddit": post.get("subreddit", ""),
85
+ "author": post.get("author", ""),
86
+ "upvotes": post.get("ups", 0),
87
+ "comments": post.get("num_comments", 0),
88
+ "awards": post.get("total_awards_received", 0),
89
+ }
90
+ })
91
+ return results
92
+
93
+
94
+ def search(query: str, n: int = 5) -> list[dict]:
95
+ """主搜索入口"""
96
+ # 优先尝试 rdt CLI
97
+ try:
98
+ result = subprocess.run(
99
+ ["rdt", "search", query, "--limit", str(n), "--json"],
100
+ capture_output=True, text=True, timeout=10,
101
+ encoding="utf-8", errors="replace", # Windows GBK locale 下 UTF-8 输出会解码崩
102
+ )
103
+ if result.returncode == 0 and result.stdout.strip():
104
+ return _parse_rdt_json(result.stdout, n)
105
+ except (FileNotFoundError, subprocess.TimeoutExpired):
106
+ pass
107
+
108
+ # fallback: Reddit JSON API
109
+ return search_reddit_api(query, n)
110
+
111
+
112
+ def _parse_rdt_json(raw: str, n: int) -> list[dict]:
113
+ """解析 rdt CLI JSON 输出"""
114
+ results = []
115
+ try:
116
+ data = json.loads(raw)
117
+ if isinstance(data, list):
118
+ items = data
119
+ elif isinstance(data, dict):
120
+ items = data.get("results", data.get("posts", []))
121
+ else:
122
+ items = []
123
+ for post in items[:n]:
124
+ if not isinstance(post, dict):
125
+ continue
126
+ title = post.get("title", "")
127
+ results.append({
128
+ "title": title,
129
+ "url": post.get("url", post.get("permalink", "")),
130
+ "snippet": (post.get("selftext", "") or title)[:300],
131
+ "source": "reddit",
132
+ "score": max(1.0 - len(results) * 0.1, 0.1),
133
+ "social_meta": {
134
+ "platform": "reddit",
135
+ "content_type": "post",
136
+ "subreddit": post.get("subreddit", ""),
137
+ "author": post.get("author", ""),
138
+ "upvotes": post.get("ups", post.get("upvotes", 0)),
139
+ "comments": post.get("num_comments", 0),
140
+ }
141
+ })
142
+ except (json.JSONDecodeError, ValueError):
143
+ pass
144
+ return results
145
+
146
+
147
+ def main():
148
+ import argparse
149
+ parser = argparse.ArgumentParser(description="Reddit search engine")
150
+ parser.add_argument("action", nargs="?", default="search")
151
+ parser.add_argument("query", nargs="?")
152
+ parser.add_argument("-n", type=int, default=5)
153
+ parser.add_argument("--json", action="store_true")
154
+ args = parser.parse_args()
155
+
156
+ if not args.query:
157
+ print("[]")
158
+ return
159
+
160
+ results = search(args.query, args.n)
161
+ if args.json:
162
+ print(json.dumps(results, ensure_ascii=False))
163
+ else:
164
+ for i, r in enumerate(results, 1):
165
+ print(f"### {i}. {r['title']}")
166
+ print(f"- **URL**: {r['url']}")
167
+ print(f"- {r['snippet'][:200]}")
168
+ print()
169
+
170
+
171
+ if __name__ == "__main__":
172
+ main()
@@ -0,0 +1,397 @@
1
+ #!/usr/bin/env python3
2
+ """Twitter/X 搜索引擎
3
+
4
+ 后端优先级:
5
+ 1. FxTwitter API v2(api.fxtwitter.com,零 Key)— 主路径
6
+ 2. twitter CLI(tw)
7
+ 3. nitter 公开实例 HTML
8
+
9
+ FxTwitter 能力:
10
+ - GET /2/search?q=...&count=N 推文搜索
11
+ - GET /2/status/{id} 单条推文
12
+ - 旧版 GET /:user?/status/:id 兼容
13
+
14
+ 文档:https://docs.fxembed.com/api/twitter/
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import json
20
+ import re
21
+ import subprocess
22
+ import sys
23
+ import time
24
+ import urllib.parse
25
+ from typing import Any
26
+
27
+ # 出口调度唯一入口(issue #13 同类修复):urlopen 不认 config.yaml 的
28
+ # network.proxy,裸用会在「必须经代理才能出网」的环境里整源连不上。
29
+ import os
30
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
31
+ from fxtwitter_api import extract_status_id as _fx_extract_status_id # noqa: E402
32
+ from fxtwitter_api import http_get as _fx_http_get # noqa: E402
33
+
34
+ FXTWITTER_BASE = "https://api.fxtwitter.com"
35
+ USER_AGENT = "argo-search/1.0 (+https://github.com/taxueseek/argo; fxtwitter)"
36
+
37
+ # 推文 URL / snowflake id 的解析与 HTTP 出口统一收在 fxtwitter_api(打包器
38
+ # tweet.py 用的是同一份);本模块只保留「引擎侧阈值」——纯数字要求 ≥10 位,
39
+ # 见 _extract_status_id。此前这里自带一份正则,与打包器那份迟早漂移。
40
+
41
+
42
+ def _http_get_with_retry(
43
+ url: str,
44
+ headers: dict | None = None,
45
+ timeout: int = 10,
46
+ max_retries: int = 2,
47
+ ) -> tuple[bytes, int]:
48
+ """带重试的 HTTP GET(实现唯一在 fxtwitter_api.http_get)。
49
+
50
+ 保留本名与签名:既有测试 patch 的正是 `tw._http_get_with_retry`,薄壳让
51
+ patch 语义不变;同时引擎与打包器共用同一份「重试 + 代理 + SSRF 守卫」纪律,
52
+ 不再各写一遍。
53
+ """
54
+ return _fx_http_get(url, headers=headers, timeout=timeout,
55
+ max_retries=max_retries)
56
+ return b"", 0
57
+
58
+
59
+ def _extract_status_id(query: str) -> str | None:
60
+ """从查询中提取推文 ID(URL 或纯数字)。
61
+
62
+ 解析器唯一实现在 fxtwitter_api.extract_status_id;这里只固定搜索侧的阈值
63
+ min_digits=10——`argo search "20"` 里的 20 可能是页码或数量,不该当推文 ID;
64
+ 而 `argo tweet "20"` 是用户显式给的 ID(那边传 1)。
65
+ """
66
+ return _fx_extract_status_id(query or "", min_digits=10)
67
+
68
+
69
+ def _status_to_result(item: dict[str, Any], rank: int = 0) -> dict[str, Any] | None:
70
+ """将 FxTwitter status 对象转为统一 schema。"""
71
+ if not isinstance(item, dict):
72
+ return None
73
+ # v2 status 可能包在 status 键下,或直接是 status 对象
74
+ if "text" not in item and isinstance(item.get("status"), dict):
75
+ item = item["status"]
76
+ text = (item.get("text") or "").strip()
77
+ url = item.get("url") or ""
78
+ sid = str(item.get("id") or "")
79
+ if not url and sid:
80
+ author = item.get("author") if isinstance(item.get("author"), dict) else {}
81
+ handle = author.get("screen_name") or "i"
82
+ url = f"https://x.com/{handle}/status/{sid}"
83
+ if not text and not url:
84
+ return None
85
+
86
+ author = item.get("author") if isinstance(item.get("author"), dict) else {}
87
+ likes = item.get("likes") or 0
88
+ reposts = item.get("reposts") if item.get("reposts") is not None else item.get("retweets") or 0
89
+ replies = item.get("replies") or 0
90
+ views = item.get("views")
91
+ quotes = item.get("quotes") or 0
92
+ bookmarks = item.get("bookmarks") or 0
93
+
94
+ title = text[:100] + ("..." if len(text) > 100 else "")
95
+ if not title:
96
+ title = f"Tweet {sid}" if sid else url
97
+
98
+ # 互动分:简单归一化,便于排序参考
99
+ engagement = int(likes or 0) + int(reposts or 0) * 2 + int(replies or 0)
100
+ score = max(1.0 - rank * 0.05, 0.1)
101
+ if engagement > 0:
102
+ score = min(1.0, score + min(engagement / 10000.0, 0.3))
103
+
104
+ return {
105
+ "title": title,
106
+ "url": url,
107
+ "snippet": text[:300],
108
+ "source": "twitter",
109
+ "score": round(score, 3),
110
+ "published_at": item.get("created_at") or None,
111
+ "social_meta": {
112
+ "platform": "twitter",
113
+ "content_type": "tweet",
114
+ "id": sid,
115
+ "author": author.get("screen_name") or "",
116
+ "author_name": author.get("name") or "",
117
+ "likes": likes,
118
+ "retweets": reposts,
119
+ "reposts": reposts,
120
+ "replies": replies,
121
+ "views": views,
122
+ "quotes": quotes,
123
+ "bookmarks": bookmarks,
124
+ "lang": item.get("lang"),
125
+ "provider": "fxtwitter",
126
+ },
127
+ }
128
+
129
+
130
+ def fetch_status(status_id: str, timeout: int = 10) -> list[dict[str, Any]]:
131
+ """按推文 ID 拉取单条(v2 优先,旧端点保底)。"""
132
+ endpoints = [
133
+ f"{FXTWITTER_BASE}/2/status/{status_id}",
134
+ f"{FXTWITTER_BASE}/status/{status_id}",
135
+ ]
136
+ last_err: Exception | None = None
137
+ for url in endpoints:
138
+ try:
139
+ body, _ = _http_get_with_retry(url, timeout=timeout, max_retries=2)
140
+ data = json.loads(body.decode("utf-8"))
141
+ if not isinstance(data, dict):
142
+ continue
143
+ code = data.get("code", 200)
144
+ if code not in (200, None, "200"):
145
+ continue
146
+ # v2: {status: {...}} ; v1: {tweet: {...}}
147
+ item = data.get("status") or data.get("tweet") or data
148
+ result = _status_to_result(item if isinstance(item, dict) else {}, rank=0)
149
+ if result:
150
+ return [result]
151
+ except Exception as e:
152
+ last_err = e
153
+ continue
154
+ if last_err is not None:
155
+ # 全部端点异常 ≠ 「没有这条推文」:带 error 浮上来(同 search_fxtwitter)
156
+ return [{"error": f"twitter {type(last_err).__name__}: {last_err}",
157
+ "source": "twitter"}]
158
+ return []
159
+
160
+
161
+ def search_fxtwitter(query: str, n: int = 5, timeout: int = 10) -> list[dict[str, Any]]:
162
+ """通过 FxTwitter /2/search 搜索推文。"""
163
+ if not query or not query.strip():
164
+ return []
165
+
166
+ # URL / ID 查询 → 单条拉取
167
+ status_id = _extract_status_id(query)
168
+ if status_id:
169
+ return fetch_status(status_id, timeout=timeout)[:n]
170
+
171
+ params = urllib.parse.urlencode({"q": query.strip(), "count": max(1, min(int(n), 20))})
172
+ url = f"{FXTWITTER_BASE}/2/search?{params}"
173
+ try:
174
+ body, _ = _http_get_with_retry(url, timeout=timeout, max_retries=2)
175
+ data = json.loads(body.decode("utf-8"))
176
+ except Exception as e:
177
+ # 吞异常返空会让 MCP 社交搜索把网络故障静默报成「平台成功、0 结果」
178
+ # (mcp_handlers 的 err=None 设计依赖引擎抛异常或返 error 占位,
179
+ # 见 tests/test_regression_p0p1.py「零结果应以 errors 提示」契约)。
180
+ # 对齐 zhihu_engine 范式:失败必须带着 error 字段浮上来。
181
+ return [{"error": f"twitter {type(e).__name__}: {e}", "source": "twitter"}]
182
+
183
+ if not isinstance(data, dict):
184
+ return []
185
+ code = data.get("code", 200)
186
+ if code not in (200, None, "200"):
187
+ return []
188
+
189
+ results: list[dict[str, Any]] = []
190
+ for i, item in enumerate(data.get("results") or []):
191
+ if not isinstance(item, dict):
192
+ continue
193
+ # 仅处理 status 类型;忽略用户卡等
194
+ if item.get("type") and item.get("type") != "status":
195
+ continue
196
+ parsed = _status_to_result(item, rank=i)
197
+ if parsed:
198
+ results.append(parsed)
199
+ if len(results) >= n:
200
+ break
201
+ return results
202
+
203
+
204
+ # Nitter 实例健康状态(进程内记忆):记录每个实例的连续失败次数和
205
+ # 最后健康时间,优先尝试最近健康的实例。实例不可达时不永久封禁,
206
+ # 而是逐步降权——Nitter 实例可用性波动大,单次失败不等于永久不可用。
207
+ _nitter_health: dict[str, dict] = {}
208
+ _NITTER_MAX_FAILURES = 3
209
+
210
+
211
+ def _nitter_healthy_instances() -> list[str]:
212
+ """按健康状态排序的 Nitter 实例列表,最近健康的优先。"""
213
+ instances = [
214
+ "https://nitter.net",
215
+ "https://nitter.privacydev.net",
216
+ "https://nitter.poast.org",
217
+ "https://nitter.privacyredirect.com",
218
+ "https://nitter.tiekoetter.com",
219
+ ]
220
+ def _sort_key(base: str) -> tuple:
221
+ h = _nitter_health.get(base)
222
+ if h is None:
223
+ return (0, 0.0) # 未记录过,优先尝试
224
+ fails = h.get("fails", 0)
225
+ last_ok = h.get("last_ok", 0.0)
226
+ return (fails, -last_ok) # 失败少的优先,最近健康的优先
227
+ return sorted(instances, key=_sort_key)
228
+
229
+
230
+ def search_nitter(query: str, n: int = 5) -> list[dict]:
231
+ """通过 nitter 公开实例搜索推文(零认证,保底)。
232
+
233
+ 多实例故障转移:按健康状态排序,优先尝试最近健康的实例。
234
+ 实例失败时降权(不永久封禁),成功时恢复权重。
235
+ """
236
+ encoded_query = urllib.parse.quote(query)
237
+
238
+ for base in _nitter_healthy_instances():
239
+ try:
240
+ url = f"{base}/search?f=tweets&q={encoded_query}&since=2025-01-01"
241
+ headers = {
242
+ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
243
+ }
244
+ body, _ = _http_get_with_retry(url, headers=headers, timeout=8, max_retries=1)
245
+ html = body.decode("utf-8", errors="replace")
246
+ # 更新健康状态
247
+ _nitter_health[base] = {"fails": 0, "last_ok": time.time()}
248
+ results = _parse_nitter_html(html, n)
249
+ # 空结果也算实例健康(只是没有匹配内容),不降权
250
+ return results
251
+ except Exception:
252
+ # 失败时增加失败计数
253
+ h = _nitter_health.get(base) or {}
254
+ h["fails"] = h.get("fails", 0) + 1
255
+ _nitter_health[base] = h
256
+ continue
257
+ return []
258
+
259
+
260
+ def _parse_nitter_html(html: str, n: int) -> list[dict]:
261
+ """解析 nitter 搜索结果 HTML。"""
262
+ results = []
263
+ tweet_pattern = re.compile(
264
+ r'<div class="tweet-content[^"]*".*?'
265
+ r'<a class="tweet-link" href="([^"]+)".*?'
266
+ r'<div class="tweet-content[^"]*"[^>]*>(.*?)</div>',
267
+ re.DOTALL,
268
+ )
269
+ for match in tweet_pattern.finditer(html):
270
+ url = match.group(1)
271
+ content = match.group(2)
272
+ content = re.sub(r"<[^>]+>", " ", content).strip()
273
+ content = re.sub(r"\s+", " ", content)
274
+ if len(content) > 10:
275
+ results.append({
276
+ "title": content[:100] + ("..." if len(content) > 100 else ""),
277
+ "url": url if url.startswith("http") else f"https://twitter.com{url}",
278
+ "snippet": content[:300],
279
+ "source": "twitter",
280
+ "score": max(1.0 - len(results) * 0.1, 0.1),
281
+ "social_meta": {
282
+ "platform": "twitter",
283
+ "content_type": "tweet",
284
+ "provider": "nitter",
285
+ },
286
+ })
287
+ if len(results) >= n:
288
+ break
289
+ return results
290
+
291
+
292
+ def search(query: str, n: int = 5) -> list[dict]:
293
+ """主搜索入口:FxTwitter → tw CLI → nitter。"""
294
+ # 1) FxTwitter(主路径)
295
+ try:
296
+ results = search_fxtwitter(query, n=n, timeout=10)
297
+ if results:
298
+ return results
299
+ except Exception:
300
+ pass
301
+
302
+ # 2) twitter CLI
303
+ try:
304
+ result = subprocess.run(
305
+ ["tw", "search", query, "--limit", str(n), "--json"],
306
+ capture_output=True,
307
+ text=True,
308
+ encoding="utf-8", # Windows GBK locale 下 UTF-8 输出会解码崩
309
+ errors="replace",
310
+ timeout=10,
311
+ )
312
+ if result.returncode == 0 and result.stdout.strip():
313
+ parsed = _parse_tw_json(result.stdout, n)
314
+ if parsed:
315
+ return parsed
316
+ except (FileNotFoundError, subprocess.TimeoutExpired):
317
+ pass
318
+
319
+ # 3) nitter 保底
320
+ return search_nitter(query, n)
321
+
322
+
323
+ def _parse_tw_json(raw: str, n: int) -> list[dict]:
324
+ """解析 twitter CLI JSON 输出。"""
325
+ results = []
326
+ for line in raw.strip().split("\n"):
327
+ try:
328
+ tweet = json.loads(line)
329
+ if not isinstance(tweet, dict):
330
+ continue
331
+ results.append({
332
+ "title": (tweet.get("text", "") or tweet.get("full_text", ""))[:100],
333
+ "url": tweet.get("url", f"https://twitter.com/i/status/{tweet.get('id', '')}"),
334
+ "snippet": tweet.get("text", "") or tweet.get("full_text", ""),
335
+ "source": "twitter",
336
+ "score": max(1.0 - len(results) * 0.1, 0.1),
337
+ "social_meta": {
338
+ "platform": "twitter",
339
+ "content_type": "tweet",
340
+ "author": tweet.get("user", {}).get("screen_name", ""),
341
+ "likes": tweet.get("favorite_count", 0),
342
+ "retweets": tweet.get("retweet_count", 0),
343
+ "replies": tweet.get("reply_count", 0),
344
+ "provider": "tw_cli",
345
+ },
346
+ })
347
+ except (json.JSONDecodeError, ValueError):
348
+ continue
349
+ if len(results) >= n:
350
+ break
351
+ return results
352
+
353
+
354
+ def main():
355
+ import argparse
356
+
357
+ parser = argparse.ArgumentParser(description="Twitter search engine (FxTwitter)")
358
+ parser.add_argument("action", nargs="?", default="search")
359
+ parser.add_argument("query", nargs="?")
360
+ parser.add_argument("-n", type=int, default=5)
361
+ parser.add_argument("--json", action="store_true")
362
+ args = parser.parse_args()
363
+
364
+ if not args.query:
365
+ print("[]")
366
+ return
367
+
368
+ action = (args.action or "search").lower()
369
+ if action == "status" and args.query:
370
+ results = fetch_status(args.query)
371
+ elif action in ("search", "s") and args.query:
372
+ results = search(args.query, args.n)
373
+ elif args.query:
374
+ # 未知 action 时把 query 当搜索词
375
+ results = search(args.query, args.n)
376
+ else:
377
+ # 兼容:`twitter_engine.py "查询词"`
378
+ results = search(args.action, args.n)
379
+
380
+ if args.json:
381
+ print(json.dumps(results, ensure_ascii=False))
382
+ else:
383
+ for i, r in enumerate(results, 1):
384
+ print(f"### {i}. {r['title']}")
385
+ print(f"- **URL**: {r['url']}")
386
+ print(f"- {r['snippet'][:200]}")
387
+ meta = r.get("social_meta") or {}
388
+ if meta.get("likes") is not None:
389
+ print(
390
+ f"- likes={meta.get('likes')} reposts={meta.get('retweets') or meta.get('reposts')} "
391
+ f"replies={meta.get('replies')} @{meta.get('author', '')}"
392
+ )
393
+ print()
394
+
395
+
396
+ if __name__ == "__main__":
397
+ main()