argo-search 1.0.1 → 2.9.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (472) hide show
  1. package/LICENSE +201 -21
  2. package/README.en.md +735 -0
  3. package/README.es.md +777 -0
  4. package/README.ja.md +777 -0
  5. package/README.ko.md +777 -0
  6. package/README.md +778 -320
  7. package/SKILL.md +135 -0
  8. package/assets/readme/hero.svg +121 -0
  9. package/assets/readme/made-with-beautify.svg +10 -0
  10. package/assets/readme/proof-routes.svg +73 -0
  11. package/assets/readme/why-better.svg +104 -0
  12. package/assets/readme/workflow.svg +83 -0
  13. package/backends/domain_profiles.json +2356 -96
  14. package/backends/engine_registry.yaml +3428 -503
  15. package/backends/query_synonyms_cn.json +35 -0
  16. package/backends/quota_profiles.json +2488 -160
  17. package/backends/source_types_cn.json +109 -0
  18. package/backends/text_lexicon.json +44 -0
  19. package/bin/argo +772 -0
  20. package/bin/argo.js +102 -39
  21. package/config.yaml +5139 -367
  22. package/cordis.patch.yml +21 -0
  23. package/data/countries.json +40 -0
  24. package/data/lang_matrix/lang_capability.json +842 -0
  25. package/data/lang_matrix/matrix_2026-09-10.json +11921 -0
  26. package/data/regions_cn.json +1 -0
  27. package/docs/ADDING_NEW_ENGINE.md +184 -0
  28. package/docs/ARGO_INTRO.md +124 -0
  29. package/docs/DATA_SOURCE_GAPS.md +81 -0
  30. package/docs/DESIGN_LOCAL_DATA_FUSION.md +90 -0
  31. package/docs/ENGINE_CATALOG.md +575 -0
  32. package/docs/MCP_SETUP.md +108 -0
  33. package/docs/PLATFORM_PATHS.md +111 -0
  34. package/docs/RELEASE_NOTES_v2.8.0.md +71 -0
  35. package/docs/RELEASE_NOTES_v2.8.2.md +63 -0
  36. package/docs/RELEASE_NOTES_v2.8.3.md +55 -0
  37. package/docs/RELEASE_NOTES_v2.8.4.md +79 -0
  38. package/docs/RELEASE_NOTES_v2.8.5.md +102 -0
  39. package/docs/RELEASE_NOTES_v2.8.6.md +66 -0
  40. package/docs/RELEASE_NOTES_v2.8.7.md +65 -0
  41. package/docs/RELEASE_NOTES_v2.8.8.md +67 -0
  42. package/docs/RELEASE_NOTES_v2.8.9.md +67 -0
  43. package/docs/RELEASE_NOTES_v2.9.0.md +64 -0
  44. package/docs/RELEASE_NOTES_v2.9.8.md +73 -0
  45. package/docs/SEARCH_ARCHIVE.md +115 -0
  46. package/docs/assets/architecture.svg +104 -0
  47. package/docs/assets/hero.svg +98 -0
  48. package/docs/engines/bangumi.md +27 -0
  49. package/docs/engines/biorxiv.md +27 -0
  50. package/docs/engines/clawhub.md +27 -0
  51. package/docs/engines/crt_sh.md +27 -0
  52. package/docs/engines/datacite.md +27 -0
  53. package/docs/engines/deps_dev.md +27 -0
  54. package/docs/engines/douban_movie.md +27 -0
  55. package/docs/engines/endoflife.md +27 -0
  56. package/docs/engines/flk_law.md +27 -0
  57. package/docs/engines/fxtwitter.md +27 -0
  58. package/docs/engines/google_news.md +27 -0
  59. package/docs/engines/gov_regulations.md +27 -0
  60. package/docs/engines/guqian.md +27 -0
  61. package/docs/engines/hackernews.md +27 -0
  62. package/docs/engines/iplant.md +27 -0
  63. package/docs/engines/keenable.md +27 -0
  64. package/docs/engines/met_museum.md +27 -0
  65. package/docs/engines/nasa_images.md +27 -0
  66. package/docs/engines/netease_music.md +27 -0
  67. package/docs/engines/npm.md +27 -0
  68. package/docs/engines/nvd.md +27 -0
  69. package/docs/engines/openreview.md +27 -0
  70. package/docs/engines/openstd.md +27 -0
  71. package/docs/engines/parallel_free.md +27 -0
  72. package/docs/engines/people_daily.md +27 -0
  73. package/docs/engines/realtime_index.md +27 -0
  74. package/docs/engines/sec_edgar.md +27 -0
  75. package/docs/engines/seltz.md +27 -0
  76. package/docs/engines/skillsmp.md +27 -0
  77. package/docs/engines/sspai.md +27 -0
  78. package/docs/engines/std_samr.md +27 -0
  79. package/docs/engines/train.md +27 -0
  80. package/docs/engines/un_comtrade.md +27 -0
  81. package/docs/engines/wikisource.md +27 -0
  82. package/docs/engines/zdic.md +27 -0
  83. package/docs/engines/zenodo.md +27 -0
  84. package/docs/engines/zhihu_hot_app.md +27 -0
  85. package/docs//344/270/272/344/273/200/344/271/210/351/200/211/346/213/251argo.md +176 -0
  86. package/engines/README.md +10 -0
  87. package/engines/_template_cli.yaml +39 -0
  88. package/engines/_template_http.yaml +37 -0
  89. package/engines/plugins/_template_plugin.py +50 -0
  90. package/engines/specs/_cisa_kev.yaml +54 -0
  91. package/engines/specs/_osv.yaml +63 -0
  92. package/engines/specs/aljazeera.yaml +43 -0
  93. package/engines/specs/antara.yaml +43 -0
  94. package/engines/specs/artic.yaml +20 -0
  95. package/engines/specs/aviation_weather.yaml +39 -0
  96. package/engines/specs/bangumi.yaml +21 -0
  97. package/engines/specs/biorxiv.yaml +24 -0
  98. package/engines/specs/carbon_intensity.yaml +32 -0
  99. package/engines/specs/cfets.yaml +41 -0
  100. package/engines/specs/cinii.yaml +44 -0
  101. package/engines/specs/clawhub.yaml +49 -0
  102. package/engines/specs/cleveland.yaml +19 -0
  103. package/engines/specs/cn_ai_news.yaml +43 -0
  104. package/engines/specs/core.yaml +35 -0
  105. package/engines/specs/crt_sh.yaml +38 -0
  106. package/engines/specs/ct24.yaml +43 -0
  107. package/engines/specs/datacite.yaml +27 -0
  108. package/engines/specs/datagov_uk.yaml +41 -0
  109. package/engines/specs/ddgs_images.yaml +45 -0
  110. package/engines/specs/deepseek_search.yaml +74 -0
  111. package/engines/specs/deezer.yaml +20 -0
  112. package/engines/specs/defillama.yaml +38 -0
  113. package/engines/specs/deps_dev.yaml +21 -0
  114. package/engines/specs/digup.yaml +36 -0
  115. package/engines/specs/doi.yaml +39 -0
  116. package/engines/specs/douban_movie.yaml +22 -0
  117. package/engines/specs/dw_news.yaml +47 -0
  118. package/engines/specs/egov_law.yaml +21 -0
  119. package/engines/specs/elpais.yaml +43 -0
  120. package/engines/specs/endoflife.yaml +21 -0
  121. package/engines/specs/energy_charts.yaml +21 -0
  122. package/engines/specs/factcheck_org.yaml +52 -0
  123. package/engines/specs/faz.yaml +43 -0
  124. package/engines/specs/federal_register.yaml +57 -0
  125. package/engines/specs/firecrawl.yaml +53 -0
  126. package/engines/specs/flk_law.yaml +24 -0
  127. package/engines/specs/folha.yaml +43 -0
  128. package/engines/specs/france24.yaml +44 -0
  129. package/engines/specs/full_fact.yaml +47 -0
  130. package/engines/specs/fxtwitter.yaml +36 -0
  131. package/engines/specs/gbfs_nyc.yaml +32 -0
  132. package/engines/specs/gdacs.yaml +21 -0
  133. package/engines/specs/gdelt.yaml +64 -0
  134. package/engines/specs/gleif.yaml +55 -0
  135. package/engines/specs/google_news.yaml +22 -0
  136. package/engines/specs/gov_regulations.yaml +38 -0
  137. package/engines/specs/guardian_rss.yaml +59 -0
  138. package/engines/specs/hkex.yaml +40 -0
  139. package/engines/specs/hurriyet.yaml +43 -0
  140. package/engines/specs/iplant.yaml +25 -0
  141. package/engines/specs/jikan.yaml +24 -0
  142. package/engines/specs/jolpica.yaml +20 -0
  143. package/engines/specs/k10plus.yaml +20 -0
  144. package/engines/specs/keenable.yaml +44 -0
  145. package/engines/specs/keenable_free.yaml +36 -0
  146. package/engines/specs/lefigaro.yaml +43 -0
  147. package/engines/specs/listenbrainz.yaml +19 -0
  148. package/engines/specs/matichon.yaml +43 -0
  149. package/engines/specs/met_museum.yaml +21 -0
  150. package/engines/specs/nasa_images.yaml +45 -0
  151. package/engines/specs/netease_music.yaml +38 -0
  152. package/engines/specs/nhk.yaml +43 -0
  153. package/engines/specs/nhtsa_vpic.yaml +20 -0
  154. package/engines/specs/noaa_swpc.yaml +20 -0
  155. package/engines/specs/nvd.yaml +38 -0
  156. package/engines/specs/obis.yaml +20 -0
  157. package/engines/specs/opencitations.yaml +52 -0
  158. package/engines/specs/openf1.yaml +21 -0
  159. package/engines/specs/openfoodfacts.yaml +45 -0
  160. package/engines/specs/openligadb.yaml +20 -0
  161. package/engines/specs/openreview.yaml +37 -0
  162. package/engines/specs/openstd.yaml +21 -0
  163. package/engines/specs/people_daily.yaml +22 -0
  164. package/engines/specs/realtime_index.yaml +34 -0
  165. package/engines/specs/ror.yaml +20 -0
  166. package/engines/specs/satnogs.yaml +20 -0
  167. package/engines/specs/sec_edgar.yaml +46 -0
  168. package/engines/specs/shfe.yaml +39 -0
  169. package/engines/specs/skillsmp.yaml +50 -0
  170. package/engines/specs/sogou_weixin_browser.yaml +30 -0
  171. package/engines/specs/soilgrids.yaml +25 -0
  172. package/engines/specs/sspai.yaml +36 -0
  173. package/engines/specs/stackexchange.yaml +46 -0
  174. package/engines/specs/std_samr.yaml +21 -0
  175. package/engines/specs/tass.yaml +43 -0
  176. package/engines/specs/tavily_free.yaml +56 -0
  177. package/engines/specs/tle_mirror.yaml +21 -0
  178. package/engines/specs/train.yaml +37 -0
  179. package/engines/specs/tvmaze.yaml +20 -0
  180. package/engines/specs/twitter_syndication.yaml +31 -0
  181. package/engines/specs/un_comtrade.yaml +23 -0
  182. package/engines/specs/unpaywall.yaml +54 -0
  183. package/engines/specs/vnpress.yaml +43 -0
  184. package/engines/specs/weather.yaml +32 -0
  185. package/engines/specs/who_don.yaml +21 -0
  186. package/engines/specs/who_gho.yaml +20 -0
  187. package/engines/specs/wikimedia_commons.yaml +34 -0
  188. package/engines/specs/wikisource.yaml +22 -0
  189. package/engines/specs/worms.yaml +19 -0
  190. package/engines/specs/x_trending.yaml +34 -0
  191. package/engines/specs/yna.yaml +43 -0
  192. package/engines/specs/ynet.yaml +43 -0
  193. package/engines/specs/yuanbao.yaml +93 -0
  194. package/engines/specs/zbmath.yaml +40 -0
  195. package/engines/specs/zdic.yaml +22 -0
  196. package/engines/specs/zenodo.yaml +27 -0
  197. package/engines/specs/zhihu_hot_app.yaml +35 -0
  198. package/mcp/clients.yaml +106 -0
  199. package/package.json +55 -12
  200. package/packages/dsh-plugin/README.md +152 -0
  201. package/packages/dsh-plugin/cordis.patch.yml +48 -0
  202. package/packages/dsh-plugin/dsh/client.js +1533 -0
  203. package/packages/dsh-plugin/dsh/diagnostics.js +54 -0
  204. package/packages/dsh-plugin/dsh/index.js +527 -0
  205. package/packages/dsh-plugin/dsh/native-tools.mjs +749 -0
  206. package/packages/dsh-plugin/dsh/runner.js +666 -0
  207. package/packages/dsh-plugin/dsh/wide-research.js +878 -0
  208. package/packages/dsh-plugin/package.json +38 -0
  209. package/packages/dsh-plugin/scripts/sync-local.sh +70 -0
  210. package/packages/dsh-plugin/test/client.test.js +406 -0
  211. package/packages/dsh-plugin/test/config.test.js +30 -0
  212. package/packages/dsh-plugin/test/diagnostics.test.js +28 -0
  213. package/packages/dsh-plugin/test/index.test.js +1382 -0
  214. package/references/query-templates.md +96 -0
  215. package/references/query-templates.yaml +118 -0
  216. package/scripts/ab_eval.py +219 -0
  217. package/scripts/ab_eval_p0p1.py +266 -0
  218. package/scripts/adaptive.py +294 -70
  219. package/scripts/answer.py +154 -0
  220. package/scripts/archive_run.py +591 -0
  221. package/scripts/argo_engine_registry.py +172 -0
  222. package/scripts/argo_paths.py +728 -0
  223. package/scripts/article.py +136 -0
  224. package/scripts/article_meta.py +131 -0
  225. package/scripts/arxiv_source.py +201 -0
  226. package/scripts/badcase_ledger.py +252 -0
  227. package/scripts/batch_probe.py +241 -0
  228. package/scripts/benchmark.py +19 -0
  229. package/scripts/bounded_run.py +144 -0
  230. package/scripts/browser_auth.py +323 -0
  231. package/scripts/budget.py +247 -0
  232. package/scripts/budget_cli.py +93 -0
  233. package/scripts/build_release_assets.sh +65 -0
  234. package/scripts/burstiness_detector.py +147 -0
  235. package/scripts/cache.py +1066 -67
  236. package/scripts/cache_guard.py +271 -0
  237. package/scripts/cache_key_vdom.py +56 -0
  238. package/scripts/cache_similarity.py +170 -0
  239. package/scripts/caliber_isolation.py +195 -0
  240. package/scripts/calibrate_lowquality.py +210 -0
  241. package/scripts/candidate_envelope.py +366 -0
  242. package/scripts/chrome_cdp.py +871 -0
  243. package/scripts/ci_drift_gate.py +90 -0
  244. package/scripts/circuit_breaker.py +389 -0
  245. package/scripts/citations.py +334 -0
  246. package/scripts/cite_cli.py +59 -0
  247. package/scripts/cjk_tokens.py +95 -0
  248. package/scripts/clarify.py +47 -24
  249. package/scripts/cli_io.py +240 -0
  250. package/scripts/config.py +683 -45
  251. package/scripts/config_cache.py +402 -0
  252. package/scripts/conn_pool.py +220 -0
  253. package/scripts/content_boundary.py +46 -0
  254. package/scripts/content_security.py +685 -0
  255. package/scripts/content_signals.py +751 -0
  256. package/scripts/crawl.py +74 -35
  257. package/scripts/cross_lingual_trace.py +293 -0
  258. package/scripts/daemon.py +866 -0
  259. package/scripts/digup_bridge.py +214 -0
  260. package/scripts/dispatch_result.py +98 -0
  261. package/scripts/engine_admission.py +366 -0
  262. package/scripts/engine_dispatch.py +982 -0
  263. package/scripts/engine_env.py +539 -0
  264. package/scripts/engine_failure.py +411 -0
  265. package/scripts/engine_families.py +738 -0
  266. package/scripts/engine_policy.py +218 -0
  267. package/scripts/engine_requires.py +190 -0
  268. package/scripts/engine_status.py +322 -0
  269. package/scripts/engine_validate.py +638 -0
  270. package/scripts/engines.py +799 -294
  271. package/scripts/engines_base.py +1521 -0
  272. package/scripts/engines_builders.py +310 -0
  273. package/scripts/engines_builders_batch10.py +204 -0
  274. package/scripts/engines_builders_batch11.py +174 -0
  275. package/scripts/engines_builders_batch12.py +103 -0
  276. package/scripts/engines_builders_batch13.py +340 -0
  277. package/scripts/engines_builders_batch14.py +656 -0
  278. package/scripts/engines_builders_batch15.py +249 -0
  279. package/scripts/engines_builders_batch9.py +1468 -0
  280. package/scripts/engines_builders_cn.py +2283 -0
  281. package/scripts/engines_builders_data.py +2923 -0
  282. package/scripts/engines_builders_data_macro.py +775 -0
  283. package/scripts/engines_builders_feeds.py +255 -0
  284. package/scripts/engines_builders_intl.py +1239 -0
  285. package/scripts/engines_builders_keyless.py +111 -0
  286. package/scripts/engines_builders_search.py +440 -0
  287. package/scripts/engines_builders_tech.py +1359 -0
  288. package/scripts/evidence.py +620 -166
  289. package/scripts/evidence_loop.py +624 -0
  290. package/scripts/evidence_tier.py +298 -0
  291. package/scripts/except_sets.py +25 -0
  292. package/scripts/extract.py +55 -16
  293. package/scripts/fact_align.py +207 -0
  294. package/scripts/fact_crosscheck.py +230 -0
  295. package/scripts/fetch.py +25 -12
  296. package/scripts/fetch_browser.py +87 -0
  297. package/scripts/fetch_jsrun.py +122 -0
  298. package/scripts/fetch_quality.py +225 -0
  299. package/scripts/fetch_render_tinyfish.py +117 -0
  300. package/scripts/fetch_v3.py +1804 -0
  301. package/scripts/focus_extract.py +350 -0
  302. package/scripts/fulltext_store.py +169 -0
  303. package/scripts/fxtwitter_api.py +325 -0
  304. package/scripts/gen_engine_catalog.py +364 -0
  305. package/scripts/gen_native_tools.py +91 -0
  306. package/scripts/gen_routing_snapshot.py +195 -0
  307. package/scripts/health_check.py +140 -402
  308. package/scripts/health_probe.py +23 -8
  309. package/scripts/hot_state.py +134 -0
  310. package/scripts/http_client.py +1053 -0
  311. package/scripts/image_ops.py +430 -0
  312. package/scripts/image_vision/vision_probe.swift +126 -0
  313. package/scripts/install.ps1 +166 -0
  314. package/scripts/install.sh +221 -0
  315. package/scripts/job.py +1186 -0
  316. package/scripts/jump_resolver.py +235 -0
  317. package/scripts/lang_capability.py +150 -0
  318. package/scripts/lang_detect.py +289 -0
  319. package/scripts/lang_matrix_probe.py +329 -0
  320. package/scripts/lang_pref.py +311 -0
  321. package/scripts/link_source.py +340 -0
  322. package/scripts/local_image.py +970 -0
  323. package/scripts/local_seek.py +402 -0
  324. package/scripts/macro_countries.py +97 -0
  325. package/scripts/matrix_search_eval.py +1187 -0
  326. package/scripts/mcp_compact.py +234 -0
  327. package/scripts/mcp_diag.py +8 -3
  328. package/scripts/mcp_handlers.py +492 -0
  329. package/scripts/mcp_handlers_surface.py +157 -0
  330. package/scripts/mcp_launch.sh +124 -0
  331. package/scripts/mcp_local.py +241 -0
  332. package/scripts/mcp_server.py +125 -419
  333. package/scripts/mcp_setup.py +431 -0
  334. package/scripts/mcp_tools.py +331 -0
  335. package/scripts/mcp_transport.py +191 -0
  336. package/scripts/mcp_warm.py +127 -0
  337. package/scripts/min_version_scan.py +335 -0
  338. package/scripts/net_proxy.py +258 -0
  339. package/scripts/network_aware.py +152 -0
  340. package/scripts/output_transparency.py +200 -0
  341. package/scripts/paid_policy.py +98 -0
  342. package/scripts/paper.py +274 -0
  343. package/scripts/pdf_extract.py +518 -0
  344. package/scripts/phrase_filter.py +66 -0
  345. package/scripts/plan.py +422 -0
  346. package/scripts/postprocess_signals.py +150 -0
  347. package/scripts/pre-commit +48 -0
  348. package/scripts/query_enhance.py +157 -0
  349. package/scripts/query_operators.py +196 -0
  350. package/scripts/query_rewriter.py +116 -7
  351. package/scripts/query_signals.py +300 -0
  352. package/scripts/query_templates.py +167 -0
  353. package/scripts/query_understanding.py +411 -0
  354. package/scripts/query_variants.py +298 -0
  355. package/scripts/quota.py +633 -76
  356. package/scripts/rank_signals.py +775 -0
  357. package/scripts/ranking_eval.py +319 -0
  358. package/scripts/readability_extract.py +433 -0
  359. package/scripts/recompute.py +415 -0
  360. package/scripts/recovery.py +616 -0
  361. package/scripts/recovery_l0.py +160 -0
  362. package/scripts/recovery_runner.py +150 -0
  363. package/scripts/redskill/redskill_engine.py +332 -0
  364. package/scripts/regression_p0p1.py +318 -0
  365. package/scripts/replay_eval.py +658 -0
  366. package/scripts/rerank_tfidf.py +225 -0
  367. package/scripts/research.py +607 -403
  368. package/scripts/research_cli.py +361 -0
  369. package/scripts/research_dossier.py +492 -0
  370. package/scripts/research_expand.py +169 -0
  371. package/scripts/research_gates.py +272 -0
  372. package/scripts/research_report.py +209 -0
  373. package/scripts/research_strategy.py +37 -0
  374. package/scripts/research_work_packages.py +195 -0
  375. package/scripts/result_lang.py +311 -0
  376. package/scripts/retry_policy.py +76 -0
  377. package/scripts/robots_guard.py +259 -0
  378. package/scripts/route.py +921 -254
  379. package/scripts/route_bangs.py +40 -0
  380. package/scripts/route_cache.py +243 -0
  381. package/scripts/route_combo.py +917 -0
  382. package/scripts/route_domains.py +341 -0
  383. package/scripts/route_lang.py +605 -0
  384. package/scripts/route_log.py +63 -0
  385. package/scripts/route_policy.py +250 -0
  386. package/scripts/rrf_classic.py +66 -0
  387. package/scripts/screenshot.py +73 -0
  388. package/scripts/search.py +886 -406
  389. package/scripts/search_attribution.py +185 -0
  390. package/scripts/search_benchmark.py +407 -0
  391. package/scripts/search_cli.py +640 -0
  392. package/scripts/search_entry.py +395 -0
  393. package/scripts/search_output.py +690 -0
  394. package/scripts/search_pipeline.py +926 -0
  395. package/scripts/search_rank.py +991 -0
  396. package/scripts/search_types.py +129 -16
  397. package/scripts/seek_locator.py +68 -0
  398. package/scripts/semantic_evidence.py +391 -0
  399. package/scripts/serp_guard.py +336 -0
  400. package/scripts/serp_kill_log.py +74 -0
  401. package/scripts/single_flight.py +120 -0
  402. package/scripts/social_engines/__init__.py +10 -0
  403. package/scripts/social_engines/bilibili_engine.py +131 -0
  404. package/scripts/social_engines/hackernews_engine.py +113 -0
  405. package/scripts/social_engines/reddit_engine.py +172 -0
  406. package/scripts/social_engines/twitter_engine.py +397 -0
  407. package/scripts/social_engines/v2ex_engine.py +101 -0
  408. package/scripts/social_engines/weibo_engine.py +128 -0
  409. package/scripts/social_engines/xiaohongshu_engine.py +137 -0
  410. package/scripts/social_engines/zhihu_engine.py +139 -0
  411. package/scripts/social_research.py +201 -0
  412. package/scripts/sogou_weixin_browser.py +318 -0
  413. package/scripts/source_ledger.py +249 -0
  414. package/scripts/stage_timing.py +58 -0
  415. package/scripts/stats_cli.py +107 -0
  416. package/scripts/straggler_policy.py +60 -0
  417. package/scripts/stylometry_detector.py +231 -0
  418. package/scripts/sync_backends.py +353 -0
  419. package/scripts/sync_installs.py +29 -0
  420. package/scripts/template_cli.py +127 -0
  421. package/scripts/text_lexicon.py +72 -0
  422. package/scripts/tfidf_router.py +109 -37
  423. package/scripts/time_utils.py +233 -0
  424. package/scripts/topic_research_profiles.py +643 -0
  425. package/scripts/train.py +737 -0
  426. package/scripts/tweet.py +596 -0
  427. package/scripts/url_canon.py +217 -0
  428. package/scripts/url_optimize.py +56 -0
  429. package/scripts/url_safety.py +236 -0
  430. package/scripts/usage_log.py +116 -0
  431. package/scripts/usage_log_clean.py +113 -0
  432. package/scripts/v2ex_nodes.py +311 -0
  433. package/scripts/variant_recall.py +188 -0
  434. package/scripts/watch.py +205 -0
  435. package/scripts/wechat_article.py +258 -0
  436. package/scripts/wide_research.py +447 -0
  437. package/scripts/wx.py +349 -0
  438. package/scripts/x_article_fetch.py +141 -0
  439. package/scripts/x_search.py +234 -0
  440. package/scripts/x_trending.py +189 -0
  441. package/scripts/yaml_load.py +67 -0
  442. package/scripts/zhihu_quota.py +139 -0
  443. package/sub-skills/ego-search/SKILL.md +253 -0
  444. package/sub-skills/ego-search/scripts/ego_search.py +800 -0
  445. package/sub-skills/ego-search/scripts/merge.py +226 -0
  446. package/sub-skills/ego-search/scripts/quality.py +64 -0
  447. package/sub-skills/ego-search/scripts/runtime.py +172 -0
  448. package/sub-skills/ego-search/scripts/safety.py +109 -0
  449. package/sub-skills/ego-search/scripts/serp_spec.py +109 -0
  450. package/sub-skills/ego-search/scripts/webbridge_adapter.py +374 -0
  451. package/sub-skills/js-run/SKILL.md +67 -0
  452. package/sub-skills/js-run/scripts/jsrun.py +112 -0
  453. package/sub-skills/js-run/scripts/shim.js +188 -0
  454. package/sub-skills/local-search/SKILL.md +50 -53
  455. package/sub-skills/local-search/bm25.py +186 -0
  456. package/sub-skills/local-search/cli_runner.py +331 -0
  457. package/sub-skills/local-search/config.yaml +152 -55
  458. package/sub-skills/local-search/engine_registry.py +91 -26
  459. package/sub-skills/local-search/{health_check.py → local_health_check.py} +140 -10
  460. package/sub-skills/local-search/local_search_adapter.py +13 -2
  461. package/sub-skills/local-search/parse_maps.yaml +40 -42
  462. package/sub-skills/local-search/parsers.py +328 -0
  463. package/sub-skills/local-search/search_v3.py +716 -242
  464. package/sub-skills/local-search/smart_router.py +47 -7
  465. package/sub-skills/local-seek/SKILL.md +110 -0
  466. package/sub-skills/local-seek/config/domains.yaml +20 -0
  467. package/sub-skills/local-seek/scripts/eval_rank.py +216 -0
  468. package/sub-skills/local-seek/scripts/eval_seek.py +224 -0
  469. package/sub-skills/local-seek/scripts/seek.py +1465 -0
  470. package/sub-skills/local-seek/scripts/seek_frecency.py +126 -0
  471. package/sub-skills/local-seek/scripts/seek_fts.py +430 -0
  472. package/sub-skills/local-seek/scripts/seek_rank.py +197 -0
@@ -0,0 +1,871 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ chrome_cdp.py — 零依赖 Chrome DevTools Protocol 驱动
4
+
5
+ 用 Python stdlib(socket + ssl + http.client)直接驱动 Chrome,
6
+ 实现 Hound actions.py 的页面交互能力:
7
+ - click / fill / scroll / press / wait / wait_selector
8
+ - evaluate JavaScript
9
+ - 截图
10
+ - 页面导航 + 内容提取
11
+
12
+ 不依赖 Playwright / Patchright / websocket-client / selenium。
13
+ Chrome 进程由调用方管理,本模块只负责 CDP 协议通信。
14
+
15
+ CDP 协议:https://chromedevtools.github.io/devtools-protocol/
16
+ 帧格式:JSON over HTTP(长连接用于事件)+ WebSocket 风格双向通信。
17
+ 这里用「每命令新建 TCP 连接」的简化模式(Stateless CDP),
18
+ 避免实现 WebSocket 握手的复杂性。
19
+
20
+ 用法:
21
+ from chrome_cdp import ChromeCDP
22
+ cdp = ChromeCDP(port=9222)
23
+ cdp.navigate("https://example.com")
24
+ cdp.wait_for_selector(".content")
25
+ html = cdp.get_html()
26
+ cdp.click("button.load-more")
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import json
32
+ import socket
33
+ import ssl
34
+ import subprocess
35
+ import tempfile
36
+ import time
37
+ import os
38
+ import signal
39
+ import hashlib
40
+ import struct
41
+ import re
42
+ from typing import Any
43
+ from urllib.parse import urlparse
44
+
45
+
46
+ # ─── HTTP/1.1 帧解析(用于从 CDP 流中分离 JSON 消息)────────────────────────
47
+
48
+ def _http_read_response(sock: socket.socket, timeout: float = 10.0) -> dict:
49
+ """从 socket 读取一个完整的 HTTP/1.1 响应(含 chunked body)。"""
50
+ sock.settimeout(timeout)
51
+ data = b""
52
+ # 读 headers 直到 \r\n\r\n
53
+ while b"\r\n\r\n" not in data:
54
+ chunk = sock.recv(4096)
55
+ if not chunk:
56
+ break
57
+ data += chunk
58
+
59
+ if not data:
60
+ return {"status": 0, "headers": {}, "body": b""}
61
+
62
+ header_end = data.index(b"\r\n\r\n")
63
+ header_bytes = data[:header_end].decode("utf-8", errors="replace")
64
+ body = data[header_end + 4:]
65
+
66
+ # 解析 status + headers
67
+ lines = header_header = header_bytes.split("\r\n")
68
+ status_line = lines[0]
69
+ status = int(status_line.split()[1])
70
+ headers = {}
71
+ for line in lines[1:]:
72
+ if ":" in line:
73
+ k, v = line.split(":", 1)
74
+ headers[k.strip().lower()] = v.strip()
75
+
76
+ # chunked transfer-encoding
77
+ if headers.get("transfer-encoding") == "chunked":
78
+ decoded = b""
79
+ while body:
80
+ # chunk size 行
81
+ crlf = body.index(b"\r\n")
82
+ size_hex = body[:crlf].decode("ascii").split(";")[0].strip()
83
+ chunk_size = int(size_hex, 16)
84
+ if chunk_size == 0:
85
+ break
86
+ decoded += body[crlf + 2:crlf + 2 + chunk_size]
87
+ body = body[crlf + 2 + chunk_size + 2:] # skip chunk + \r\n
88
+ body = decoded
89
+ elif "content-length" in headers:
90
+ content_length = int(headers["content-length"])
91
+ while len(body) < content_length:
92
+ chunk = sock.recv(min(65536, content_length - len(body)))
93
+ if not chunk:
94
+ break
95
+ body += chunk
96
+ body = body[:content_length]
97
+
98
+ return {"status": status, "headers": headers, "body": body}
99
+
100
+
101
+ def _http_request(host: str, port: int, path: str,
102
+ method: str = "GET", body: str = "",
103
+ headers: dict | None = None, timeout: float = 10.0) -> dict:
104
+ """发送 HTTP/1.1 请求并读取完整响应。"""
105
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
106
+ sock.settimeout(timeout)
107
+ try:
108
+ sock.connect((host, port))
109
+
110
+ hdrs = {"Host": host, "Connection": "close"}
111
+ if headers:
112
+ hdrs.update(headers)
113
+ if body:
114
+ hdrs["Content-Length"] = str(len(body.encode()))
115
+
116
+ request_line = f"{method} {path} HTTP/1.1\r\n"
117
+ header_block = "".join(f"{k}: {v}\r\n" for k, v in hdrs.items())
118
+ raw = request_line + header_block + "\r\n"
119
+ if body:
120
+ raw += body
121
+
122
+ sock.sendall(raw.encode())
123
+ return _http_read_response(sock, timeout)
124
+ finally:
125
+ # 异常路径(超时/连接拒绝)也必须关闭 fd
126
+ sock.close()
127
+
128
+
129
+ # ─── WebSocket 握手(CDP 必须用 WS 双向通信)────────────────────────────────
130
+ # 但 CDP 也支持 HTTP 长轮询模式获取事件。
131
+ # 这里实现一个「最小 WS 客户端」:只发文本帧、收文本帧。
132
+
133
+ def _ws_handshake(sock: socket.socket, path: str, host: str) -> None:
134
+ """完成 WebSocket 握手。"""
135
+ key = os.urandom(16)
136
+ import base64
137
+ ws_key = base64.b64encode(key).decode()
138
+
139
+ request = (
140
+ f"GET {path} HTTP/1.1\r\n"
141
+ f"Host: {host}\r\n"
142
+ f"Upgrade: websocket\r\n"
143
+ f"Connection: Upgrade\r\n"
144
+ f"Sec-WebSocket-Key: {ws_key}\r\n"
145
+ f"Sec-WebSocket-Version: 13\r\n"
146
+ f"\r\n"
147
+ )
148
+ sock.sendall(request.encode())
149
+
150
+ # 读响应
151
+ data = b""
152
+ while b"\r\n\r\n" not in data:
153
+ chunk = sock.recv(4096)
154
+ if not chunk:
155
+ break
156
+ data += chunk
157
+
158
+
159
+ def _ws_send(sock: socket.socket, payload: str) -> None:
160
+ """发送一个 WebSocket 文本帧(masked)。"""
161
+ data = payload.encode("utf-8")
162
+ frame = bytearray()
163
+ frame.append(0x81) # FIN + text opcode
164
+
165
+ mask_key = os.urandom(4)
166
+ length = len(data)
167
+ if length < 126:
168
+ frame.append(0x80 | length) # mask bit set
169
+ elif length < 65536:
170
+ frame.append(0x80 | 126)
171
+ frame.extend(struct.pack(">H", length))
172
+ else:
173
+ frame.append(0x80 | 127)
174
+ frame.extend(struct.pack(">Q", length))
175
+
176
+ frame.extend(mask_key)
177
+ masked = bytearray(b ^ mask_key[i % 4] for i, b in enumerate(data))
178
+ frame.extend(masked)
179
+ sock.sendall(frame)
180
+
181
+
182
+ def _ws_recv(sock: socket.socket, timeout: float = 5.0) -> str | None:
183
+ """接收一个 WebSocket 帧,返回文本 payload。"""
184
+ sock.settimeout(timeout)
185
+ try:
186
+ header = sock.recv(2)
187
+ if len(header) < 2:
188
+ return None
189
+ opcode = header[0] & 0x0F
190
+ masked = (header[1] & 0x80) != 0
191
+ length = header[1] & 0x7F
192
+
193
+ if length == 126:
194
+ length = struct.unpack(">H", sock.recv(2))[0]
195
+ elif length == 127:
196
+ length = struct.unpack(">Q", sock.recv(8))[0]
197
+
198
+ if masked:
199
+ mask_key = sock.recv(4)
200
+
201
+ payload = b""
202
+ while len(payload) < length:
203
+ chunk = sock.recv(min(65536, length - len(payload)))
204
+ if not chunk:
205
+ break
206
+ payload += chunk
207
+
208
+ if masked:
209
+ payload = bytearray(b ^ mask_key[i % 4] for i, b in enumerate(payload))
210
+
211
+ if opcode == 0x1: # text
212
+ return payload.decode("utf-8", errors="replace")
213
+ return None
214
+ except socket.timeout:
215
+ return None
216
+
217
+
218
+ # ─── CDP 命令发送器(HTTP 模式,无 WS 事件订阅)─────────────────────────────
219
+
220
+ def _cdp_send_http(host: str, port: int, session_id: str,
221
+ method: str, params: dict | None = None, timeout: float = 15.0) -> dict:
222
+ """通过 HTTP 发送 CDP 命令(JSON-RPC over HTTP)。
223
+
224
+ Chrome 的 /json/protocol 是只读的,但每个 session 的 CDP 命令
225
+ 必须通过 WebSocket 发送。所以这里退化为:
226
+ 1. 用 HTTP GET /json 获取 ws endpoint
227
+ 2. 用最小 WS 客户端发命令
228
+ """
229
+ # 获取 ws path
230
+ resp = _http_request(host, port, f"/json/protocol", timeout=5)
231
+ # 不需要这个,直接建 WS 连接
232
+ return {}
233
+
234
+
235
+ # ─── 最小 CDP WebSocket 客户端 ───────────────────────────────────────────────
236
+
237
+ class _CDPSession:
238
+ """单个 target 的 CDP WebSocket 连接。"""
239
+
240
+ def __init__(self, host: str, port: int, ws_path: str, timeout: float = 15.0):
241
+ self.host = host
242
+ self.port = port
243
+ self.timeout = timeout
244
+ self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
245
+ self._sock.settimeout(timeout)
246
+ self._sock.connect((host, port))
247
+ _ws_handshake(self._sock, ws_path, host)
248
+ self._msg_id = 0
249
+ self._pending: dict[int, str] = {} # id -> 已收数据
250
+ # 事件帧分发:send() 等响应的路上经过的事件帧转交订阅者
251
+ # (真 networkidle 依赖此通道统计 Network in-flight;此前事件帧
252
+ # 直接丢弃,navigate 的 networkidle 只能退化成 readyState 轮询)
253
+ self._event_handlers: list = []
254
+
255
+ def on_event(self, cb) -> None:
256
+ """订阅 CDP 事件帧:cb(method, params)。异常只吞不抛(观察者不能反过来打断命令路)。"""
257
+ self._event_handlers.append(cb)
258
+
259
+ def _dispatch(self, msg: dict) -> None:
260
+ method = msg.get("method")
261
+ params = msg.get("params") or {}
262
+ for cb in self._event_handlers:
263
+ try:
264
+ cb(method, params)
265
+ except Exception:
266
+ pass
267
+
268
+ def pump(self, duration: float = 0.2) -> None:
269
+ """只读事件 duration 秒并分发(无挂起命令时的观察窗口)。
270
+
271
+ 期间出现的响应帧(无对应挂起命令)静默丢弃——所有 send() 都同步
272
+ 吃掉自己的响应,剩下的只可能是迟到者。
273
+ """
274
+ deadline = time.time() + duration
275
+ while True:
276
+ remaining = deadline - time.time()
277
+ if remaining <= 0:
278
+ break
279
+ text = _ws_recv(self._sock, timeout=min(0.1, remaining))
280
+ if text is None:
281
+ continue
282
+ try:
283
+ msg = json.loads(text)
284
+ except json.JSONDecodeError:
285
+ continue
286
+ if "id" in msg:
287
+ continue
288
+ self._dispatch(msg)
289
+
290
+ def send(self, method: str, params: dict | None = None) -> dict | None:
291
+ """发送 CDP 命令并等待对应 id 的响应。
292
+
293
+ Chrome 在 enable Runtime/Page 后会持续推送事件帧(executionContextCreated、
294
+ frameStartedNavigating 等),这些帧没有 id 字段。必须持续读取直到找到
295
+ 匹配 id 的响应帧,或超时。
296
+ """
297
+ self._msg_id += 1
298
+ msg_id = self._msg_id
299
+ cmd = json.dumps({"id": msg_id, "method": method, "params": params or {}})
300
+ _ws_send(self._sock, cmd)
301
+
302
+ # 持续读取帧,跳过事件帧,找到匹配 id 的响应
303
+ deadline = time.time() + self.timeout
304
+ while time.time() < deadline:
305
+ remaining = deadline - time.time()
306
+ if remaining <= 0:
307
+ break
308
+ text = _ws_recv(self._sock, timeout=min(1.0, remaining))
309
+ if text is None:
310
+ continue
311
+ try:
312
+ msg = json.loads(text)
313
+ except json.JSONDecodeError:
314
+ continue
315
+ # 事件帧(无 id)→ 分发给订阅者后继续等响应
316
+ if "id" not in msg:
317
+ self._dispatch(msg)
318
+ continue
319
+ if msg.get("id") == msg_id:
320
+ if "error" in msg:
321
+ return {"error": msg["error"]}
322
+ return msg.get("result", {})
323
+ return {"error": "timeout", "method": method}
324
+
325
+ def close(self) -> None:
326
+ try:
327
+ self._sock.close()
328
+ except Exception:
329
+ pass
330
+
331
+
332
+ # ─── Chrome 进程管理 ─────────────────────────────────────────────────────────
333
+
334
+ class _ChromeProcess:
335
+ """启动/管理 Chrome 进程(默认 headless;可带持久 profile / 可见窗口)。"""
336
+
337
+ def __init__(self, port: int = 0, chrome_path: str | None = None,
338
+ headless: bool = True, user_data_dir: str | None = None,
339
+ start_url: str = "about:blank"):
340
+ self.port = port or self._find_free_port()
341
+ self.chrome_path = chrome_path or self._find_chrome()
342
+ self.headless = headless
343
+ self._proc: subprocess.Popen | None = None
344
+ # 持久 profile(browser_auth 登录态车道)由调用方传入并负责生命周期;
345
+ # 临时 profile 仍是「每次冷启、启动前清残骸」的一次性语义。
346
+ self._persistent = user_data_dir is not None
347
+ # 跨平台:/tmp 在 Windows 不存在,统一用系统临时目录
348
+ self._user_data_dir = user_data_dir or os.path.join(
349
+ tempfile.gettempdir(), f"argo_chrome_{self.port}")
350
+ self._start_url = start_url
351
+
352
+ @staticmethod
353
+ def _find_free_port() -> int:
354
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
355
+ s.bind(("", 0))
356
+ port = s.getsockname()[1]
357
+ s.close()
358
+ return port
359
+
360
+ @staticmethod
361
+ def _find_chrome() -> str:
362
+ candidates = [
363
+ os.environ.get("CHROME_PATH", ""),
364
+ # macOS
365
+ "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
366
+ "/Applications/Chromium.app/Contents/MacOS/Chromium",
367
+ # Linux 常见可执行名(含 stable / chromium-browser / snap)
368
+ "/usr/bin/google-chrome",
369
+ "/usr/bin/google-chrome-stable",
370
+ "/usr/bin/chromium",
371
+ "/usr/bin/chromium-browser",
372
+ "/snap/bin/google-chrome",
373
+ "/snap/bin/chromium",
374
+ # Windows:Chrome / Edge 常见安装路径
375
+ os.path.expandvars(r"%ProgramFiles%\Google\Chrome\Application\chrome.exe"),
376
+ os.path.expandvars(r"%ProgramFiles(x86)%\Google\Chrome\Application\chrome.exe"),
377
+ os.path.expandvars(r"%LocalAppData%\Google\Chrome\Application\chrome.exe"),
378
+ os.path.expandvars(r"%ProgramFiles%\Microsoft\Edge\Application\msedge.exe"),
379
+ os.path.expandvars(r"%ProgramFiles(x86)%\Microsoft\Edge\Application\msedge.exe"),
380
+ ]
381
+ for c in candidates:
382
+ if c and os.path.isfile(c):
383
+ return c
384
+ # Linux 常经 PATH 而非绝对路径发现(如 /snap/bin/chromium 是 symlink、
385
+ # chromium-browser 只在 PATH 中):shutil.which 作为绝对路径后的保底,
386
+ # 保留 macOS/Windows 既有优先顺序,不改变原行为。
387
+ import shutil
388
+ for name in ("google-chrome", "google-chrome-stable",
389
+ "chromium", "chromium-browser"):
390
+ hit = shutil.which(name)
391
+ if hit:
392
+ return hit
393
+ raise FileNotFoundError("Chrome not found. Install Chrome/Edge or set CHROME_PATH / chrome_path.")
394
+
395
+ def start(self) -> None:
396
+ """启动 Chrome with remote debugging(headless 由 self.headless 决定)。"""
397
+ import shutil
398
+ # 清理旧 profile(避免 Hangouts 这类背景页)。持久 profile(登录态)
399
+ # 是要保命的数据,绝不 rmtree——那是 browser_auth 的 logout 才能做的事。
400
+ if not self._persistent and os.path.isdir(self._user_data_dir):
401
+ shutil.rmtree(self._user_data_dir, ignore_errors=True)
402
+
403
+ cmd = [self.chrome_path]
404
+ if self.headless:
405
+ cmd.append("--headless=new")
406
+ cmd += [
407
+ "--no-sandbox",
408
+ "--disable-gpu",
409
+ "--disable-dev-shm-usage",
410
+ "--disable-extensions",
411
+ "--disable-background-networking",
412
+ "--no-first-run",
413
+ "--no-default-browser-check",
414
+ "--disable-background-timer-throttling",
415
+ "--disable-backgrounding-occluded-windows",
416
+ "--disable-renderer-backgrounding",
417
+ f"--remote-debugging-port={self.port}",
418
+ f"--user-data-dir={self._user_data_dir}",
419
+ "--window-size=1280,800",
420
+ self._start_url,
421
+ ]
422
+ self._proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
423
+ try:
424
+ # 等待 CDP ready
425
+ for _ in range(30):
426
+ time.sleep(0.3)
427
+ try:
428
+ resp = _http_request("localhost", self.port, "/json/version", timeout=2)
429
+ if resp["status"] == 200 and resp["body"]:
430
+ return
431
+ except Exception:
432
+ pass
433
+ raise RuntimeError(f"Chrome CDP failed to start on port {self.port}")
434
+ except BaseException:
435
+ # CDP 启动失败:杀掉已拉起的孤儿 Chrome,避免进程泄漏
436
+ try:
437
+ self._proc.kill()
438
+ except Exception:
439
+ pass
440
+ self._proc = None
441
+ raise
442
+
443
+ def stop(self) -> None:
444
+ if self._proc:
445
+ try:
446
+ self._proc.terminate()
447
+ self._proc.wait(timeout=5)
448
+ except Exception:
449
+ try:
450
+ self._proc.kill()
451
+ except Exception:
452
+ pass
453
+ self._proc = None
454
+
455
+ def __enter__(self):
456
+ self.start()
457
+ return self
458
+
459
+ def __exit__(self, *args):
460
+ self.stop()
461
+
462
+
463
+ # ─── 真 networkidle 判据(纯逻辑,独立可测)─────────────────────────────────
464
+
465
+ class NetworkIdleTracker:
466
+ """消费 CDP Network/Page 事件,维护 in-flight 计数,回答「网络静默了吗」。
467
+
468
+ 口径:
469
+ - requestWillBeSent +1;**带 redirectResponse 时不加**——重定向是同一次
470
+ in-flight 的改道,原请求不会再来 loadingFinished,改道请求接着飞,
471
+ 照加必泄漏(计数只涨不跌,networkidle 永不达成)。
472
+ - loadingFinished / loadingFailed 各 -1;下限钳 0(attach 晚了会错过
473
+ requestWillBeSent,只有 finished 事件到达时不许把计数打成负数)。
474
+ - Page.loadEventFired 置 load 位;idle = 无在途请求 且(load 已发 或
475
+ readyState 已 complete)。
476
+ """
477
+
478
+ def __init__(self):
479
+ self.inflight = 0
480
+ self.load_fired = False
481
+
482
+ def feed(self, method: str | None, params: dict) -> None:
483
+ if method == "Network.requestWillBeSent":
484
+ if not params.get("redirectResponse"):
485
+ self.inflight += 1
486
+ elif method in ("Network.loadingFinished", "Network.loadingFailed"):
487
+ self.inflight = max(0, self.inflight - 1)
488
+ elif method == "Page.loadEventFired":
489
+ self.load_fired = True
490
+
491
+ def idle(self, ready_state: str | None) -> bool:
492
+ if self.inflight > 0:
493
+ return False
494
+ if self.load_fired:
495
+ return True
496
+ return ready_state == "complete"
497
+
498
+
499
+ # ─── 主 API ──────────────────────────────────────────────────────────────────
500
+
501
+ class ChromeCDP:
502
+ """高级 CDP 封装:页面交互 + 内容提取。
503
+
504
+ 实现 Hound actions.py 的能力:
505
+ - navigate(url)
506
+ - click(selector)
507
+ - fill(selector, text)
508
+ - press(key)
509
+ - scroll(times)
510
+ - wait(ms)
511
+ - wait_selector(selector)
512
+ - evaluate(js)
513
+ - get_html()
514
+ - get_text()
515
+ - screenshot(path)
516
+ """
517
+
518
+ def __init__(self, port: int = 0, chrome_path: str | None = None,
519
+ auto_start: bool = True, user_data_dir: str | None = None,
520
+ headless: bool = True):
521
+ self._chrome = _ChromeProcess(port=port, chrome_path=chrome_path,
522
+ headless=headless,
523
+ user_data_dir=user_data_dir)
524
+ self._session: _CDPSession | None = None
525
+ self._target_id: str | None = None
526
+ self._net: NetworkIdleTracker | None = None
527
+ if auto_start:
528
+ try:
529
+ self.start()
530
+ except BaseException:
531
+ # 启动失败(如 target 获取/WS 建连异常):__exit__ 不会执行,
532
+ # 必须在此清理已拉起的 Chrome,避免进程泄漏
533
+ try:
534
+ self._chrome.stop()
535
+ except Exception:
536
+ pass
537
+ raise
538
+
539
+ def start(self) -> None:
540
+ """启动 Chrome 并建立 CDP 会话。"""
541
+ self._chrome.start()
542
+ # 获取 page target
543
+ resp = _http_request("localhost", self._chrome.port, "/json", timeout=5)
544
+ targets = json.loads(resp["body"])
545
+ page_targets = [t for t in targets if t.get("type") == "page"]
546
+ if not page_targets:
547
+ raise RuntimeError("No page target found")
548
+ self._target_id = page_targets[0]["id"]
549
+ ws_url = page_targets[0]["webSocketDebuggerUrl"]
550
+ # 格式:ws://hostname:PORT/path → 提取 /path
551
+ # 找到 "ws://" 后的第三个 "/" 即为 path 开始
552
+ scheme_end = ws_url.index("://") + 3
553
+ host_end = ws_url.index("/", scheme_end)
554
+ ws_path = ws_url[host_end:]
555
+ self._session = _CDPSession("localhost", self._chrome.port, ws_path)
556
+ # 启用 Runtime + Page + Network(Network 为真 networkidle 供事件)
557
+ self._session.send("Page.enable")
558
+ self._session.send("Runtime.enable")
559
+ try:
560
+ self._session.send("Network.enable")
561
+ except Exception:
562
+ pass # 老内核不认 Network domain:等待策略自动退化,不致命
563
+ self._net = NetworkIdleTracker()
564
+ self._session.on_event(self._on_event)
565
+
566
+ def _on_event(self, method, params) -> None:
567
+ if self._net is not None:
568
+ self._net.feed(method, params)
569
+
570
+ def stop(self) -> None:
571
+ if self._session:
572
+ self._session.close()
573
+ self._session = None
574
+ self._chrome.stop()
575
+
576
+ def __enter__(self):
577
+ return self
578
+
579
+ def __exit__(self, *args):
580
+ self.stop()
581
+
582
+ # ── 导航 ──
583
+
584
+ def navigate(self, url: str, wait_until: str = "networkidle",
585
+ timeout: float = 10.0) -> None:
586
+ """导航到 URL,可选等待加载状态。
587
+
588
+ networkidle:CDP Network 事件计数的真等待(在途请求清零 + load 完成再
589
+ 静默 0.5s);超时按 False 放行——wait 是观察条件不是闸门,SPA 长轮询
590
+ 永不 idle 时不得卡死整条任务(可降级纪律)。
591
+ """
592
+ self._net = NetworkIdleTracker() # 新导航新口径:上一页的计数不带入
593
+ self._session.send("Page.navigate", {"url": url})
594
+ if wait_until == "networkidle":
595
+ self._wait_network_idle(timeout=timeout)
596
+ elif wait_until == "load":
597
+ self.wait_for_event("Page.loadEventFired", timeout=timeout)
598
+
599
+ def _wait_network_idle(self, timeout: float = 10.0, quiet: float = 0.5) -> bool:
600
+ """真 networkidle:事件计数 + readyState 兜底,静默 quiet 秒算稳。
601
+
602
+ 返回是否达成(超时 False 放行)。无事件通道(Network.enable 失败的
603
+ 老内核)时退化为旧的 readyState 轮询语义。
604
+ """
605
+ if self._net is None or self._session is None:
606
+ time.sleep(0.5)
607
+ return False
608
+ deadline = time.time() + timeout
609
+ quiet_since: float | None = None
610
+ last_probe = 0.0
611
+ ready: str | None = None
612
+ while time.time() < deadline:
613
+ self._session.pump(0.1)
614
+ now = time.time()
615
+ if now - last_probe >= 0.5:
616
+ last_probe = now
617
+ try:
618
+ ready = self.evaluate("document.readyState")
619
+ except Exception:
620
+ ready = None
621
+ if self._net.idle(ready):
622
+ if quiet_since is None:
623
+ quiet_since = now
624
+ elif now - quiet_since >= quiet:
625
+ return True
626
+ else:
627
+ quiet_since = None
628
+ return False
629
+
630
+ def wait_for_event(self, event_method: str, timeout: float = 10.0) -> bool:
631
+ """等待特定 CDP 事件。"""
632
+ deadline = time.time() + timeout
633
+ while time.time() < deadline:
634
+ text = _ws_recv(self._session._sock, timeout=min(1.0, deadline - time.time()))
635
+ if text:
636
+ try:
637
+ msg = json.loads(text)
638
+ if msg.get("method") == event_method:
639
+ return True
640
+ except json.JSONDecodeError:
641
+ pass
642
+ return False
643
+
644
+ # ── 页面交互 ──
645
+
646
+ def click(self, selector: str) -> bool:
647
+ """点击匹配 CSS 选择器的元素。"""
648
+ js = f"""
649
+ (() => {{
650
+ const el = document.querySelector('{selector}');
651
+ if (!el) return false;
652
+ el.click();
653
+ return true;
654
+ }})()
655
+ """
656
+ r = self.evaluate(js)
657
+ return r is True
658
+
659
+ def fill(self, selector: str, text: str) -> bool:
660
+ """填写 input/textarea 的值并触发 input 事件。"""
661
+ escaped = text.replace("\\", "\\\\").replace("'", "\\'").replace("\n", "\\n")
662
+ js = f"""
663
+ (() => {{
664
+ const el = document.querySelector('{selector}');
665
+ if (!el) return false;
666
+ el.value = '{escaped}';
667
+ el.dispatchEvent(new Event('input', {{bubbles: true}}));
668
+ el.dispatchEvent(new Event('change', {{bubbles: true}}));
669
+ return true;
670
+ }})()
671
+ """
672
+ r = self.evaluate(js)
673
+ return r is True
674
+
675
+ def press(self, key: str) -> None:
676
+ """模拟按键(Enter、Tab、Escape 等)。"""
677
+ key_map = {"Enter": "Enter", "Tab": "Tab", "Escape": "Escape",
678
+ "Backspace": "Backspace", "ArrowDown": "ArrowDown"}
679
+ kp = key_map.get(key, key)
680
+ self._session.send("Input.dispatchKeyEvent", {
681
+ "type": "keyDown",
682
+ "key": kp,
683
+ "code": kp,
684
+ "windowsVirtualKeyCode": 13 if key == "Enter" else 0,
685
+ })
686
+ self._session.send("Input.dispatchKeyEvent", {
687
+ "type": "keyUp", "key": kp, "code": kp,
688
+ })
689
+
690
+ def scroll(self, times: int = 3) -> None:
691
+ """向下滚动页面。"""
692
+ for _ in range(times):
693
+ self.evaluate("window.scrollBy(0, window.innerHeight)")
694
+ time.sleep(0.3)
695
+
696
+ def wait(self, ms: int) -> None:
697
+ """等待指定毫秒。"""
698
+ time.sleep(ms / 1000.0)
699
+
700
+ def wait_selector(self, selector: str, timeout: float = 10.0) -> bool:
701
+ """等待选择器匹配的元素出现。"""
702
+ deadline = time.time() + timeout
703
+ js = f"document.querySelector('{selector}') !== null"
704
+ while time.time() < deadline:
705
+ r = self.evaluate(js)
706
+ if r is True:
707
+ return True
708
+ time.sleep(0.3)
709
+ return False
710
+
711
+ def evaluate(self, expression: str) -> Any:
712
+ """在页面上下文执行 JS 表达式并返回结果。"""
713
+ r = self._session.send("Runtime.evaluate", {
714
+ "expression": expression,
715
+ "returnByValue": True,
716
+ })
717
+ if r and "result" in r:
718
+ val = r["result"].get("value")
719
+ return val
720
+ return None
721
+
722
+ def fetch_json(self, url: str, method: str = "GET",
723
+ headers: dict | None = None, body: str | None = None) -> dict | None:
724
+ """页面内带凭证 fetch(A2「浏览器代发请求」)。
725
+
726
+ cookie 由浏览器按页面域自动附加(credentials: include),argo 只收
727
+ 响应体——cookie 值从头到尾不经过 argo,更不进模型上下文。
728
+ 适用同源接口(或带 CORS 的接口);返回 {"status": int, "body": str},
729
+ 页面/网络/超时失败返回 None。只读语义:本方法不给写操作背书。
730
+ """
731
+ try:
732
+ expr = (
733
+ "(async () => {"
734
+ f"const r = await fetch({json.dumps(url)},"
735
+ f"{{method:{json.dumps(method)},"
736
+ f"credentials:'include',"
737
+ f"headers:{json.dumps(headers or {'Accept': 'application/json'})},"
738
+ + (f"body:{json.dumps(body)}," if body is not None else "")
739
+ + "});"
740
+ "const t = await r.text();"
741
+ "return {status:r.status, body:t};"
742
+ "})()"
743
+ )
744
+ r = self._session.send("Runtime.evaluate", {
745
+ "expression": expr,
746
+ "awaitPromise": True,
747
+ "returnByValue": True,
748
+ })
749
+ except Exception:
750
+ return None
751
+ if not r or "error" in r or r.get("exceptionDetails"):
752
+ return None
753
+ val = (r.get("result") or {}).get("value")
754
+ if isinstance(val, dict) and "status" in val and "body" in val:
755
+ return val
756
+ return None
757
+
758
+ # ── 内容提取 ──
759
+
760
+ def get_html(self) -> str:
761
+ """获取当前页面完整 HTML。"""
762
+ r = self.evaluate("document.documentElement.outerHTML")
763
+ return r or ""
764
+
765
+ def get_text(self) -> str:
766
+ """获取页面可见文本。"""
767
+ r = " " * 0
768
+ r = self.evaluate("document.body ? document.body.innerText : ''")
769
+ return r or ""
770
+
771
+ def get_title(self) -> str:
772
+ """获取页面标题。"""
773
+ r = self.evaluate("document.title")
774
+ return r or ""
775
+
776
+ def get_url(self) -> str:
777
+ """获取当前 URL。"""
778
+ r = self.evaluate("window.location.href")
779
+ return r or ""
780
+
781
+ # ── 截图 ──
782
+
783
+ def screenshot(self, path: str | None = None, full_page: bool = False) -> str:
784
+ """截图并保存为 PNG,返回文件路径。"""
785
+ params: dict[str, Any] = {"format": "png"}
786
+ if full_page:
787
+ # 获取页面完整高度
788
+ height = self.evaluate("document.documentElement.scrollHeight") or 800
789
+ self._session.send("Emulation.setDeviceMetricsOverride", {
790
+ "width": 1280, "height": int(height), "deviceScaleFactor": 1, "mobile": False
791
+ })
792
+ params["clip"] = {"x": 0, "y": 0, "width": 1280, "height": int(height), "scale": 1}
793
+
794
+ r = self._session.send("Page.captureScreenshot", params)
795
+ if r and "data" in r:
796
+ import base64
797
+ data = base64.b64decode(r["data"])
798
+ path = path or os.path.join(
799
+ tempfile.gettempdir(), f"argo_screenshot_{int(time.time())}.png")
800
+ with open(path, "wb") as f:
801
+ f.write(data)
802
+ if full_page:
803
+ self._session.send("Emulation.clearDeviceMetricsOverride")
804
+ return path
805
+ return ""
806
+
807
+ # ── 高级:执行 actions 序列 ──
808
+
809
+ def execute_actions(self, actions: list[dict]) -> dict:
810
+ """执行 Hound 风格的 actions 数组。
811
+
812
+ 支持的 action 类型:
813
+ - {"click": "selector"}
814
+ - {"fill": {"selector": "...", "text": "..."}}
815
+ - {"press": "Enter"}
816
+ - {"scroll": 3}
817
+ - {"wait": 1000} # 毫秒
818
+ - {"wait_selector": ".content"}
819
+ - {"evaluate": "document.title"}
820
+
821
+ 返回 {"success": bool, "results": [...], "error": str|None}
822
+ """
823
+ results = []
824
+ for action in actions:
825
+ try:
826
+ if "click" in action:
827
+ ok = self.click(action["click"])
828
+ results.append({"click": action["click"], "ok": ok})
829
+ elif "fill" in action:
830
+ params = action["fill"]
831
+ ok = self.fill(params["selector"], params["text"])
832
+ results.append({"fill": params["selector"], "ok": ok})
833
+ elif "press" in action:
834
+ self.press(action["press"])
835
+ results.append({"press": action["press"], "ok": True})
836
+ elif "scroll" in action:
837
+ self.scroll(action["scroll"])
838
+ results.append({"scroll": action["scroll"], "ok": True})
839
+ elif "wait" in action:
840
+ self.wait(action["wait"])
841
+ results.append({"wait": action["wait"], "ok": True})
842
+ elif "wait_selector" in action:
843
+ ok = self.wait_selector(action["wait_selector"])
844
+ results.append({"wait_selector": action["wait_selector"], "ok": ok})
845
+ elif "evaluate" in action:
846
+ val = self.evaluate(action["evaluate"])
847
+ results.append({"evaluate": action["evaluate"], "result": val})
848
+ else:
849
+ results.append({"action": action, "ok": False, "error": "unknown action"})
850
+ except Exception as e:
851
+ results.append({"action": action, "ok": False, "error": str(e)[:100]})
852
+ return {"success": False, "results": results, "error": str(e)[:100]}
853
+
854
+ all_ok = all(r.get("ok", True) for r in results)
855
+ return {"success": all_ok, "results": results}
856
+
857
+
858
+ # ─── CLI 测试 ────────────────────────────────────────────────────────────────
859
+
860
+ if __name__ == "__main__":
861
+ import sys
862
+ url = sys.argv[1] if len(sys.argv) > 1 else "https://example.com"
863
+
864
+ with ChromeCDP() as cdp:
865
+ print(f"Navigating to {url}...")
866
+ cdp.navigate(url)
867
+ print(f"Title: {cdp.get_title()}")
868
+ print(f"URL: {cdp.get_url()}")
869
+ text = cdp.get_text()
870
+ print(f"Text length: {len(text)}")
871
+ print(f"First 200 chars: {text[:200]}")