@jackwener/opencli 1.3.2 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (508) hide show
  1. package/.github/pull_request_template.md +3 -1
  2. package/.github/workflows/build-extension.yml +7 -1
  3. package/.github/workflows/ci.yml +29 -3
  4. package/.github/workflows/docs.yml +1 -1
  5. package/.github/workflows/e2e-headed.yml +20 -0
  6. package/.github/workflows/release.yml +1 -1
  7. package/.github/workflows/security.yml +0 -3
  8. package/CHANGELOG.md +55 -0
  9. package/CONTRIBUTING.md +6 -3
  10. package/README.md +37 -10
  11. package/README.zh-CN.md +37 -10
  12. package/SKILL.md +7 -2
  13. package/TESTING.md +1 -0
  14. package/chatwise-opencli.ps1 +82 -0
  15. package/dist/analysis.d.ts +38 -0
  16. package/dist/analysis.js +166 -0
  17. package/dist/browser/cdp.d.ts +0 -4
  18. package/dist/browser/cdp.js +59 -38
  19. package/dist/browser/cdp.test.d.ts +1 -0
  20. package/dist/browser/cdp.test.js +52 -0
  21. package/dist/browser/daemon-client.js +2 -1
  22. package/dist/browser/discover.js +2 -1
  23. package/dist/browser/dom-snapshot.d.ts +2 -2
  24. package/dist/browser/dom-snapshot.js +54 -1
  25. package/dist/browser/dom-snapshot.test.js +36 -0
  26. package/dist/browser/errors.js +2 -1
  27. package/dist/browser/index.d.ts +3 -2
  28. package/dist/browser/index.js +2 -1
  29. package/dist/browser/mcp.d.ts +0 -2
  30. package/dist/browser/mcp.js +2 -3
  31. package/dist/browser/page.d.ts +4 -3
  32. package/dist/browser/page.js +44 -35
  33. package/dist/browser/stealth.d.ts +16 -0
  34. package/dist/browser/stealth.js +155 -0
  35. package/dist/browser.test.js +47 -1
  36. package/dist/build-manifest.js +15 -9
  37. package/dist/build-manifest.test.js +12 -0
  38. package/dist/cascade.js +4 -2
  39. package/dist/cli-manifest.json +639 -258
  40. package/dist/cli.js +57 -29
  41. package/dist/clis/_shared/desktop-commands.d.ts +22 -0
  42. package/dist/clis/_shared/desktop-commands.js +108 -0
  43. package/dist/clis/antigravity/serve.js +5 -2
  44. package/dist/clis/arxiv/search.js +1 -1
  45. package/dist/clis/bilibili/dynamic.test.d.ts +1 -0
  46. package/dist/clis/bilibili/dynamic.test.js +68 -0
  47. package/dist/clis/bilibili/favorite.js +4 -2
  48. package/dist/clis/bilibili/following.js +3 -2
  49. package/dist/clis/bilibili/subtitle.js +8 -7
  50. package/dist/clis/bilibili/utils.js +2 -2
  51. package/dist/clis/boss/batchgreet.js +1 -1
  52. package/dist/clis/boss/chatlist.js +1 -1
  53. package/dist/clis/boss/chatmsg.js +1 -1
  54. package/dist/clis/boss/detail.js +1 -1
  55. package/dist/clis/boss/exchange.js +1 -1
  56. package/dist/clis/boss/greet.js +1 -1
  57. package/dist/clis/boss/invite.js +1 -1
  58. package/dist/clis/boss/joblist.js +1 -1
  59. package/dist/clis/boss/mark.js +4 -3
  60. package/dist/clis/boss/recommend.js +1 -1
  61. package/dist/clis/boss/resume.js +1 -1
  62. package/dist/clis/boss/search.js +1 -1
  63. package/dist/clis/boss/send.js +5 -4
  64. package/dist/clis/boss/stats.js +1 -1
  65. package/dist/clis/chatgpt/ask.js +4 -0
  66. package/dist/clis/chatgpt/new.js +5 -1
  67. package/dist/clis/chatgpt/read.js +5 -1
  68. package/dist/clis/chatgpt/send.js +2 -1
  69. package/dist/clis/chatgpt/status.js +5 -1
  70. package/dist/clis/chatwise/ask.js +8 -2
  71. package/dist/clis/chatwise/export.js +2 -0
  72. package/dist/clis/chatwise/history.js +2 -0
  73. package/dist/clis/chatwise/model.js +8 -3
  74. package/dist/clis/chatwise/new.js +3 -18
  75. package/dist/clis/chatwise/read.js +2 -0
  76. package/dist/clis/chatwise/screenshot.js +3 -27
  77. package/dist/clis/chatwise/send.js +8 -2
  78. package/dist/clis/chatwise/shared.d.ts +2 -0
  79. package/dist/clis/chatwise/shared.js +6 -0
  80. package/dist/clis/chatwise/status.js +3 -22
  81. package/dist/clis/codex/ask.js +6 -2
  82. package/dist/clis/codex/dump.js +2 -25
  83. package/dist/clis/codex/new.js +2 -25
  84. package/dist/clis/codex/screenshot.js +2 -27
  85. package/dist/clis/codex/send.js +6 -4
  86. package/dist/clis/codex/status.js +2 -22
  87. package/dist/clis/cursor/ask.js +2 -1
  88. package/dist/clis/cursor/composer.js +2 -1
  89. package/dist/clis/cursor/dump.js +2 -25
  90. package/dist/clis/cursor/new.js +2 -18
  91. package/dist/clis/cursor/read.js +2 -1
  92. package/dist/clis/cursor/screenshot.js +1 -30
  93. package/dist/clis/cursor/send.js +2 -1
  94. package/dist/clis/cursor/status.js +2 -21
  95. package/dist/clis/dictionary/examples.yaml +25 -0
  96. package/dist/clis/dictionary/search.yaml +27 -0
  97. package/dist/clis/dictionary/synonyms.yaml +25 -0
  98. package/dist/clis/douban/book-hot.js +1 -1
  99. package/dist/clis/douban/movie-hot.js +1 -1
  100. package/dist/clis/douban/search.js +1 -1
  101. package/dist/clis/douban/utils.d.ts +4 -1
  102. package/dist/clis/douban/utils.js +156 -1
  103. package/dist/clis/doubao/ask.js +1 -1
  104. package/dist/clis/doubao/new.js +1 -1
  105. package/dist/clis/doubao/read.js +1 -1
  106. package/dist/clis/doubao/send.js +1 -1
  107. package/dist/clis/doubao/status.js +1 -1
  108. package/dist/clis/doubao-app/ask.js +1 -1
  109. package/dist/clis/doubao-app/new.js +1 -1
  110. package/dist/clis/doubao-app/read.js +1 -1
  111. package/dist/clis/doubao-app/send.js +1 -1
  112. package/dist/clis/grok/ask.d.ts +4 -0
  113. package/dist/clis/grok/ask.js +28 -10
  114. package/dist/clis/grok/ask.test.js +18 -0
  115. package/dist/clis/jd/item.d.ts +1 -0
  116. package/dist/clis/jd/item.js +96 -0
  117. package/dist/clis/jd/item.test.d.ts +1 -0
  118. package/dist/clis/jd/item.test.js +28 -0
  119. package/dist/clis/jike/feed.js +1 -1
  120. package/dist/clis/jike/search.js +1 -1
  121. package/dist/clis/linkedin/search.js +5 -4
  122. package/dist/clis/linkedin/timeline.d.ts +21 -0
  123. package/dist/clis/linkedin/timeline.js +503 -0
  124. package/dist/clis/linkedin/timeline.test.d.ts +1 -0
  125. package/dist/clis/linkedin/timeline.test.js +81 -0
  126. package/dist/clis/medium/feed.js +1 -1
  127. package/dist/clis/medium/search.js +1 -1
  128. package/dist/clis/medium/user.js +1 -1
  129. package/dist/clis/medium/{shared.js → utils.js} +2 -1
  130. package/dist/clis/pixiv/detail.yaml +49 -0
  131. package/dist/clis/pixiv/download.d.ts +7 -0
  132. package/dist/clis/pixiv/download.js +78 -0
  133. package/dist/clis/pixiv/download.test.d.ts +1 -0
  134. package/dist/clis/pixiv/download.test.js +87 -0
  135. package/dist/clis/pixiv/illusts.d.ts +8 -0
  136. package/dist/clis/pixiv/illusts.js +65 -0
  137. package/dist/clis/pixiv/illusts.test.d.ts +1 -0
  138. package/dist/clis/pixiv/illusts.test.js +99 -0
  139. package/dist/clis/pixiv/ranking.yaml +53 -0
  140. package/dist/clis/pixiv/search.d.ts +6 -0
  141. package/dist/clis/pixiv/search.js +43 -0
  142. package/dist/clis/pixiv/search.test.d.ts +1 -0
  143. package/dist/clis/pixiv/search.test.js +83 -0
  144. package/dist/clis/pixiv/test-utils.d.ts +12 -0
  145. package/dist/clis/pixiv/test-utils.js +23 -0
  146. package/dist/clis/pixiv/user.yaml +46 -0
  147. package/dist/clis/pixiv/utils.d.ts +27 -0
  148. package/dist/clis/pixiv/utils.js +49 -0
  149. package/dist/clis/reddit/comment.js +2 -1
  150. package/dist/clis/reddit/read.js +4 -3
  151. package/dist/clis/reddit/read.test.d.ts +1 -0
  152. package/dist/clis/reddit/read.test.js +28 -0
  153. package/dist/clis/reddit/save.js +2 -1
  154. package/dist/clis/reddit/saved.js +7 -3
  155. package/dist/clis/reddit/subscribe.js +2 -1
  156. package/dist/clis/reddit/upvote.js +2 -1
  157. package/dist/clis/reddit/upvoted.js +7 -3
  158. package/dist/clis/sinablog/article.js +1 -1
  159. package/dist/clis/sinablog/hot.js +1 -1
  160. package/dist/clis/sinablog/user.js +1 -1
  161. package/dist/clis/substack/feed.js +1 -1
  162. package/dist/clis/substack/publication.js +1 -1
  163. package/dist/clis/substack/search.js +3 -2
  164. package/dist/clis/substack/{shared.js → utils.js} +3 -2
  165. package/dist/clis/tiktok/search.yaml +2 -1
  166. package/dist/clis/twitter/accept.js +2 -1
  167. package/dist/clis/twitter/article.js +4 -1
  168. package/dist/clis/twitter/block.js +2 -1
  169. package/dist/clis/twitter/bookmark.js +2 -1
  170. package/dist/clis/twitter/bookmarks.js +3 -2
  171. package/dist/clis/twitter/delete.js +2 -1
  172. package/dist/clis/twitter/follow.js +2 -1
  173. package/dist/clis/twitter/followers.js +3 -2
  174. package/dist/clis/twitter/following.js +3 -2
  175. package/dist/clis/twitter/hide-reply.js +2 -1
  176. package/dist/clis/twitter/like.js +2 -1
  177. package/dist/clis/twitter/notifications.js +2 -1
  178. package/dist/clis/twitter/post.js +2 -1
  179. package/dist/clis/twitter/profile.js +5 -2
  180. package/dist/clis/twitter/reply-dm.js +2 -1
  181. package/dist/clis/twitter/reply.js +2 -1
  182. package/dist/clis/twitter/search.js +30 -13
  183. package/dist/clis/twitter/search.test.d.ts +1 -0
  184. package/dist/clis/twitter/search.test.js +104 -0
  185. package/dist/clis/twitter/thread.js +2 -2
  186. package/dist/clis/twitter/timeline.js +3 -2
  187. package/dist/clis/twitter/trending.js +3 -2
  188. package/dist/clis/twitter/unblock.js +2 -1
  189. package/dist/clis/twitter/unbookmark.js +2 -1
  190. package/dist/clis/twitter/unfollow.js +2 -1
  191. package/dist/clis/v2ex/daily.js +3 -2
  192. package/dist/clis/v2ex/me.js +3 -2
  193. package/dist/clis/v2ex/notifications.js +4 -4
  194. package/dist/clis/web/read.d.ts +16 -0
  195. package/dist/clis/web/read.js +202 -0
  196. package/dist/clis/xueqiu/danjuan-utils.d.ts +55 -0
  197. package/dist/clis/xueqiu/danjuan-utils.js +126 -0
  198. package/dist/clis/xueqiu/danjuan-utils.test.d.ts +1 -0
  199. package/dist/clis/xueqiu/danjuan-utils.test.js +41 -0
  200. package/dist/clis/xueqiu/fund-holdings.d.ts +1 -0
  201. package/dist/clis/xueqiu/fund-holdings.js +28 -0
  202. package/dist/clis/xueqiu/fund-snapshot.d.ts +1 -0
  203. package/dist/clis/xueqiu/fund-snapshot.js +25 -0
  204. package/dist/clis/youtube/transcript.js +5 -4
  205. package/dist/clis/youtube/video.js +3 -2
  206. package/dist/constants.d.ts +2 -0
  207. package/dist/constants.js +2 -0
  208. package/dist/daemon.js +9 -4
  209. package/dist/discovery.js +11 -10
  210. package/dist/doctor.js +4 -2
  211. package/dist/download/index.d.ts +4 -12
  212. package/dist/download/index.js +33 -12
  213. package/dist/download/index.test.js +79 -2
  214. package/dist/download/media-download.js +4 -2
  215. package/dist/engine.test.js +76 -4
  216. package/dist/execution.d.ts +1 -9
  217. package/dist/execution.js +56 -46
  218. package/dist/explore.js +12 -111
  219. package/dist/external-clis.yaml +0 -8
  220. package/dist/external.js +7 -5
  221. package/dist/external.test.js +4 -0
  222. package/dist/generate.d.ts +0 -9
  223. package/dist/generate.js +4 -20
  224. package/dist/hooks.d.ts +46 -0
  225. package/dist/hooks.js +56 -0
  226. package/dist/hooks.test.d.ts +4 -0
  227. package/dist/hooks.test.js +92 -0
  228. package/dist/interceptor.js +70 -23
  229. package/dist/main.js +2 -0
  230. package/dist/output.js +12 -6
  231. package/dist/pipeline/executor.js +1 -1
  232. package/dist/pipeline/steps/browser.js +1 -3
  233. package/dist/pipeline/steps/download.js +42 -26
  234. package/dist/pipeline/steps/download.test.d.ts +1 -0
  235. package/dist/pipeline/steps/download.test.js +101 -0
  236. package/dist/pipeline/steps/fetch.js +40 -22
  237. package/dist/pipeline/steps/fetch.test.d.ts +1 -0
  238. package/dist/pipeline/steps/fetch.test.js +123 -0
  239. package/dist/pipeline/steps/transform.js +2 -6
  240. package/dist/pipeline/template.js +66 -52
  241. package/dist/pipeline/template.test.js +28 -0
  242. package/dist/pipeline/transform.test.js +18 -0
  243. package/dist/plugin.d.ts +40 -1
  244. package/dist/plugin.js +214 -17
  245. package/dist/plugin.test.d.ts +1 -1
  246. package/dist/plugin.test.js +219 -3
  247. package/dist/record.js +6 -98
  248. package/dist/registry-api.d.ts +2 -0
  249. package/dist/registry-api.js +1 -0
  250. package/dist/registry.d.ts +5 -2
  251. package/dist/registry.js +1 -2
  252. package/dist/runtime.d.ts +0 -1
  253. package/dist/runtime.js +14 -4
  254. package/dist/snapshotFormatter.d.ts +7 -14
  255. package/dist/snapshotFormatter.js +38 -78
  256. package/dist/utils.d.ts +9 -0
  257. package/dist/utils.js +29 -0
  258. package/dist/validate.js +3 -5
  259. package/dist/yaml-schema.d.ts +26 -0
  260. package/dist/yaml-schema.js +5 -0
  261. package/docs/.vitepress/config.mts +3 -0
  262. package/docs/adapters/browser/dictionary.md +27 -0
  263. package/docs/adapters/browser/douban.md +18 -8
  264. package/docs/adapters/browser/jd.md +27 -0
  265. package/docs/adapters/browser/linkedin.md +6 -0
  266. package/docs/adapters/browser/pixiv.md +92 -0
  267. package/docs/adapters/browser/web.md +30 -0
  268. package/docs/adapters/browser/wikipedia.md +0 -9
  269. package/docs/adapters/browser/xueqiu.md +27 -9
  270. package/docs/adapters/desktop/antigravity.md +0 -3
  271. package/docs/adapters/index.md +11 -9
  272. package/docs/comparison.md +125 -0
  273. package/docs/developer/contributing.md +21 -2
  274. package/docs/developer/testing.md +14 -8
  275. package/docs/developer/ts-adapter.md +18 -0
  276. package/docs/developer/yaml-adapter.md +16 -0
  277. package/docs/guide/plugins.md +10 -0
  278. package/docs/zh/guide/plugins.md +10 -0
  279. package/extension/dist/background.js +519 -444
  280. package/extension/manifest.json +1 -1
  281. package/extension/package.json +1 -1
  282. package/extension/src/background.test.ts +46 -1
  283. package/extension/src/background.ts +108 -33
  284. package/extension/src/cdp.ts +9 -9
  285. package/package.json +3 -2
  286. package/scripts/check-doc-coverage.sh +2 -0
  287. package/src/analysis.ts +170 -0
  288. package/src/browser/cdp.test.ts +66 -0
  289. package/src/browser/cdp.ts +64 -41
  290. package/src/browser/daemon-client.ts +4 -3
  291. package/src/browser/discover.ts +2 -1
  292. package/src/browser/dom-snapshot.test.ts +42 -0
  293. package/src/browser/dom-snapshot.ts +56 -3
  294. package/src/browser/errors.ts +2 -1
  295. package/src/browser/index.ts +3 -2
  296. package/src/browser/mcp.ts +2 -4
  297. package/src/browser/page.ts +43 -35
  298. package/src/browser/stealth.ts +156 -0
  299. package/src/browser.test.ts +51 -1
  300. package/src/build-manifest.test.ts +14 -0
  301. package/src/build-manifest.ts +13 -32
  302. package/src/cascade.ts +5 -3
  303. package/src/cli.ts +66 -34
  304. package/src/clis/_shared/desktop-commands.ts +121 -0
  305. package/src/clis/antigravity/serve.ts +6 -3
  306. package/src/clis/arxiv/search.ts +1 -1
  307. package/src/clis/bilibili/dynamic.test.ts +79 -0
  308. package/src/clis/bilibili/favorite.ts +5 -2
  309. package/src/clis/bilibili/following.ts +3 -2
  310. package/src/clis/bilibili/subtitle.ts +8 -7
  311. package/src/clis/bilibili/utils.ts +2 -2
  312. package/src/clis/boss/batchgreet.ts +1 -1
  313. package/src/clis/boss/chatlist.ts +1 -1
  314. package/src/clis/boss/chatmsg.ts +1 -1
  315. package/src/clis/boss/detail.ts +1 -1
  316. package/src/clis/boss/exchange.ts +1 -1
  317. package/src/clis/boss/greet.ts +1 -1
  318. package/src/clis/boss/invite.ts +1 -1
  319. package/src/clis/boss/joblist.ts +1 -1
  320. package/src/clis/boss/mark.ts +4 -3
  321. package/src/clis/boss/recommend.ts +1 -1
  322. package/src/clis/boss/resume.ts +1 -1
  323. package/src/clis/boss/search.ts +1 -1
  324. package/src/clis/boss/send.ts +5 -4
  325. package/src/clis/boss/stats.ts +1 -1
  326. package/src/clis/chatgpt/ask.ts +5 -0
  327. package/src/clis/chatgpt/new.ts +7 -2
  328. package/src/clis/chatgpt/read.ts +7 -2
  329. package/src/clis/chatgpt/send.ts +3 -2
  330. package/src/clis/chatgpt/status.ts +6 -1
  331. package/src/clis/chatwise/ask.ts +7 -2
  332. package/src/clis/chatwise/export.ts +2 -0
  333. package/src/clis/chatwise/history.ts +2 -0
  334. package/src/clis/chatwise/model.ts +7 -3
  335. package/src/clis/chatwise/new.ts +3 -20
  336. package/src/clis/chatwise/read.ts +2 -0
  337. package/src/clis/chatwise/screenshot.ts +3 -32
  338. package/src/clis/chatwise/send.ts +7 -2
  339. package/src/clis/chatwise/shared.ts +8 -0
  340. package/src/clis/chatwise/status.ts +3 -24
  341. package/src/clis/codex/ask.ts +5 -2
  342. package/src/clis/codex/dump.ts +2 -27
  343. package/src/clis/codex/new.ts +2 -28
  344. package/src/clis/codex/screenshot.ts +2 -32
  345. package/src/clis/codex/send.ts +5 -4
  346. package/src/clis/codex/status.ts +2 -24
  347. package/src/clis/cursor/ask.ts +2 -1
  348. package/src/clis/cursor/composer.ts +2 -1
  349. package/src/clis/cursor/dump.ts +2 -27
  350. package/src/clis/cursor/new.ts +2 -20
  351. package/src/clis/cursor/read.ts +2 -1
  352. package/src/clis/cursor/screenshot.ts +1 -36
  353. package/src/clis/cursor/send.ts +2 -1
  354. package/src/clis/cursor/status.ts +2 -22
  355. package/src/clis/dictionary/examples.yaml +25 -0
  356. package/src/clis/dictionary/search.yaml +27 -0
  357. package/src/clis/dictionary/synonyms.yaml +25 -0
  358. package/src/clis/douban/book-hot.ts +1 -1
  359. package/src/clis/douban/movie-hot.ts +1 -1
  360. package/src/clis/douban/search.ts +1 -1
  361. package/src/clis/douban/utils.ts +165 -1
  362. package/src/clis/doubao/ask.ts +1 -1
  363. package/src/clis/doubao/new.ts +1 -1
  364. package/src/clis/doubao/read.ts +1 -1
  365. package/src/clis/doubao/send.ts +1 -1
  366. package/src/clis/doubao/status.ts +1 -1
  367. package/src/clis/doubao-app/ask.ts +1 -1
  368. package/src/clis/doubao-app/new.ts +1 -1
  369. package/src/clis/doubao-app/read.ts +1 -1
  370. package/src/clis/doubao-app/send.ts +1 -1
  371. package/src/clis/grok/ask.test.ts +25 -0
  372. package/src/clis/grok/ask.ts +25 -12
  373. package/src/clis/jd/item.test.ts +35 -0
  374. package/src/clis/jd/item.ts +101 -0
  375. package/src/clis/jike/feed.ts +1 -1
  376. package/src/clis/jike/search.ts +1 -1
  377. package/src/clis/linkedin/search.ts +5 -4
  378. package/src/clis/linkedin/timeline.test.ts +99 -0
  379. package/src/clis/linkedin/timeline.ts +532 -0
  380. package/src/clis/medium/feed.ts +1 -1
  381. package/src/clis/medium/search.ts +1 -1
  382. package/src/clis/medium/user.ts +1 -1
  383. package/src/clis/medium/{shared.ts → utils.ts} +2 -1
  384. package/src/clis/pixiv/detail.yaml +49 -0
  385. package/src/clis/pixiv/download.test.ts +114 -0
  386. package/src/clis/pixiv/download.ts +91 -0
  387. package/src/clis/pixiv/illusts.test.ts +115 -0
  388. package/src/clis/pixiv/illusts.ts +78 -0
  389. package/src/clis/pixiv/ranking.yaml +53 -0
  390. package/src/clis/pixiv/search.test.ts +97 -0
  391. package/src/clis/pixiv/search.ts +53 -0
  392. package/src/clis/pixiv/test-utils.ts +29 -0
  393. package/src/clis/pixiv/user.yaml +46 -0
  394. package/src/clis/pixiv/utils.ts +62 -0
  395. package/src/clis/reddit/comment.ts +2 -1
  396. package/src/clis/reddit/read.test.ts +34 -0
  397. package/src/clis/reddit/read.ts +4 -3
  398. package/src/clis/reddit/save.ts +2 -1
  399. package/src/clis/reddit/saved.ts +6 -2
  400. package/src/clis/reddit/subscribe.ts +2 -1
  401. package/src/clis/reddit/upvote.ts +2 -1
  402. package/src/clis/reddit/upvoted.ts +6 -2
  403. package/src/clis/sinablog/article.ts +1 -1
  404. package/src/clis/sinablog/hot.ts +1 -1
  405. package/src/clis/sinablog/user.ts +1 -1
  406. package/src/clis/substack/feed.ts +1 -1
  407. package/src/clis/substack/publication.ts +1 -1
  408. package/src/clis/substack/search.ts +3 -2
  409. package/src/clis/substack/{shared.ts → utils.ts} +3 -2
  410. package/src/clis/tiktok/search.yaml +2 -1
  411. package/src/clis/twitter/accept.ts +2 -1
  412. package/src/clis/twitter/article.ts +3 -1
  413. package/src/clis/twitter/block.ts +2 -1
  414. package/src/clis/twitter/bookmark.ts +2 -1
  415. package/src/clis/twitter/bookmarks.ts +3 -2
  416. package/src/clis/twitter/delete.ts +2 -1
  417. package/src/clis/twitter/follow.ts +2 -1
  418. package/src/clis/twitter/followers.ts +3 -2
  419. package/src/clis/twitter/following.ts +3 -2
  420. package/src/clis/twitter/hide-reply.ts +2 -1
  421. package/src/clis/twitter/like.ts +2 -1
  422. package/src/clis/twitter/notifications.ts +2 -1
  423. package/src/clis/twitter/post.ts +2 -1
  424. package/src/clis/twitter/profile.ts +4 -2
  425. package/src/clis/twitter/reply-dm.ts +2 -1
  426. package/src/clis/twitter/reply.ts +2 -1
  427. package/src/clis/twitter/search.test.ts +113 -0
  428. package/src/clis/twitter/search.ts +38 -14
  429. package/src/clis/twitter/thread.ts +2 -2
  430. package/src/clis/twitter/timeline.ts +3 -2
  431. package/src/clis/twitter/trending.ts +3 -2
  432. package/src/clis/twitter/unblock.ts +2 -1
  433. package/src/clis/twitter/unbookmark.ts +2 -1
  434. package/src/clis/twitter/unfollow.ts +2 -1
  435. package/src/clis/v2ex/daily.ts +3 -2
  436. package/src/clis/v2ex/me.ts +3 -2
  437. package/src/clis/v2ex/notifications.ts +3 -4
  438. package/src/clis/web/read.ts +210 -0
  439. package/src/clis/xueqiu/danjuan-utils.test.ts +49 -0
  440. package/src/clis/xueqiu/danjuan-utils.ts +176 -0
  441. package/src/clis/xueqiu/fund-holdings.ts +32 -0
  442. package/src/clis/xueqiu/fund-snapshot.ts +27 -0
  443. package/src/clis/youtube/transcript.ts +5 -4
  444. package/src/clis/youtube/video.ts +3 -2
  445. package/src/constants.ts +3 -0
  446. package/src/daemon.ts +7 -5
  447. package/src/discovery.ts +12 -34
  448. package/src/doctor.ts +5 -3
  449. package/src/download/index.test.ts +93 -2
  450. package/src/download/index.ts +44 -23
  451. package/src/download/media-download.ts +5 -3
  452. package/src/engine.test.ts +84 -3
  453. package/src/execution.ts +62 -46
  454. package/src/explore.ts +21 -90
  455. package/src/external-clis.yaml +0 -8
  456. package/src/external.test.ts +9 -0
  457. package/src/external.ts +12 -10
  458. package/src/generate.ts +4 -41
  459. package/src/hooks.test.ts +126 -0
  460. package/src/hooks.ts +90 -0
  461. package/src/interceptor.ts +73 -23
  462. package/src/main.ts +2 -0
  463. package/src/output.ts +14 -6
  464. package/src/pipeline/executor.ts +1 -1
  465. package/src/pipeline/steps/browser.ts +1 -3
  466. package/src/pipeline/steps/download.test.ts +136 -0
  467. package/src/pipeline/steps/download.ts +47 -34
  468. package/src/pipeline/steps/fetch.test.ts +179 -0
  469. package/src/pipeline/steps/fetch.ts +39 -23
  470. package/src/pipeline/steps/transform.ts +2 -6
  471. package/src/pipeline/template.test.ts +28 -0
  472. package/src/pipeline/template.ts +67 -79
  473. package/src/pipeline/transform.test.ts +20 -0
  474. package/src/plugin.test.ts +251 -3
  475. package/src/plugin.ts +265 -21
  476. package/src/record.ts +12 -84
  477. package/src/registry-api.ts +2 -0
  478. package/src/registry.ts +7 -4
  479. package/src/runtime.ts +14 -4
  480. package/src/snapshotFormatter.ts +43 -121
  481. package/src/utils.ts +39 -0
  482. package/src/validate.ts +3 -6
  483. package/src/yaml-schema.ts +28 -0
  484. package/tests/e2e/browser-auth.test.ts +25 -0
  485. package/tests/e2e/plugin-management.test.ts +137 -0
  486. package/tests/e2e/public-commands.test.ts +34 -1
  487. package/vitest.config.ts +19 -1
  488. package/.github/workflows/pkg-pr-new.yml +0 -30
  489. package/dist/clis/douban/shared.d.ts +0 -4
  490. package/dist/clis/douban/shared.js +0 -155
  491. package/src/clis/douban/shared.ts +0 -165
  492. /package/dist/clis/boss/{common.d.ts → utils.d.ts} +0 -0
  493. /package/dist/clis/boss/{common.js → utils.js} +0 -0
  494. /package/dist/clis/doubao/{common.d.ts → utils.d.ts} +0 -0
  495. /package/dist/clis/doubao/{common.js → utils.js} +0 -0
  496. /package/dist/clis/doubao-app/{common.d.ts → utils.d.ts} +0 -0
  497. /package/dist/clis/doubao-app/{common.js → utils.js} +0 -0
  498. /package/dist/clis/jike/{shared.d.ts → utils.d.ts} +0 -0
  499. /package/dist/clis/jike/{shared.js → utils.js} +0 -0
  500. /package/dist/clis/medium/{shared.d.ts → utils.d.ts} +0 -0
  501. /package/dist/clis/sinablog/{shared.d.ts → utils.d.ts} +0 -0
  502. /package/dist/clis/sinablog/{shared.js → utils.js} +0 -0
  503. /package/dist/clis/substack/{shared.d.ts → utils.d.ts} +0 -0
  504. /package/src/clis/boss/{common.ts → utils.ts} +0 -0
  505. /package/src/clis/doubao/{common.ts → utils.ts} +0 -0
  506. /package/src/clis/doubao-app/{common.ts → utils.ts} +0 -0
  507. /package/src/clis/jike/{shared.ts → utils.ts} +0 -0
  508. /package/src/clis/sinablog/{shared.ts → utils.ts} +0 -0
@@ -10,30 +10,16 @@
10
10
  import * as fs from 'node:fs';
11
11
  import * as path from 'node:path';
12
12
  import { render } from '../template.js';
13
+ import { getErrorMessage } from '../../errors.js';
13
14
  import { httpDownload, ytdlpDownload, saveDocument, detectContentType, requiresYtdlp, sanitizeFilename, generateFilename, exportCookiesToNetscape, getTempDir, formatCookieHeader, } from '../../download/index.js';
14
15
  import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
15
- /**
16
- * Simple async concurrency limiter for downloads.
17
- */
18
- async function mapConcurrent(items, limit, fn) {
19
- const results = new Array(items.length);
20
- let index = 0;
21
- async function worker() {
22
- while (index < items.length) {
23
- const i = index++;
24
- results[i] = await fn(items[i], i);
25
- }
26
- }
27
- const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
28
- await Promise.all(workers);
29
- return results;
30
- }
16
+ import { mapConcurrent } from '../../utils.js';
31
17
  /**
32
18
  * Extract cookies from browser page.
33
19
  */
34
20
  async function extractBrowserCookies(page, domain) {
35
21
  try {
36
- const cookies = await page.getCookies(domain ? { domain } : {});
22
+ const cookies = await page.getCookies({ domain });
37
23
  return formatCookieHeader(cookies);
38
24
  }
39
25
  catch {
@@ -61,6 +47,13 @@ async function extractCookiesArray(page, domain) {
61
47
  return [];
62
48
  }
63
49
  }
50
+ function dedupeCookies(cookies) {
51
+ const deduped = new Map();
52
+ for (const cookie of cookies) {
53
+ deduped.set(`${cookie.domain}\t${cookie.path}\t${cookie.name}`, cookie);
54
+ }
55
+ return [...deduped.values()];
56
+ }
64
57
  /**
65
58
  * Download step handler for YAML pipelines.
66
59
  *
@@ -101,21 +94,28 @@ export async function stepDownload(page, params, data, args) {
101
94
  }
102
95
  // Create progress tracker
103
96
  const tracker = new DownloadProgressTracker(items.length, showProgress);
104
- // Extract cookies if browser is available
105
- let cookies = '';
97
+ // Cache cookie lookups per domain so mixed-domain batches stay isolated without repeated browser calls.
98
+ const cookieHeaderCache = new Map();
106
99
  let cookiesFile;
107
100
  if (page) {
108
- cookies = await extractBrowserCookies(page);
109
101
  // For yt-dlp, we need to export cookies to Netscape format
110
102
  if (useYtdlp || items.some((item, index) => {
111
103
  const url = String(render(urlTemplate, { args, data, item, index }));
112
104
  return requiresYtdlp(url);
113
105
  })) {
114
106
  try {
115
- // Try to get domain from first URL
116
- const firstUrl = String(render(urlTemplate, { args, data, item: items[0], index: 0 }));
117
- const domain = new URL(firstUrl).hostname;
118
- const cookiesArray = await extractCookiesArray(page, domain);
107
+ const ytdlpDomains = [...new Set(items.flatMap((item, index) => {
108
+ const url = String(render(urlTemplate, { args, data, item, index }));
109
+ if (!useYtdlp && !requiresYtdlp(url))
110
+ return [];
111
+ try {
112
+ return [new URL(url).hostname];
113
+ }
114
+ catch {
115
+ return [];
116
+ }
117
+ }))];
118
+ const cookiesArray = dedupeCookies((await Promise.all(ytdlpDomains.map((domain) => extractCookiesArray(page, domain)))).flat());
119
119
  if (cookiesArray.length > 0) {
120
120
  const tempDir = getTempDir();
121
121
  fs.mkdirSync(tempDir, { recursive: true });
@@ -199,6 +199,21 @@ export async function stepDownload(page, params, data, args) {
199
199
  }
200
200
  else {
201
201
  // Direct HTTP download
202
+ let cookies = '';
203
+ if (page) {
204
+ try {
205
+ const targetDomain = new URL(url).hostname;
206
+ let cookiePromise = cookieHeaderCache.get(targetDomain);
207
+ if (!cookiePromise) {
208
+ cookiePromise = extractBrowserCookies(page, targetDomain);
209
+ cookieHeaderCache.set(targetDomain, cookiePromise);
210
+ }
211
+ cookies = await cookiePromise;
212
+ }
213
+ catch {
214
+ cookies = '';
215
+ }
216
+ }
202
217
  result = await httpDownload(url, destPath, {
203
218
  cookies,
204
219
  timeout,
@@ -214,9 +229,10 @@ export async function stepDownload(page, params, data, args) {
214
229
  }
215
230
  }
216
231
  catch (err) {
217
- result = { success: false, size: 0, error: err.message };
232
+ const msg = getErrorMessage(err);
233
+ result = { success: false, size: 0, error: msg };
218
234
  if (progressBar) {
219
- progressBar.fail(err.message);
235
+ progressBar.fail(msg);
220
236
  }
221
237
  }
222
238
  tracker.onFileComplete(result.success);
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,101 @@
1
+ import { beforeEach, describe, expect, it, vi } from 'vitest';
2
+ import * as os from 'node:os';
3
+ import * as path from 'node:path';
4
+ const { mockHttpDownload, mockYtdlpDownload, mockExportCookiesToNetscape } = vi.hoisted(() => ({
5
+ mockHttpDownload: vi.fn(),
6
+ mockYtdlpDownload: vi.fn(),
7
+ mockExportCookiesToNetscape: vi.fn(),
8
+ }));
9
+ vi.mock('../../download/index.js', async () => {
10
+ const actual = await vi.importActual('../../download/index.js');
11
+ return {
12
+ ...actual,
13
+ httpDownload: mockHttpDownload,
14
+ ytdlpDownload: mockYtdlpDownload,
15
+ exportCookiesToNetscape: mockExportCookiesToNetscape,
16
+ };
17
+ });
18
+ import { stepDownload } from './download.js';
19
+ function createMockPage(getCookies) {
20
+ return {
21
+ goto: vi.fn(),
22
+ evaluate: vi.fn().mockResolvedValue(null),
23
+ getCookies,
24
+ snapshot: vi.fn().mockResolvedValue(''),
25
+ click: vi.fn(),
26
+ typeText: vi.fn(),
27
+ pressKey: vi.fn(),
28
+ scrollTo: vi.fn(),
29
+ getFormState: vi.fn().mockResolvedValue({}),
30
+ wait: vi.fn(),
31
+ tabs: vi.fn().mockResolvedValue([]),
32
+ closeTab: vi.fn(),
33
+ newTab: vi.fn(),
34
+ selectTab: vi.fn(),
35
+ networkRequests: vi.fn().mockResolvedValue([]),
36
+ consoleMessages: vi.fn().mockResolvedValue([]),
37
+ scroll: vi.fn(),
38
+ autoScroll: vi.fn(),
39
+ installInterceptor: vi.fn(),
40
+ getInterceptedRequests: vi.fn().mockResolvedValue([]),
41
+ screenshot: vi.fn().mockResolvedValue(''),
42
+ };
43
+ }
44
+ describe('stepDownload', () => {
45
+ beforeEach(() => {
46
+ mockHttpDownload.mockReset();
47
+ mockHttpDownload.mockResolvedValue({ success: true, size: 2 });
48
+ mockYtdlpDownload.mockReset();
49
+ mockYtdlpDownload.mockResolvedValue({ success: true, size: 2 });
50
+ mockExportCookiesToNetscape.mockReset();
51
+ });
52
+ it('scopes browser cookies to each direct-download target domain', async () => {
53
+ const page = createMockPage(vi.fn().mockImplementation(async (opts) => {
54
+ const domain = opts?.domain ?? 'unknown';
55
+ return [{ name: 'sid', value: domain, domain }];
56
+ }));
57
+ await stepDownload(page, {
58
+ url: '${{ item.url }}',
59
+ dir: path.join(os.tmpdir(), 'opencli-download-test'),
60
+ filename: '${{ index }}.txt',
61
+ progress: false,
62
+ concurrency: 1,
63
+ }, [
64
+ { url: 'https://a.example/file-1.txt' },
65
+ { url: 'https://b.example/file-2.txt' },
66
+ ], {});
67
+ expect(mockHttpDownload).toHaveBeenNthCalledWith(1, 'https://a.example/file-1.txt', path.join(os.tmpdir(), 'opencli-download-test', '0.txt'), expect.objectContaining({ cookies: 'sid=a.example' }));
68
+ expect(mockHttpDownload).toHaveBeenNthCalledWith(2, 'https://b.example/file-2.txt', path.join(os.tmpdir(), 'opencli-download-test', '1.txt'), expect.objectContaining({ cookies: 'sid=b.example' }));
69
+ });
70
+ it('builds yt-dlp cookies from all target domains instead of only the first item', async () => {
71
+ const getCookies = vi.fn().mockImplementation(async (opts) => {
72
+ const domain = opts?.domain ?? 'unknown';
73
+ return [{
74
+ name: `sid-${domain}`,
75
+ value: domain,
76
+ domain,
77
+ path: '/',
78
+ secure: false,
79
+ httpOnly: false,
80
+ }];
81
+ });
82
+ const page = createMockPage(getCookies);
83
+ await stepDownload(page, {
84
+ url: '${{ item.url }}',
85
+ dir: '/tmp/opencli-download-test',
86
+ filename: '${{ index }}.mp4',
87
+ progress: false,
88
+ concurrency: 1,
89
+ }, [
90
+ { url: 'https://www.youtube.com/watch?v=one' },
91
+ { url: 'https://www.bilibili.com/video/BV1xx411c7mD' },
92
+ ], {});
93
+ expect(getCookies).toHaveBeenCalledWith({ domain: 'www.youtube.com' });
94
+ expect(getCookies).toHaveBeenCalledWith({ domain: 'www.bilibili.com' });
95
+ expect(mockExportCookiesToNetscape).toHaveBeenCalledWith(expect.arrayContaining([
96
+ expect.objectContaining({ name: 'sid-www.youtube.com', domain: 'www.youtube.com' }),
97
+ expect.objectContaining({ name: 'sid-www.bilibili.com', domain: 'www.bilibili.com' }),
98
+ ]), expect.any(String));
99
+ expect(mockYtdlpDownload).toHaveBeenCalledTimes(2);
100
+ });
101
+ });
@@ -1,24 +1,10 @@
1
1
  /**
2
2
  * Pipeline step: fetch — HTTP API requests.
3
3
  */
4
+ import { CliError, getErrorMessage } from '../../errors.js';
5
+ import { log } from '../../logger.js';
4
6
  import { render } from '../template.js';
5
- function isRecord(value) {
6
- return typeof value === 'object' && value !== null && !Array.isArray(value);
7
- }
8
- /** Simple async concurrency limiter */
9
- async function mapConcurrent(items, limit, fn) {
10
- const results = new Array(items.length);
11
- let index = 0;
12
- async function worker() {
13
- while (index < items.length) {
14
- const i = index++;
15
- results[i] = await fn(items[i], i);
16
- }
17
- }
18
- const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
19
- await Promise.all(workers);
20
- return results;
21
- }
7
+ import { isRecord, mapConcurrent } from '../../utils.js';
22
8
  /** Single URL fetch helper */
23
9
  async function fetchSingle(page, url, method, queryParams, headers, args, data) {
24
10
  const renderedParams = {};
@@ -34,19 +20,32 @@ async function fetchSingle(page, url, method, queryParams, headers, args, data)
34
20
  }
35
21
  if (page === null) {
36
22
  const resp = await fetch(finalUrl, { method: method.toUpperCase(), headers: renderedHeaders });
23
+ if (!resp.ok) {
24
+ throw new CliError('FETCH_ERROR', `HTTP ${resp.status} ${resp.statusText} from ${finalUrl}`);
25
+ }
37
26
  return resp.json();
38
27
  }
39
28
  const headersJs = JSON.stringify(renderedHeaders);
40
29
  const urlJs = JSON.stringify(finalUrl);
41
30
  const methodJs = JSON.stringify(method.toUpperCase());
42
- return page.evaluate(`
31
+ // Return error status instead of throwing inside evaluate to avoid CDP wrapper
32
+ // rewriting the message (CDP prepends "Evaluate error: " to thrown errors).
33
+ const result = await page.evaluate(`
43
34
  async () => {
44
35
  const resp = await fetch(${urlJs}, {
45
36
  method: ${methodJs}, headers: ${headersJs}, credentials: "include"
46
37
  });
38
+ if (!resp.ok) {
39
+ return { __httpError: resp.status, statusText: resp.statusText };
40
+ }
47
41
  return await resp.json();
48
42
  }
49
43
  `);
44
+ if (result && typeof result === 'object' && '__httpError' in result) {
45
+ const { __httpError: status, statusText } = result;
46
+ throw new CliError('FETCH_ERROR', `HTTP ${status} ${statusText} from ${finalUrl}`);
47
+ }
48
+ return result;
50
49
  }
51
50
  /**
52
51
  * Batch fetch: send all URLs into the browser as a single evaluate() call.
@@ -56,10 +55,11 @@ async function fetchSingle(page, url, method, queryParams, headers, args, data)
56
55
  async function fetchBatchInBrowser(page, urls, method, headers, concurrency) {
57
56
  const headersJs = JSON.stringify(headers);
58
57
  const urlsJs = JSON.stringify(urls);
58
+ const methodJs = JSON.stringify(method);
59
59
  return (await page.evaluate(`
60
60
  async () => {
61
61
  const urls = ${urlsJs};
62
- const method = "${method}";
62
+ const method = ${methodJs};
63
63
  const headers = ${headersJs};
64
64
  const concurrency = ${concurrency};
65
65
 
@@ -71,9 +71,13 @@ async function fetchBatchInBrowser(page, urls, method, headers, concurrency) {
71
71
  const i = idx++;
72
72
  try {
73
73
  const resp = await fetch(urls[i], { method, headers, credentials: "include" });
74
+ if (!resp.ok) {
75
+ throw new Error('HTTP ' + resp.status + ' ' + resp.statusText + ' from ' + urls[i]);
76
+ }
74
77
  results[i] = await resp.json();
75
78
  } catch (e) {
76
- results[i] = { error: e.message };
79
+ results[i] = { error: e instanceof Error ? e.message : String(e) };
80
+ // Note: getErrorMessage() is a Node.js utility — can't use it inside evaluate()
77
81
  }
78
82
  }
79
83
  }
@@ -111,12 +115,26 @@ export async function stepFetch(page, params, data, args) {
111
115
  });
112
116
  // BATCH IPC: if browser is available, batch all fetches into a single evaluate() call
113
117
  if (page !== null) {
114
- return fetchBatchInBrowser(page, urls, method.toUpperCase(), renderedHeaders, concurrency);
118
+ const results = await fetchBatchInBrowser(page, urls, method.toUpperCase(), renderedHeaders, concurrency);
119
+ for (let i = 0; i < results.length; i++) {
120
+ const r = results[i];
121
+ if (r && typeof r === 'object' && 'error' in r) {
122
+ log.warn(`Batch fetch failed for ${urls[i]}: ${r.error}`);
123
+ }
124
+ }
125
+ return results;
115
126
  }
116
127
  // Non-browser: use concurrent pool (already optimized)
117
128
  return mapConcurrent(data, concurrency, async (item, index) => {
118
129
  const itemUrl = String(render(urlTemplate, { args, data, item, index }));
119
- return fetchSingle(null, itemUrl, method, queryParams, headers, args, data);
130
+ try {
131
+ return await fetchSingle(null, itemUrl, method, queryParams, headers, args, data);
132
+ }
133
+ catch (error) {
134
+ const message = getErrorMessage(error);
135
+ log.warn(`Batch fetch failed for ${itemUrl}: ${message}`);
136
+ return { error: message };
137
+ }
120
138
  });
121
139
  }
122
140
  const url = render(urlOrObj, { args, data });
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,123 @@
1
+ import { afterEach, describe, expect, it, vi } from 'vitest';
2
+ import { CliError } from '../../errors.js';
3
+ import { stepFetch } from './fetch.js';
4
+ afterEach(() => {
5
+ vi.restoreAllMocks();
6
+ vi.unstubAllGlobals();
7
+ });
8
+ describe('stepFetch', () => {
9
+ // W1 + W4: non-browser single fetch throws CliError with FETCH_ERROR code and full message
10
+ it('throws CliError with FETCH_ERROR code on non-ok responses without a browser session', async () => {
11
+ const jsonMock = vi.fn().mockResolvedValue({ error: 'rate limited' });
12
+ const fetchMock = vi.fn().mockResolvedValue({
13
+ ok: false,
14
+ status: 429,
15
+ statusText: 'Too Many Requests',
16
+ json: jsonMock,
17
+ });
18
+ vi.stubGlobal('fetch', fetchMock);
19
+ const err = await stepFetch(null, { url: 'https://api.example.com/items' }, null, {}).catch((e) => e);
20
+ expect(err).toBeInstanceOf(CliError);
21
+ expect(err.code).toBe('FETCH_ERROR');
22
+ expect(err.message).toBe('HTTP 429 Too Many Requests from https://api.example.com/items');
23
+ expect(jsonMock).not.toHaveBeenCalled();
24
+ });
25
+ // W1 + W3: browser single fetch returns error status from evaluate, outer code throws CliError
26
+ it('throws CliError with FETCH_ERROR code on non-ok responses inside the browser session', async () => {
27
+ const jsonMock = vi.fn().mockResolvedValue({ error: 'auth required' });
28
+ const fetchMock = vi.fn().mockResolvedValue({
29
+ ok: false,
30
+ status: 401,
31
+ statusText: 'Unauthorized',
32
+ json: jsonMock,
33
+ });
34
+ vi.stubGlobal('fetch', fetchMock);
35
+ // Simulate real CDP behavior: evaluate returns a value, errors are thrown outside
36
+ const page = {
37
+ evaluate: vi.fn(async (js) => Function(`return (${js})`)()()),
38
+ };
39
+ const err = await stepFetch(page, { url: 'https://api.example.com/items' }, null, {}).catch((e) => e);
40
+ expect(err).toBeInstanceOf(CliError);
41
+ expect(err.code).toBe('FETCH_ERROR');
42
+ expect(err.message).toBe('HTTP 401 Unauthorized from https://api.example.com/items');
43
+ expect(jsonMock).not.toHaveBeenCalled();
44
+ });
45
+ it('returns per-item HTTP errors for batch fetches without a browser session', async () => {
46
+ const jsonMock = vi.fn().mockResolvedValue({ error: 'upstream unavailable' });
47
+ const fetchMock = vi.fn().mockResolvedValue({
48
+ ok: false,
49
+ status: 503,
50
+ statusText: 'Service Unavailable',
51
+ json: jsonMock,
52
+ });
53
+ vi.stubGlobal('fetch', fetchMock);
54
+ await expect(stepFetch(null, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }], {})).resolves.toEqual([
55
+ { error: 'HTTP 503 Service Unavailable from https://api.example.com/items/1' },
56
+ ]);
57
+ expect(jsonMock).not.toHaveBeenCalled();
58
+ });
59
+ it('returns per-item HTTP errors for batch browser fetches', async () => {
60
+ const jsonMock = vi.fn().mockResolvedValue({ error: 'upstream unavailable' });
61
+ const fetchMock = vi.fn().mockResolvedValue({
62
+ ok: false,
63
+ status: 503,
64
+ statusText: 'Service Unavailable',
65
+ json: jsonMock,
66
+ });
67
+ vi.stubGlobal('fetch', fetchMock);
68
+ const page = {
69
+ evaluate: vi.fn(async (js) => Function(`return (${js})`)()()),
70
+ };
71
+ await expect(stepFetch(page, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }], {})).resolves.toEqual([
72
+ { error: 'HTTP 503 Service Unavailable from https://api.example.com/items/1' },
73
+ ]);
74
+ expect(jsonMock).not.toHaveBeenCalled();
75
+ });
76
+ it('stringifies non-Error batch browser failures consistently', async () => {
77
+ vi.stubGlobal('fetch', vi.fn().mockRejectedValue('socket hang up'));
78
+ const page = {
79
+ evaluate: vi.fn(async (js) => Function(`return (${js})`)()()),
80
+ };
81
+ await expect(stepFetch(page, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }], {})).resolves.toEqual([
82
+ { error: 'socket hang up' },
83
+ ]);
84
+ });
85
+ it('stringifies non-Error batch non-browser failures consistently', async () => {
86
+ vi.stubGlobal('fetch', vi.fn().mockRejectedValue('socket hang up'));
87
+ await expect(stepFetch(null, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }], {})).resolves.toEqual([
88
+ { error: 'socket hang up' },
89
+ ]);
90
+ });
91
+ // W2: batch item failures emit a warning log
92
+ it('logs a warning for each failed batch item in non-browser mode', async () => {
93
+ const { log } = await import('../../logger.js');
94
+ const warnSpy = vi.spyOn(log, 'warn');
95
+ vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
96
+ ok: false,
97
+ status: 503,
98
+ statusText: 'Service Unavailable',
99
+ json: vi.fn(),
100
+ }));
101
+ await stepFetch(null, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }, { id: 2 }], {});
102
+ expect(warnSpy).toHaveBeenCalledTimes(2);
103
+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/1'));
104
+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/2'));
105
+ });
106
+ it('logs a warning for each failed batch item in browser mode', async () => {
107
+ const { log } = await import('../../logger.js');
108
+ const warnSpy = vi.spyOn(log, 'warn');
109
+ vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
110
+ ok: false,
111
+ status: 502,
112
+ statusText: 'Bad Gateway',
113
+ json: vi.fn(),
114
+ }));
115
+ const page = {
116
+ evaluate: vi.fn(async (js) => Function(`return (${js})`)()()),
117
+ };
118
+ await stepFetch(page, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }, { id: 2 }], {});
119
+ expect(warnSpy).toHaveBeenCalledTimes(2);
120
+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/1'));
121
+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/2'));
122
+ });
123
+ });
@@ -2,9 +2,7 @@
2
2
  * Pipeline steps: data transforms — select, map, filter, sort, limit.
3
3
  */
4
4
  import { render, evalExpr } from '../template.js';
5
- function isRecord(value) {
6
- return typeof value === 'object' && value !== null && !Array.isArray(value);
7
- }
5
+ import { isRecord } from '../../utils.js';
8
6
  export async function stepSelect(_page, params, data, args) {
9
7
  const pathStr = String(render(params, { args, data }));
10
8
  if (data && typeof data === 'object') {
@@ -61,9 +59,7 @@ export async function stepSort(_page, params, data, _args) {
61
59
  return [...data].sort((a, b) => {
62
60
  const left = isRecord(a) ? a[key] : undefined;
63
61
  const right = isRecord(b) ? b[key] : undefined;
64
- const va = left ?? '';
65
- const vb = right ?? '';
66
- const cmp = va < vb ? -1 : va > vb ? 1 : 0;
62
+ const cmp = String(left ?? '').localeCompare(String(right ?? ''), undefined, { numeric: true });
67
63
  return reverse ? -cmp : cmp;
68
64
  });
69
65
  }
@@ -1,9 +1,8 @@
1
1
  /**
2
2
  * Pipeline template engine: ${{ ... }} expression rendering.
3
3
  */
4
- function isRecord(value) {
5
- return typeof value === 'object' && value !== null && !Array.isArray(value);
6
- }
4
+ import vm from 'node:vm';
5
+ import { isRecord } from '../utils.js';
7
6
  export function render(template, ctx) {
8
7
  if (typeof template !== 'string')
9
8
  return template;
@@ -29,46 +28,28 @@ export function evalExpr(expr, ctx) {
29
28
  const data = ctx.data;
30
29
  const index = ctx.index ?? 0;
31
30
  // ── Pipe filters: expr | filter1(arg) | filter2 ──
32
- // Supports: default(val), join(sep), upper, lower, truncate(n), trim, replace(old,new)
33
- if (expr.includes('|') && !expr.includes('||')) {
34
- const segments = expr.split('|').map(s => s.trim());
35
- const mainExpr = segments[0];
36
- let result = resolvePath(mainExpr, { args, item, data, index });
37
- for (let i = 1; i < segments.length; i++) {
38
- result = applyFilter(segments[i], result);
31
+ // Split on single | (not ||) so "item.a || item.b | upper" works correctly.
32
+ const pipeSegments = expr.split(/(?<!\|)\|(?!\|)/).map(s => s.trim());
33
+ if (pipeSegments.length > 1) {
34
+ let result = evalExpr(pipeSegments[0], ctx);
35
+ for (let i = 1; i < pipeSegments.length; i++) {
36
+ result = applyFilter(pipeSegments[i], result);
39
37
  }
40
38
  return result;
41
39
  }
42
- // Arithmetic: index + 1
43
- const arithMatch = expr.match(/^([\w][\w.]*)\s*([+\-*/])\s*(\d+)$/);
44
- if (arithMatch) {
45
- const [, varName, op, numStr] = arithMatch;
46
- const val = resolvePath(varName, { args, item, data, index });
47
- if (val !== null && val !== undefined) {
48
- const numVal = Number(val);
49
- const num = Number(numStr);
50
- if (!isNaN(numVal)) {
51
- switch (op) {
52
- case '+': return numVal + num;
53
- case '-': return numVal - num;
54
- case '*': return numVal * num;
55
- case '/': return num !== 0 ? numVal / num : 0;
56
- }
57
- }
58
- }
59
- }
60
- // JS-like fallback expression: item.tweetCount || 'N/A'
61
- const orMatch = expr.match(/^(.+?)\s*\|\|\s*(.+)$/);
62
- if (orMatch) {
63
- const left = evalExpr(orMatch[1].trim(), ctx);
64
- if (left)
65
- return left;
66
- const right = orMatch[2].trim();
67
- return right.replace(/^['"]|['"]$/g, '');
68
- }
40
+ // Fast path: quoted string literal — skip VM overhead
41
+ const strLit = expr.match(/^(['"])(.*)\1$/);
42
+ if (strLit)
43
+ return strLit[2];
44
+ // Fast path: numeric literal
45
+ if (/^\d+(\.\d+)?$/.test(expr))
46
+ return Number(expr);
47
+ // Try resolving as a simple dotted path (item.foo.bar, args.limit, index)
69
48
  const resolved = resolvePath(expr, { args, item, data, index });
70
49
  if (resolved !== null && resolved !== undefined)
71
50
  return resolved;
51
+ // Fallback: evaluate as JS in a sandboxed VM.
52
+ // Handles ||, ??, arithmetic, ternary, method calls, etc. natively.
72
53
  return evalJsExpr(expr, { args, item, data, index });
73
54
  }
74
55
  /**
@@ -87,7 +68,7 @@ function applyFilter(filterExpr, value) {
87
68
  case 'default': {
88
69
  if (value === null || value === undefined || value === '') {
89
70
  const intVal = parseInt(filterArg, 10);
90
- if (!isNaN(intVal) && String(intVal) === filterArg.trim())
71
+ if (!Number.isNaN(intVal) && String(intVal) === filterArg.trim())
91
72
  return intVal;
92
73
  return filterArg;
93
74
  }
@@ -103,7 +84,7 @@ function applyFilter(filterExpr, value) {
103
84
  return typeof value === 'string' ? value.trim() : value;
104
85
  case 'truncate': {
105
86
  const n = parseInt(filterArg, 10) || 50;
106
- return typeof value === 'string' && value.length > n ? value.slice(0, n) + '...' : value;
87
+ return typeof value === 'string' && value.length > n ? `${value.slice(0, n)}...` : value;
107
88
  }
108
89
  case 'replace': {
109
90
  if (typeof value !== 'string')
@@ -132,6 +113,7 @@ function applyFilter(filterExpr, value) {
132
113
  case 'sanitize':
133
114
  // Remove invalid filename characters
134
115
  return typeof value === 'string'
116
+ // biome-ignore lint/suspicious/noControlCharactersInRegex: intentional - strips C0 control chars from filenames
135
117
  ? value.replace(/[<>:"/\\|?*\x00-\x1f]/g, '_')
136
118
  : value;
137
119
  case 'ext': {
@@ -196,26 +178,58 @@ export function resolvePath(pathStr, ctx) {
196
178
  }
197
179
  /**
198
180
  * Evaluate arbitrary JS expressions as a last-resort fallback.
199
- *
200
- * ⚠️ SECURITY NOTE: Uses `new Function()` to execute the expression.
201
- * This is acceptable here because:
202
- * 1. YAML adapters are authored by trusted repo contributors only.
203
- * 2. The expression runs in the same Node.js process (no sandbox).
204
- * 3. Only a curated set of globals is exposed (no require/import/process/fs).
205
- * If opencli ever loads untrusted third-party adapters, this MUST be replaced
206
- * with a proper sandboxed evaluator.
181
+ * Runs inside a `node:vm` sandbox with dynamic code generation disabled.
182
+ */
183
+ const FORBIDDEN_EXPR_PATTERNS = /\b(constructor|__proto__|prototype|globalThis|process|require|import|eval)\b/;
184
+ /**
185
+ * Deep-copy plain data to sever prototype chains, preventing sandbox escape
186
+ * via `args.constructor.constructor('return process')()` etc.
207
187
  */
188
+ function sanitizeContext(obj) {
189
+ if (obj === null || obj === undefined)
190
+ return obj;
191
+ if (typeof obj !== 'object' && typeof obj !== 'function')
192
+ return obj;
193
+ try {
194
+ return JSON.parse(JSON.stringify(obj));
195
+ }
196
+ catch {
197
+ return {};
198
+ }
199
+ }
208
200
  function evalJsExpr(expr, ctx) {
209
201
  // Guard against absurdly long expressions that could indicate injection.
210
202
  if (expr.length > 2000)
211
203
  return undefined;
212
- const args = ctx.args ?? {};
213
- const item = ctx.item ?? {};
214
- const data = ctx.data;
204
+ // Block obvious sandbox escape attempts.
205
+ if (FORBIDDEN_EXPR_PATTERNS.test(expr))
206
+ return undefined;
207
+ const args = sanitizeContext(ctx.args ?? {});
208
+ const item = sanitizeContext(ctx.item ?? {});
209
+ const data = sanitizeContext(ctx.data);
215
210
  const index = ctx.index ?? 0;
216
211
  try {
217
- const fn = new Function('args', 'item', 'data', 'index', 'encodeURIComponent', 'decodeURIComponent', 'JSON', 'Math', 'Number', 'String', 'Boolean', 'Array', 'Object', 'Date', `"use strict"; return (${expr});`);
218
- return fn(args, item, data, index, encodeURIComponent, decodeURIComponent, JSON, Math, Number, String, Boolean, Array, Object, Date);
212
+ return vm.runInNewContext(`(${expr})`, {
213
+ args,
214
+ item,
215
+ data,
216
+ index,
217
+ encodeURIComponent,
218
+ decodeURIComponent,
219
+ JSON,
220
+ Math,
221
+ Number,
222
+ String,
223
+ Boolean,
224
+ Array,
225
+ Date,
226
+ }, {
227
+ timeout: 50,
228
+ contextCodeGeneration: {
229
+ strings: false,
230
+ wasm: false,
231
+ },
232
+ });
219
233
  }
220
234
  catch {
221
235
  return undefined;