@jackwener/opencli 1.3.3 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (496) hide show
  1. package/.github/pull_request_template.md +3 -1
  2. package/.github/workflows/build-extension.yml +7 -1
  3. package/.github/workflows/ci.yml +29 -3
  4. package/.github/workflows/docs.yml +1 -1
  5. package/.github/workflows/e2e-headed.yml +20 -0
  6. package/.github/workflows/release.yml +1 -1
  7. package/.github/workflows/security.yml +0 -3
  8. package/CHANGELOG.md +55 -0
  9. package/CONTRIBUTING.md +6 -3
  10. package/README.md +30 -3
  11. package/README.zh-CN.md +30 -3
  12. package/SKILL.md +7 -1
  13. package/TESTING.md +1 -0
  14. package/chatwise-opencli.ps1 +82 -0
  15. package/dist/analysis.d.ts +38 -0
  16. package/dist/analysis.js +166 -0
  17. package/dist/browser/cdp.d.ts +0 -4
  18. package/dist/browser/cdp.js +53 -41
  19. package/dist/browser/cdp.test.d.ts +1 -0
  20. package/dist/browser/cdp.test.js +52 -0
  21. package/dist/browser/dom-snapshot.d.ts +2 -2
  22. package/dist/browser/dom-snapshot.js +54 -1
  23. package/dist/browser/dom-snapshot.test.js +36 -0
  24. package/dist/browser/index.d.ts +2 -2
  25. package/dist/browser/index.js +1 -1
  26. package/dist/browser/mcp.d.ts +0 -2
  27. package/dist/browser/mcp.js +2 -3
  28. package/dist/browser/page.d.ts +4 -3
  29. package/dist/browser/page.js +34 -37
  30. package/dist/browser/stealth.d.ts +0 -2
  31. package/dist/browser/stealth.js +24 -9
  32. package/dist/browser.test.js +2 -2
  33. package/dist/build-manifest.js +15 -9
  34. package/dist/build-manifest.test.js +12 -0
  35. package/dist/cascade.js +4 -2
  36. package/dist/cli-manifest.json +639 -258
  37. package/dist/cli.js +57 -29
  38. package/dist/clis/_shared/desktop-commands.d.ts +22 -0
  39. package/dist/clis/_shared/desktop-commands.js +108 -0
  40. package/dist/clis/antigravity/serve.js +5 -2
  41. package/dist/clis/arxiv/search.js +1 -1
  42. package/dist/clis/bilibili/dynamic.test.d.ts +1 -0
  43. package/dist/clis/bilibili/dynamic.test.js +68 -0
  44. package/dist/clis/bilibili/favorite.js +4 -2
  45. package/dist/clis/bilibili/following.js +3 -2
  46. package/dist/clis/bilibili/subtitle.js +8 -7
  47. package/dist/clis/bilibili/utils.js +2 -2
  48. package/dist/clis/boss/batchgreet.js +1 -1
  49. package/dist/clis/boss/chatlist.js +1 -1
  50. package/dist/clis/boss/chatmsg.js +1 -1
  51. package/dist/clis/boss/detail.js +1 -1
  52. package/dist/clis/boss/exchange.js +1 -1
  53. package/dist/clis/boss/greet.js +1 -1
  54. package/dist/clis/boss/invite.js +1 -1
  55. package/dist/clis/boss/joblist.js +1 -1
  56. package/dist/clis/boss/mark.js +4 -3
  57. package/dist/clis/boss/recommend.js +1 -1
  58. package/dist/clis/boss/resume.js +1 -1
  59. package/dist/clis/boss/search.js +1 -1
  60. package/dist/clis/boss/send.js +5 -4
  61. package/dist/clis/boss/stats.js +1 -1
  62. package/dist/clis/chatgpt/ask.js +4 -0
  63. package/dist/clis/chatgpt/new.js +5 -1
  64. package/dist/clis/chatgpt/read.js +5 -1
  65. package/dist/clis/chatgpt/send.js +2 -1
  66. package/dist/clis/chatgpt/status.js +5 -1
  67. package/dist/clis/chatwise/ask.js +8 -2
  68. package/dist/clis/chatwise/export.js +2 -0
  69. package/dist/clis/chatwise/history.js +2 -0
  70. package/dist/clis/chatwise/model.js +8 -3
  71. package/dist/clis/chatwise/new.js +3 -18
  72. package/dist/clis/chatwise/read.js +2 -0
  73. package/dist/clis/chatwise/screenshot.js +3 -27
  74. package/dist/clis/chatwise/send.js +8 -2
  75. package/dist/clis/chatwise/shared.d.ts +2 -0
  76. package/dist/clis/chatwise/shared.js +6 -0
  77. package/dist/clis/chatwise/status.js +3 -22
  78. package/dist/clis/codex/ask.js +6 -2
  79. package/dist/clis/codex/dump.js +2 -25
  80. package/dist/clis/codex/new.js +2 -25
  81. package/dist/clis/codex/screenshot.js +2 -27
  82. package/dist/clis/codex/send.js +6 -4
  83. package/dist/clis/codex/status.js +2 -22
  84. package/dist/clis/cursor/ask.js +2 -1
  85. package/dist/clis/cursor/composer.js +2 -1
  86. package/dist/clis/cursor/dump.js +2 -25
  87. package/dist/clis/cursor/new.js +2 -18
  88. package/dist/clis/cursor/read.js +2 -1
  89. package/dist/clis/cursor/screenshot.js +1 -30
  90. package/dist/clis/cursor/send.js +2 -1
  91. package/dist/clis/cursor/status.js +2 -21
  92. package/dist/clis/dictionary/examples.yaml +25 -0
  93. package/dist/clis/dictionary/search.yaml +27 -0
  94. package/dist/clis/dictionary/synonyms.yaml +25 -0
  95. package/dist/clis/douban/book-hot.js +1 -1
  96. package/dist/clis/douban/movie-hot.js +1 -1
  97. package/dist/clis/douban/search.js +1 -1
  98. package/dist/clis/douban/utils.d.ts +4 -1
  99. package/dist/clis/douban/utils.js +156 -1
  100. package/dist/clis/doubao/ask.js +1 -1
  101. package/dist/clis/doubao/new.js +1 -1
  102. package/dist/clis/doubao/read.js +1 -1
  103. package/dist/clis/doubao/send.js +1 -1
  104. package/dist/clis/doubao/status.js +1 -1
  105. package/dist/clis/doubao-app/ask.js +1 -1
  106. package/dist/clis/doubao-app/new.js +1 -1
  107. package/dist/clis/doubao-app/read.js +1 -1
  108. package/dist/clis/doubao-app/send.js +1 -1
  109. package/dist/clis/grok/ask.d.ts +4 -0
  110. package/dist/clis/grok/ask.js +28 -10
  111. package/dist/clis/grok/ask.test.js +18 -0
  112. package/dist/clis/jd/item.d.ts +1 -0
  113. package/dist/clis/jd/item.js +96 -0
  114. package/dist/clis/jd/item.test.d.ts +1 -0
  115. package/dist/clis/jd/item.test.js +28 -0
  116. package/dist/clis/jike/feed.js +1 -1
  117. package/dist/clis/jike/search.js +1 -1
  118. package/dist/clis/linkedin/search.js +5 -4
  119. package/dist/clis/linkedin/timeline.d.ts +21 -0
  120. package/dist/clis/linkedin/timeline.js +503 -0
  121. package/dist/clis/linkedin/timeline.test.d.ts +1 -0
  122. package/dist/clis/linkedin/timeline.test.js +81 -0
  123. package/dist/clis/medium/feed.js +1 -1
  124. package/dist/clis/medium/search.js +1 -1
  125. package/dist/clis/medium/user.js +1 -1
  126. package/dist/clis/medium/{shared.js → utils.js} +2 -1
  127. package/dist/clis/pixiv/detail.yaml +49 -0
  128. package/dist/clis/pixiv/download.d.ts +7 -0
  129. package/dist/clis/pixiv/download.js +78 -0
  130. package/dist/clis/pixiv/download.test.d.ts +1 -0
  131. package/dist/clis/pixiv/download.test.js +87 -0
  132. package/dist/clis/pixiv/illusts.d.ts +8 -0
  133. package/dist/clis/pixiv/illusts.js +65 -0
  134. package/dist/clis/pixiv/illusts.test.d.ts +1 -0
  135. package/dist/clis/pixiv/illusts.test.js +99 -0
  136. package/dist/clis/pixiv/ranking.yaml +53 -0
  137. package/dist/clis/pixiv/search.d.ts +6 -0
  138. package/dist/clis/pixiv/search.js +43 -0
  139. package/dist/clis/pixiv/search.test.d.ts +1 -0
  140. package/dist/clis/pixiv/search.test.js +83 -0
  141. package/dist/clis/pixiv/test-utils.d.ts +12 -0
  142. package/dist/clis/pixiv/test-utils.js +23 -0
  143. package/dist/clis/pixiv/user.yaml +46 -0
  144. package/dist/clis/pixiv/utils.d.ts +27 -0
  145. package/dist/clis/pixiv/utils.js +49 -0
  146. package/dist/clis/reddit/comment.js +2 -1
  147. package/dist/clis/reddit/read.js +4 -3
  148. package/dist/clis/reddit/read.test.d.ts +1 -0
  149. package/dist/clis/reddit/read.test.js +28 -0
  150. package/dist/clis/reddit/save.js +2 -1
  151. package/dist/clis/reddit/saved.js +7 -3
  152. package/dist/clis/reddit/subscribe.js +2 -1
  153. package/dist/clis/reddit/upvote.js +2 -1
  154. package/dist/clis/reddit/upvoted.js +7 -3
  155. package/dist/clis/sinablog/article.js +1 -1
  156. package/dist/clis/sinablog/hot.js +1 -1
  157. package/dist/clis/sinablog/user.js +1 -1
  158. package/dist/clis/substack/feed.js +1 -1
  159. package/dist/clis/substack/publication.js +1 -1
  160. package/dist/clis/substack/search.js +3 -2
  161. package/dist/clis/substack/{shared.js → utils.js} +3 -2
  162. package/dist/clis/tiktok/search.yaml +2 -1
  163. package/dist/clis/twitter/accept.js +2 -1
  164. package/dist/clis/twitter/article.js +4 -1
  165. package/dist/clis/twitter/block.js +2 -1
  166. package/dist/clis/twitter/bookmark.js +2 -1
  167. package/dist/clis/twitter/bookmarks.js +3 -2
  168. package/dist/clis/twitter/delete.js +2 -1
  169. package/dist/clis/twitter/follow.js +2 -1
  170. package/dist/clis/twitter/followers.js +3 -2
  171. package/dist/clis/twitter/following.js +3 -2
  172. package/dist/clis/twitter/hide-reply.js +2 -1
  173. package/dist/clis/twitter/like.js +2 -1
  174. package/dist/clis/twitter/notifications.js +2 -1
  175. package/dist/clis/twitter/post.js +2 -1
  176. package/dist/clis/twitter/profile.js +5 -2
  177. package/dist/clis/twitter/reply-dm.js +2 -1
  178. package/dist/clis/twitter/reply.js +2 -1
  179. package/dist/clis/twitter/search.js +30 -13
  180. package/dist/clis/twitter/search.test.d.ts +1 -0
  181. package/dist/clis/twitter/search.test.js +104 -0
  182. package/dist/clis/twitter/thread.js +2 -2
  183. package/dist/clis/twitter/timeline.js +3 -2
  184. package/dist/clis/twitter/trending.js +3 -2
  185. package/dist/clis/twitter/unblock.js +2 -1
  186. package/dist/clis/twitter/unbookmark.js +2 -1
  187. package/dist/clis/twitter/unfollow.js +2 -1
  188. package/dist/clis/v2ex/daily.js +3 -2
  189. package/dist/clis/v2ex/me.js +3 -2
  190. package/dist/clis/v2ex/notifications.js +4 -4
  191. package/dist/clis/web/read.d.ts +16 -0
  192. package/dist/clis/web/read.js +202 -0
  193. package/dist/clis/xueqiu/danjuan-utils.d.ts +55 -0
  194. package/dist/clis/xueqiu/danjuan-utils.js +126 -0
  195. package/dist/clis/xueqiu/danjuan-utils.test.d.ts +1 -0
  196. package/dist/clis/xueqiu/danjuan-utils.test.js +41 -0
  197. package/dist/clis/xueqiu/fund-holdings.d.ts +1 -0
  198. package/dist/clis/xueqiu/fund-holdings.js +28 -0
  199. package/dist/clis/xueqiu/fund-snapshot.d.ts +1 -0
  200. package/dist/clis/xueqiu/fund-snapshot.js +25 -0
  201. package/dist/clis/youtube/transcript.js +5 -4
  202. package/dist/clis/youtube/video.js +3 -2
  203. package/dist/daemon.js +7 -3
  204. package/dist/discovery.js +11 -10
  205. package/dist/doctor.js +2 -1
  206. package/dist/download/index.d.ts +4 -12
  207. package/dist/download/index.js +33 -12
  208. package/dist/download/index.test.js +79 -2
  209. package/dist/download/media-download.js +4 -2
  210. package/dist/engine.test.js +76 -4
  211. package/dist/execution.d.ts +1 -9
  212. package/dist/execution.js +56 -46
  213. package/dist/explore.js +12 -111
  214. package/dist/external-clis.yaml +0 -8
  215. package/dist/external.js +7 -5
  216. package/dist/external.test.js +4 -0
  217. package/dist/generate.d.ts +0 -9
  218. package/dist/generate.js +4 -20
  219. package/dist/hooks.d.ts +46 -0
  220. package/dist/hooks.js +56 -0
  221. package/dist/hooks.test.d.ts +4 -0
  222. package/dist/hooks.test.js +92 -0
  223. package/dist/interceptor.js +70 -23
  224. package/dist/main.js +2 -0
  225. package/dist/output.js +12 -6
  226. package/dist/pipeline/executor.js +1 -1
  227. package/dist/pipeline/steps/browser.js +1 -3
  228. package/dist/pipeline/steps/download.js +42 -26
  229. package/dist/pipeline/steps/download.test.d.ts +1 -0
  230. package/dist/pipeline/steps/download.test.js +101 -0
  231. package/dist/pipeline/steps/fetch.js +40 -22
  232. package/dist/pipeline/steps/fetch.test.d.ts +1 -0
  233. package/dist/pipeline/steps/fetch.test.js +123 -0
  234. package/dist/pipeline/steps/transform.js +2 -6
  235. package/dist/pipeline/template.js +66 -52
  236. package/dist/pipeline/template.test.js +28 -0
  237. package/dist/pipeline/transform.test.js +18 -0
  238. package/dist/plugin.d.ts +40 -1
  239. package/dist/plugin.js +214 -17
  240. package/dist/plugin.test.d.ts +1 -1
  241. package/dist/plugin.test.js +219 -3
  242. package/dist/record.js +6 -98
  243. package/dist/registry-api.d.ts +2 -0
  244. package/dist/registry-api.js +1 -0
  245. package/dist/registry.d.ts +5 -2
  246. package/dist/registry.js +1 -2
  247. package/dist/runtime.d.ts +0 -1
  248. package/dist/runtime.js +14 -4
  249. package/dist/snapshotFormatter.d.ts +7 -14
  250. package/dist/snapshotFormatter.js +38 -78
  251. package/dist/utils.d.ts +9 -0
  252. package/dist/utils.js +29 -0
  253. package/dist/validate.js +3 -5
  254. package/dist/yaml-schema.d.ts +26 -0
  255. package/dist/yaml-schema.js +5 -0
  256. package/docs/.vitepress/config.mts +3 -0
  257. package/docs/adapters/browser/dictionary.md +27 -0
  258. package/docs/adapters/browser/jd.md +27 -0
  259. package/docs/adapters/browser/linkedin.md +6 -0
  260. package/docs/adapters/browser/pixiv.md +92 -0
  261. package/docs/adapters/browser/web.md +30 -0
  262. package/docs/adapters/browser/xueqiu.md +27 -9
  263. package/docs/adapters/index.md +3 -1
  264. package/docs/comparison.md +125 -0
  265. package/docs/developer/contributing.md +21 -2
  266. package/docs/developer/testing.md +14 -8
  267. package/docs/developer/ts-adapter.md +18 -0
  268. package/docs/developer/yaml-adapter.md +16 -0
  269. package/docs/guide/plugins.md +10 -0
  270. package/docs/zh/guide/plugins.md +10 -0
  271. package/extension/dist/background.js +519 -444
  272. package/extension/manifest.json +1 -1
  273. package/extension/package.json +1 -1
  274. package/extension/src/background.test.ts +46 -1
  275. package/extension/src/background.ts +108 -33
  276. package/extension/src/cdp.ts +9 -9
  277. package/package.json +3 -2
  278. package/scripts/check-doc-coverage.sh +2 -0
  279. package/src/analysis.ts +170 -0
  280. package/src/browser/cdp.test.ts +66 -0
  281. package/src/browser/cdp.ts +59 -44
  282. package/src/browser/dom-snapshot.test.ts +42 -0
  283. package/src/browser/dom-snapshot.ts +56 -3
  284. package/src/browser/index.ts +2 -2
  285. package/src/browser/mcp.ts +2 -4
  286. package/src/browser/page.ts +34 -37
  287. package/src/browser/stealth.ts +24 -10
  288. package/src/browser.test.ts +2 -2
  289. package/src/build-manifest.test.ts +14 -0
  290. package/src/build-manifest.ts +13 -31
  291. package/src/cascade.ts +5 -3
  292. package/src/cli.ts +66 -34
  293. package/src/clis/_shared/desktop-commands.ts +121 -0
  294. package/src/clis/antigravity/serve.ts +6 -3
  295. package/src/clis/arxiv/search.ts +1 -1
  296. package/src/clis/bilibili/dynamic.test.ts +79 -0
  297. package/src/clis/bilibili/favorite.ts +5 -2
  298. package/src/clis/bilibili/following.ts +3 -2
  299. package/src/clis/bilibili/subtitle.ts +8 -7
  300. package/src/clis/bilibili/utils.ts +2 -2
  301. package/src/clis/boss/batchgreet.ts +1 -1
  302. package/src/clis/boss/chatlist.ts +1 -1
  303. package/src/clis/boss/chatmsg.ts +1 -1
  304. package/src/clis/boss/detail.ts +1 -1
  305. package/src/clis/boss/exchange.ts +1 -1
  306. package/src/clis/boss/greet.ts +1 -1
  307. package/src/clis/boss/invite.ts +1 -1
  308. package/src/clis/boss/joblist.ts +1 -1
  309. package/src/clis/boss/mark.ts +4 -3
  310. package/src/clis/boss/recommend.ts +1 -1
  311. package/src/clis/boss/resume.ts +1 -1
  312. package/src/clis/boss/search.ts +1 -1
  313. package/src/clis/boss/send.ts +5 -4
  314. package/src/clis/boss/stats.ts +1 -1
  315. package/src/clis/chatgpt/ask.ts +5 -0
  316. package/src/clis/chatgpt/new.ts +7 -2
  317. package/src/clis/chatgpt/read.ts +7 -2
  318. package/src/clis/chatgpt/send.ts +3 -2
  319. package/src/clis/chatgpt/status.ts +6 -1
  320. package/src/clis/chatwise/ask.ts +7 -2
  321. package/src/clis/chatwise/export.ts +2 -0
  322. package/src/clis/chatwise/history.ts +2 -0
  323. package/src/clis/chatwise/model.ts +7 -3
  324. package/src/clis/chatwise/new.ts +3 -20
  325. package/src/clis/chatwise/read.ts +2 -0
  326. package/src/clis/chatwise/screenshot.ts +3 -32
  327. package/src/clis/chatwise/send.ts +7 -2
  328. package/src/clis/chatwise/shared.ts +8 -0
  329. package/src/clis/chatwise/status.ts +3 -24
  330. package/src/clis/codex/ask.ts +5 -2
  331. package/src/clis/codex/dump.ts +2 -27
  332. package/src/clis/codex/new.ts +2 -28
  333. package/src/clis/codex/screenshot.ts +2 -32
  334. package/src/clis/codex/send.ts +5 -4
  335. package/src/clis/codex/status.ts +2 -24
  336. package/src/clis/cursor/ask.ts +2 -1
  337. package/src/clis/cursor/composer.ts +2 -1
  338. package/src/clis/cursor/dump.ts +2 -27
  339. package/src/clis/cursor/new.ts +2 -20
  340. package/src/clis/cursor/read.ts +2 -1
  341. package/src/clis/cursor/screenshot.ts +1 -36
  342. package/src/clis/cursor/send.ts +2 -1
  343. package/src/clis/cursor/status.ts +2 -22
  344. package/src/clis/dictionary/examples.yaml +25 -0
  345. package/src/clis/dictionary/search.yaml +27 -0
  346. package/src/clis/dictionary/synonyms.yaml +25 -0
  347. package/src/clis/douban/book-hot.ts +1 -1
  348. package/src/clis/douban/movie-hot.ts +1 -1
  349. package/src/clis/douban/search.ts +1 -1
  350. package/src/clis/douban/utils.ts +165 -1
  351. package/src/clis/doubao/ask.ts +1 -1
  352. package/src/clis/doubao/new.ts +1 -1
  353. package/src/clis/doubao/read.ts +1 -1
  354. package/src/clis/doubao/send.ts +1 -1
  355. package/src/clis/doubao/status.ts +1 -1
  356. package/src/clis/doubao-app/ask.ts +1 -1
  357. package/src/clis/doubao-app/new.ts +1 -1
  358. package/src/clis/doubao-app/read.ts +1 -1
  359. package/src/clis/doubao-app/send.ts +1 -1
  360. package/src/clis/grok/ask.test.ts +25 -0
  361. package/src/clis/grok/ask.ts +25 -12
  362. package/src/clis/jd/item.test.ts +35 -0
  363. package/src/clis/jd/item.ts +101 -0
  364. package/src/clis/jike/feed.ts +1 -1
  365. package/src/clis/jike/search.ts +1 -1
  366. package/src/clis/linkedin/search.ts +5 -4
  367. package/src/clis/linkedin/timeline.test.ts +99 -0
  368. package/src/clis/linkedin/timeline.ts +532 -0
  369. package/src/clis/medium/feed.ts +1 -1
  370. package/src/clis/medium/search.ts +1 -1
  371. package/src/clis/medium/user.ts +1 -1
  372. package/src/clis/medium/{shared.ts → utils.ts} +2 -1
  373. package/src/clis/pixiv/detail.yaml +49 -0
  374. package/src/clis/pixiv/download.test.ts +114 -0
  375. package/src/clis/pixiv/download.ts +91 -0
  376. package/src/clis/pixiv/illusts.test.ts +115 -0
  377. package/src/clis/pixiv/illusts.ts +78 -0
  378. package/src/clis/pixiv/ranking.yaml +53 -0
  379. package/src/clis/pixiv/search.test.ts +97 -0
  380. package/src/clis/pixiv/search.ts +53 -0
  381. package/src/clis/pixiv/test-utils.ts +29 -0
  382. package/src/clis/pixiv/user.yaml +46 -0
  383. package/src/clis/pixiv/utils.ts +62 -0
  384. package/src/clis/reddit/comment.ts +2 -1
  385. package/src/clis/reddit/read.test.ts +34 -0
  386. package/src/clis/reddit/read.ts +4 -3
  387. package/src/clis/reddit/save.ts +2 -1
  388. package/src/clis/reddit/saved.ts +6 -2
  389. package/src/clis/reddit/subscribe.ts +2 -1
  390. package/src/clis/reddit/upvote.ts +2 -1
  391. package/src/clis/reddit/upvoted.ts +6 -2
  392. package/src/clis/sinablog/article.ts +1 -1
  393. package/src/clis/sinablog/hot.ts +1 -1
  394. package/src/clis/sinablog/user.ts +1 -1
  395. package/src/clis/substack/feed.ts +1 -1
  396. package/src/clis/substack/publication.ts +1 -1
  397. package/src/clis/substack/search.ts +3 -2
  398. package/src/clis/substack/{shared.ts → utils.ts} +3 -2
  399. package/src/clis/tiktok/search.yaml +2 -1
  400. package/src/clis/twitter/accept.ts +2 -1
  401. package/src/clis/twitter/article.ts +3 -1
  402. package/src/clis/twitter/block.ts +2 -1
  403. package/src/clis/twitter/bookmark.ts +2 -1
  404. package/src/clis/twitter/bookmarks.ts +3 -2
  405. package/src/clis/twitter/delete.ts +2 -1
  406. package/src/clis/twitter/follow.ts +2 -1
  407. package/src/clis/twitter/followers.ts +3 -2
  408. package/src/clis/twitter/following.ts +3 -2
  409. package/src/clis/twitter/hide-reply.ts +2 -1
  410. package/src/clis/twitter/like.ts +2 -1
  411. package/src/clis/twitter/notifications.ts +2 -1
  412. package/src/clis/twitter/post.ts +2 -1
  413. package/src/clis/twitter/profile.ts +4 -2
  414. package/src/clis/twitter/reply-dm.ts +2 -1
  415. package/src/clis/twitter/reply.ts +2 -1
  416. package/src/clis/twitter/search.test.ts +113 -0
  417. package/src/clis/twitter/search.ts +38 -14
  418. package/src/clis/twitter/thread.ts +2 -2
  419. package/src/clis/twitter/timeline.ts +3 -2
  420. package/src/clis/twitter/trending.ts +3 -2
  421. package/src/clis/twitter/unblock.ts +2 -1
  422. package/src/clis/twitter/unbookmark.ts +2 -1
  423. package/src/clis/twitter/unfollow.ts +2 -1
  424. package/src/clis/v2ex/daily.ts +3 -2
  425. package/src/clis/v2ex/me.ts +3 -2
  426. package/src/clis/v2ex/notifications.ts +3 -4
  427. package/src/clis/web/read.ts +210 -0
  428. package/src/clis/xueqiu/danjuan-utils.test.ts +49 -0
  429. package/src/clis/xueqiu/danjuan-utils.ts +176 -0
  430. package/src/clis/xueqiu/fund-holdings.ts +32 -0
  431. package/src/clis/xueqiu/fund-snapshot.ts +27 -0
  432. package/src/clis/youtube/transcript.ts +5 -4
  433. package/src/clis/youtube/video.ts +3 -2
  434. package/src/daemon.ts +5 -4
  435. package/src/discovery.ts +12 -34
  436. package/src/doctor.ts +3 -2
  437. package/src/download/index.test.ts +93 -2
  438. package/src/download/index.ts +44 -23
  439. package/src/download/media-download.ts +5 -3
  440. package/src/engine.test.ts +84 -3
  441. package/src/execution.ts +62 -46
  442. package/src/explore.ts +21 -90
  443. package/src/external-clis.yaml +0 -8
  444. package/src/external.test.ts +9 -0
  445. package/src/external.ts +12 -10
  446. package/src/generate.ts +4 -41
  447. package/src/hooks.test.ts +126 -0
  448. package/src/hooks.ts +90 -0
  449. package/src/interceptor.ts +73 -23
  450. package/src/main.ts +2 -0
  451. package/src/output.ts +14 -6
  452. package/src/pipeline/executor.ts +1 -1
  453. package/src/pipeline/steps/browser.ts +1 -3
  454. package/src/pipeline/steps/download.test.ts +136 -0
  455. package/src/pipeline/steps/download.ts +47 -34
  456. package/src/pipeline/steps/fetch.test.ts +179 -0
  457. package/src/pipeline/steps/fetch.ts +39 -23
  458. package/src/pipeline/steps/transform.ts +2 -6
  459. package/src/pipeline/template.test.ts +28 -0
  460. package/src/pipeline/template.ts +67 -79
  461. package/src/pipeline/transform.test.ts +20 -0
  462. package/src/plugin.test.ts +251 -3
  463. package/src/plugin.ts +265 -21
  464. package/src/record.ts +12 -84
  465. package/src/registry-api.ts +2 -0
  466. package/src/registry.ts +7 -4
  467. package/src/runtime.ts +14 -4
  468. package/src/snapshotFormatter.ts +43 -121
  469. package/src/utils.ts +39 -0
  470. package/src/validate.ts +3 -5
  471. package/src/yaml-schema.ts +28 -0
  472. package/tests/e2e/browser-auth.test.ts +25 -0
  473. package/tests/e2e/plugin-management.test.ts +137 -0
  474. package/tests/e2e/public-commands.test.ts +34 -1
  475. package/vitest.config.ts +19 -1
  476. package/.github/workflows/pkg-pr-new.yml +0 -30
  477. package/dist/clis/douban/shared.d.ts +0 -4
  478. package/dist/clis/douban/shared.js +0 -155
  479. package/src/clis/douban/shared.ts +0 -165
  480. /package/dist/clis/boss/{common.d.ts → utils.d.ts} +0 -0
  481. /package/dist/clis/boss/{common.js → utils.js} +0 -0
  482. /package/dist/clis/doubao/{common.d.ts → utils.d.ts} +0 -0
  483. /package/dist/clis/doubao/{common.js → utils.js} +0 -0
  484. /package/dist/clis/doubao-app/{common.d.ts → utils.d.ts} +0 -0
  485. /package/dist/clis/doubao-app/{common.js → utils.js} +0 -0
  486. /package/dist/clis/jike/{shared.d.ts → utils.d.ts} +0 -0
  487. /package/dist/clis/jike/{shared.js → utils.js} +0 -0
  488. /package/dist/clis/medium/{shared.d.ts → utils.d.ts} +0 -0
  489. /package/dist/clis/sinablog/{shared.d.ts → utils.d.ts} +0 -0
  490. /package/dist/clis/sinablog/{shared.js → utils.js} +0 -0
  491. /package/dist/clis/substack/{shared.d.ts → utils.d.ts} +0 -0
  492. /package/src/clis/boss/{common.ts → utils.ts} +0 -0
  493. /package/src/clis/doubao/{common.ts → utils.ts} +0 -0
  494. /package/src/clis/doubao-app/{common.ts → utils.ts} +0 -0
  495. /package/src/clis/jike/{shared.ts → utils.ts} +0 -0
  496. /package/src/clis/sinablog/{shared.ts → utils.ts} +0 -0
@@ -0,0 +1,136 @@
1
+ import { beforeEach, describe, expect, it, vi } from 'vitest';
2
+ import * as os from 'node:os';
3
+ import * as path from 'node:path';
4
+ import type { IPage } from '../../types.js';
5
+
6
+ const { mockHttpDownload, mockYtdlpDownload, mockExportCookiesToNetscape } = vi.hoisted(() => ({
7
+ mockHttpDownload: vi.fn(),
8
+ mockYtdlpDownload: vi.fn(),
9
+ mockExportCookiesToNetscape: vi.fn(),
10
+ }));
11
+
12
+ vi.mock('../../download/index.js', async () => {
13
+ const actual = await vi.importActual<typeof import('../../download/index.js')>('../../download/index.js');
14
+ return {
15
+ ...actual,
16
+ httpDownload: mockHttpDownload,
17
+ ytdlpDownload: mockYtdlpDownload,
18
+ exportCookiesToNetscape: mockExportCookiesToNetscape,
19
+ };
20
+ });
21
+
22
+ import { stepDownload } from './download.js';
23
+
24
+ function createMockPage(getCookies: IPage['getCookies']): IPage {
25
+ return {
26
+ goto: vi.fn(),
27
+ evaluate: vi.fn().mockResolvedValue(null),
28
+ getCookies,
29
+ snapshot: vi.fn().mockResolvedValue(''),
30
+ click: vi.fn(),
31
+ typeText: vi.fn(),
32
+ pressKey: vi.fn(),
33
+ scrollTo: vi.fn(),
34
+ getFormState: vi.fn().mockResolvedValue({}),
35
+ wait: vi.fn(),
36
+ tabs: vi.fn().mockResolvedValue([]),
37
+ closeTab: vi.fn(),
38
+ newTab: vi.fn(),
39
+ selectTab: vi.fn(),
40
+ networkRequests: vi.fn().mockResolvedValue([]),
41
+ consoleMessages: vi.fn().mockResolvedValue([]),
42
+ scroll: vi.fn(),
43
+ autoScroll: vi.fn(),
44
+ installInterceptor: vi.fn(),
45
+ getInterceptedRequests: vi.fn().mockResolvedValue([]),
46
+ screenshot: vi.fn().mockResolvedValue(''),
47
+ };
48
+ }
49
+
50
+ describe('stepDownload', () => {
51
+ beforeEach(() => {
52
+ mockHttpDownload.mockReset();
53
+ mockHttpDownload.mockResolvedValue({ success: true, size: 2 });
54
+ mockYtdlpDownload.mockReset();
55
+ mockYtdlpDownload.mockResolvedValue({ success: true, size: 2 });
56
+ mockExportCookiesToNetscape.mockReset();
57
+ });
58
+
59
+ it('scopes browser cookies to each direct-download target domain', async () => {
60
+ const page = createMockPage(vi.fn().mockImplementation(async (opts?: { domain?: string }) => {
61
+ const domain = opts?.domain ?? 'unknown';
62
+ return [{ name: 'sid', value: domain, domain }];
63
+ }));
64
+
65
+ await stepDownload(
66
+ page,
67
+ {
68
+ url: '${{ item.url }}',
69
+ dir: path.join(os.tmpdir(), 'opencli-download-test'),
70
+ filename: '${{ index }}.txt',
71
+ progress: false,
72
+ concurrency: 1,
73
+ },
74
+ [
75
+ { url: 'https://a.example/file-1.txt' },
76
+ { url: 'https://b.example/file-2.txt' },
77
+ ],
78
+ {},
79
+ );
80
+
81
+ expect(mockHttpDownload).toHaveBeenNthCalledWith(
82
+ 1,
83
+ 'https://a.example/file-1.txt',
84
+ path.join(os.tmpdir(), 'opencli-download-test', '0.txt'),
85
+ expect.objectContaining({ cookies: 'sid=a.example' }),
86
+ );
87
+ expect(mockHttpDownload).toHaveBeenNthCalledWith(
88
+ 2,
89
+ 'https://b.example/file-2.txt',
90
+ path.join(os.tmpdir(), 'opencli-download-test', '1.txt'),
91
+ expect.objectContaining({ cookies: 'sid=b.example' }),
92
+ );
93
+ });
94
+
95
+ it('builds yt-dlp cookies from all target domains instead of only the first item', async () => {
96
+ const getCookies = vi.fn().mockImplementation(async (opts?: { domain?: string }) => {
97
+ const domain = opts?.domain ?? 'unknown';
98
+ return [{
99
+ name: `sid-${domain}`,
100
+ value: domain,
101
+ domain,
102
+ path: '/',
103
+ secure: false,
104
+ httpOnly: false,
105
+ }];
106
+ });
107
+ const page = createMockPage(getCookies);
108
+
109
+ await stepDownload(
110
+ page,
111
+ {
112
+ url: '${{ item.url }}',
113
+ dir: '/tmp/opencli-download-test',
114
+ filename: '${{ index }}.mp4',
115
+ progress: false,
116
+ concurrency: 1,
117
+ },
118
+ [
119
+ { url: 'https://www.youtube.com/watch?v=one' },
120
+ { url: 'https://www.bilibili.com/video/BV1xx411c7mD' },
121
+ ],
122
+ {},
123
+ );
124
+
125
+ expect(getCookies).toHaveBeenCalledWith({ domain: 'www.youtube.com' });
126
+ expect(getCookies).toHaveBeenCalledWith({ domain: 'www.bilibili.com' });
127
+ expect(mockExportCookiesToNetscape).toHaveBeenCalledWith(
128
+ expect.arrayContaining([
129
+ expect.objectContaining({ name: 'sid-www.youtube.com', domain: 'www.youtube.com' }),
130
+ expect.objectContaining({ name: 'sid-www.bilibili.com', domain: 'www.bilibili.com' }),
131
+ ]),
132
+ expect.any(String),
133
+ );
134
+ expect(mockYtdlpDownload).toHaveBeenCalledTimes(2);
135
+ });
136
+ });
@@ -13,6 +13,7 @@ import * as path from 'node:path';
13
13
  import * as os from 'node:os';
14
14
  import type { IPage } from '../../types.js';
15
15
  import { render } from '../template.js';
16
+ import { getErrorMessage } from '../../errors.js';
16
17
  import {
17
18
  httpDownload,
18
19
  ytdlpDownload,
@@ -26,6 +27,7 @@ import {
26
27
  formatCookieHeader,
27
28
  } from '../../download/index.js';
28
29
  import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
30
+ import { mapConcurrent } from '../../utils.js';
29
31
 
30
32
  export interface DownloadResult {
31
33
  status: 'success' | 'skipped' | 'failed';
@@ -35,35 +37,14 @@ export interface DownloadResult {
35
37
  duration?: number;
36
38
  }
37
39
 
38
- /**
39
- * Simple async concurrency limiter for downloads.
40
- */
41
- async function mapConcurrent<T, R>(
42
- items: T[],
43
- limit: number,
44
- fn: (item: T, index: number) => Promise<R>,
45
- ): Promise<R[]> {
46
- const results: R[] = new Array(items.length);
47
- let index = 0;
48
-
49
- async function worker() {
50
- while (index < items.length) {
51
- const i = index++;
52
- results[i] = await fn(items[i], i);
53
- }
54
- }
55
40
 
56
- const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
57
- await Promise.all(workers);
58
- return results;
59
- }
60
41
 
61
42
  /**
62
43
  * Extract cookies from browser page.
63
44
  */
64
- async function extractBrowserCookies(page: IPage, domain?: string): Promise<string> {
45
+ async function extractBrowserCookies(page: IPage, domain: string): Promise<string> {
65
46
  try {
66
- const cookies = await page.getCookies(domain ? { domain } : {});
47
+ const cookies = await page.getCookies({ domain });
67
48
  return formatCookieHeader(cookies);
68
49
  } catch {
69
50
  return '';
@@ -94,6 +75,16 @@ async function extractCookiesArray(
94
75
  }
95
76
  }
96
77
 
78
+ function dedupeCookies(
79
+ cookies: Array<{ name: string; value: string; domain: string; path: string; secure: boolean; httpOnly: boolean }>,
80
+ ): Array<{ name: string; value: string; domain: string; path: string; secure: boolean; httpOnly: boolean }> {
81
+ const deduped = new Map<string, { name: string; value: string; domain: string; path: string; secure: boolean; httpOnly: boolean }>();
82
+ for (const cookie of cookies) {
83
+ deduped.set(`${cookie.domain}\t${cookie.path}\t${cookie.name}`, cookie);
84
+ }
85
+ return [...deduped.values()];
86
+ }
87
+
97
88
  /**
98
89
  * Download step handler for YAML pipelines.
99
90
  *
@@ -143,23 +134,29 @@ export async function stepDownload(
143
134
  // Create progress tracker
144
135
  const tracker = new DownloadProgressTracker(items.length, showProgress);
145
136
 
146
- // Extract cookies if browser is available
147
- let cookies = '';
137
+ // Cache cookie lookups per domain so mixed-domain batches stay isolated without repeated browser calls.
138
+ const cookieHeaderCache = new Map<string, Promise<string>>();
148
139
  let cookiesFile: string | undefined;
149
140
 
150
141
  if (page) {
151
- cookies = await extractBrowserCookies(page);
152
-
153
142
  // For yt-dlp, we need to export cookies to Netscape format
154
143
  if (useYtdlp || items.some((item, index) => {
155
144
  const url = String(render(urlTemplate, { args, data, item, index }));
156
145
  return requiresYtdlp(url);
157
146
  })) {
158
147
  try {
159
- // Try to get domain from first URL
160
- const firstUrl = String(render(urlTemplate, { args, data, item: items[0], index: 0 }));
161
- const domain = new URL(firstUrl).hostname;
162
- const cookiesArray = await extractCookiesArray(page, domain);
148
+ const ytdlpDomains = [...new Set(items.flatMap((item, index) => {
149
+ const url = String(render(urlTemplate, { args, data, item, index }));
150
+ if (!useYtdlp && !requiresYtdlp(url)) return [];
151
+ try {
152
+ return [new URL(url).hostname];
153
+ } catch {
154
+ return [];
155
+ }
156
+ }))];
157
+ const cookiesArray = dedupeCookies(
158
+ (await Promise.all(ytdlpDomains.map((domain) => extractCookiesArray(page, domain)))).flat(),
159
+ );
163
160
 
164
161
  if (cookiesArray.length > 0) {
165
162
  const tempDir = getTempDir();
@@ -254,6 +251,21 @@ export async function stepDownload(
254
251
  }
255
252
  } else {
256
253
  // Direct HTTP download
254
+ let cookies = '';
255
+ if (page) {
256
+ try {
257
+ const targetDomain = new URL(url).hostname;
258
+ let cookiePromise = cookieHeaderCache.get(targetDomain);
259
+ if (!cookiePromise) {
260
+ cookiePromise = extractBrowserCookies(page, targetDomain);
261
+ cookieHeaderCache.set(targetDomain, cookiePromise);
262
+ }
263
+ cookies = await cookiePromise;
264
+ } catch {
265
+ cookies = '';
266
+ }
267
+ }
268
+
257
269
  result = await httpDownload(url, destPath, {
258
270
  cookies,
259
271
  timeout,
@@ -268,10 +280,11 @@ export async function stepDownload(
268
280
  progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
269
281
  }
270
282
  }
271
- } catch (err: any) {
272
- result = { success: false, size: 0, error: err.message };
283
+ } catch (err) {
284
+ const msg = getErrorMessage(err);
285
+ result = { success: false, size: 0, error: msg };
273
286
  if (progressBar) {
274
- progressBar.fail(err.message);
287
+ progressBar.fail(msg);
275
288
  }
276
289
  }
277
290
 
@@ -0,0 +1,179 @@
1
+ import { afterEach, describe, expect, it, vi } from 'vitest';
2
+ import { CliError } from '../../errors.js';
3
+ import type { IPage } from '../../types.js';
4
+ import { stepFetch } from './fetch.js';
5
+
6
+ afterEach(() => {
7
+ vi.restoreAllMocks();
8
+ vi.unstubAllGlobals();
9
+ });
10
+
11
+ describe('stepFetch', () => {
12
+ // W1 + W4: non-browser single fetch throws CliError with FETCH_ERROR code and full message
13
+ it('throws CliError with FETCH_ERROR code on non-ok responses without a browser session', async () => {
14
+ const jsonMock = vi.fn().mockResolvedValue({ error: 'rate limited' });
15
+ const fetchMock = vi.fn().mockResolvedValue({
16
+ ok: false,
17
+ status: 429,
18
+ statusText: 'Too Many Requests',
19
+ json: jsonMock,
20
+ });
21
+ vi.stubGlobal('fetch', fetchMock);
22
+
23
+ const err = await stepFetch(null, { url: 'https://api.example.com/items' }, null, {}).catch((e: unknown) => e);
24
+ expect(err).toBeInstanceOf(CliError);
25
+ expect((err as CliError).code).toBe('FETCH_ERROR');
26
+ expect((err as CliError).message).toBe('HTTP 429 Too Many Requests from https://api.example.com/items');
27
+ expect(jsonMock).not.toHaveBeenCalled();
28
+ });
29
+
30
+ // W1 + W3: browser single fetch returns error status from evaluate, outer code throws CliError
31
+ it('throws CliError with FETCH_ERROR code on non-ok responses inside the browser session', async () => {
32
+ const jsonMock = vi.fn().mockResolvedValue({ error: 'auth required' });
33
+ const fetchMock = vi.fn().mockResolvedValue({
34
+ ok: false,
35
+ status: 401,
36
+ statusText: 'Unauthorized',
37
+ json: jsonMock,
38
+ });
39
+ vi.stubGlobal('fetch', fetchMock);
40
+
41
+ // Simulate real CDP behavior: evaluate returns a value, errors are thrown outside
42
+ const page = {
43
+ evaluate: vi.fn(async (js: string) => Function(`return (${js})`)()()),
44
+ } as unknown as IPage;
45
+
46
+ const err = await stepFetch(page, { url: 'https://api.example.com/items' }, null, {}).catch((e: unknown) => e);
47
+ expect(err).toBeInstanceOf(CliError);
48
+ expect((err as CliError).code).toBe('FETCH_ERROR');
49
+ expect((err as CliError).message).toBe('HTTP 401 Unauthorized from https://api.example.com/items');
50
+ expect(jsonMock).not.toHaveBeenCalled();
51
+ });
52
+
53
+ it('returns per-item HTTP errors for batch fetches without a browser session', async () => {
54
+ const jsonMock = vi.fn().mockResolvedValue({ error: 'upstream unavailable' });
55
+ const fetchMock = vi.fn().mockResolvedValue({
56
+ ok: false,
57
+ status: 503,
58
+ statusText: 'Service Unavailable',
59
+ json: jsonMock,
60
+ });
61
+ vi.stubGlobal('fetch', fetchMock);
62
+
63
+ await expect(stepFetch(
64
+ null,
65
+ { url: 'https://api.example.com/items/${{ item.id }}' },
66
+ [{ id: 1 }],
67
+ {},
68
+ )).resolves.toEqual([
69
+ { error: 'HTTP 503 Service Unavailable from https://api.example.com/items/1' },
70
+ ]);
71
+ expect(jsonMock).not.toHaveBeenCalled();
72
+ });
73
+
74
+ it('returns per-item HTTP errors for batch browser fetches', async () => {
75
+ const jsonMock = vi.fn().mockResolvedValue({ error: 'upstream unavailable' });
76
+ const fetchMock = vi.fn().mockResolvedValue({
77
+ ok: false,
78
+ status: 503,
79
+ statusText: 'Service Unavailable',
80
+ json: jsonMock,
81
+ });
82
+ vi.stubGlobal('fetch', fetchMock);
83
+
84
+ const page = {
85
+ evaluate: vi.fn(async (js: string) => Function(`return (${js})`)()()),
86
+ } as unknown as IPage;
87
+
88
+ await expect(stepFetch(
89
+ page,
90
+ { url: 'https://api.example.com/items/${{ item.id }}' },
91
+ [{ id: 1 }],
92
+ {},
93
+ )).resolves.toEqual([
94
+ { error: 'HTTP 503 Service Unavailable from https://api.example.com/items/1' },
95
+ ]);
96
+ expect(jsonMock).not.toHaveBeenCalled();
97
+ });
98
+
99
+ it('stringifies non-Error batch browser failures consistently', async () => {
100
+ vi.stubGlobal('fetch', vi.fn().mockRejectedValue('socket hang up'));
101
+
102
+ const page = {
103
+ evaluate: vi.fn(async (js: string) => Function(`return (${js})`)()()),
104
+ } as unknown as IPage;
105
+
106
+ await expect(stepFetch(
107
+ page,
108
+ { url: 'https://api.example.com/items/${{ item.id }}' },
109
+ [{ id: 1 }],
110
+ {},
111
+ )).resolves.toEqual([
112
+ { error: 'socket hang up' },
113
+ ]);
114
+ });
115
+
116
+ it('stringifies non-Error batch non-browser failures consistently', async () => {
117
+ vi.stubGlobal('fetch', vi.fn().mockRejectedValue('socket hang up'));
118
+
119
+ await expect(stepFetch(
120
+ null,
121
+ { url: 'https://api.example.com/items/${{ item.id }}' },
122
+ [{ id: 1 }],
123
+ {},
124
+ )).resolves.toEqual([
125
+ { error: 'socket hang up' },
126
+ ]);
127
+ });
128
+
129
+ // W2: batch item failures emit a warning log
130
+ it('logs a warning for each failed batch item in non-browser mode', async () => {
131
+ const { log } = await import('../../logger.js');
132
+ const warnSpy = vi.spyOn(log, 'warn');
133
+
134
+ vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
135
+ ok: false,
136
+ status: 503,
137
+ statusText: 'Service Unavailable',
138
+ json: vi.fn(),
139
+ }));
140
+
141
+ await stepFetch(
142
+ null,
143
+ { url: 'https://api.example.com/items/${{ item.id }}' },
144
+ [{ id: 1 }, { id: 2 }],
145
+ {},
146
+ );
147
+
148
+ expect(warnSpy).toHaveBeenCalledTimes(2);
149
+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/1'));
150
+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/2'));
151
+ });
152
+
153
+ it('logs a warning for each failed batch item in browser mode', async () => {
154
+ const { log } = await import('../../logger.js');
155
+ const warnSpy = vi.spyOn(log, 'warn');
156
+
157
+ vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
158
+ ok: false,
159
+ status: 502,
160
+ statusText: 'Bad Gateway',
161
+ json: vi.fn(),
162
+ }));
163
+
164
+ const page = {
165
+ evaluate: vi.fn(async (js: string) => Function(`return (${js})`)()()),
166
+ } as unknown as IPage;
167
+
168
+ await stepFetch(
169
+ page,
170
+ { url: 'https://api.example.com/items/${{ item.id }}' },
171
+ [{ id: 1 }, { id: 2 }],
172
+ {},
173
+ );
174
+
175
+ expect(warnSpy).toHaveBeenCalledTimes(2);
176
+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/1'));
177
+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/2'));
178
+ });
179
+ });
@@ -2,29 +2,14 @@
2
2
  * Pipeline step: fetch — HTTP API requests.
3
3
  */
4
4
 
5
+ import { CliError, getErrorMessage } from '../../errors.js';
6
+ import { log } from '../../logger.js';
5
7
  import type { IPage } from '../../types.js';
6
8
  import { render } from '../template.js';
7
9
 
8
- function isRecord(value: unknown): value is Record<string, unknown> {
9
- return typeof value === 'object' && value !== null && !Array.isArray(value);
10
- }
11
-
12
- /** Simple async concurrency limiter */
13
- async function mapConcurrent<T, R>(items: T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]> {
14
- const results: R[] = new Array(items.length);
15
- let index = 0;
10
+ import { isRecord, mapConcurrent } from '../../utils.js';
16
11
 
17
- async function worker() {
18
- while (index < items.length) {
19
- const i = index++;
20
- results[i] = await fn(items[i], i);
21
- }
22
- }
23
12
 
24
- const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
25
- await Promise.all(workers);
26
- return results;
27
- }
28
13
 
29
14
  /** Single URL fetch helper */
30
15
  async function fetchSingle(
@@ -45,20 +30,33 @@ async function fetchSingle(
45
30
 
46
31
  if (page === null) {
47
32
  const resp = await fetch(finalUrl, { method: method.toUpperCase(), headers: renderedHeaders });
33
+ if (!resp.ok) {
34
+ throw new CliError('FETCH_ERROR', `HTTP ${resp.status} ${resp.statusText} from ${finalUrl}`);
35
+ }
48
36
  return resp.json();
49
37
  }
50
38
 
51
39
  const headersJs = JSON.stringify(renderedHeaders);
52
40
  const urlJs = JSON.stringify(finalUrl);
53
41
  const methodJs = JSON.stringify(method.toUpperCase());
54
- return page.evaluate(`
42
+ // Return error status instead of throwing inside evaluate to avoid CDP wrapper
43
+ // rewriting the message (CDP prepends "Evaluate error: " to thrown errors).
44
+ const result = await page.evaluate(`
55
45
  async () => {
56
46
  const resp = await fetch(${urlJs}, {
57
47
  method: ${methodJs}, headers: ${headersJs}, credentials: "include"
58
48
  });
49
+ if (!resp.ok) {
50
+ return { __httpError: resp.status, statusText: resp.statusText };
51
+ }
59
52
  return await resp.json();
60
53
  }
61
54
  `);
55
+ if (result && typeof result === 'object' && '__httpError' in result) {
56
+ const { __httpError: status, statusText } = result as { __httpError: number; statusText: string };
57
+ throw new CliError('FETCH_ERROR', `HTTP ${status} ${statusText} from ${finalUrl}`);
58
+ }
59
+ return result;
62
60
  }
63
61
 
64
62
  /**
@@ -72,10 +70,11 @@ async function fetchBatchInBrowser(
72
70
  ): Promise<unknown[]> {
73
71
  const headersJs = JSON.stringify(headers);
74
72
  const urlsJs = JSON.stringify(urls);
73
+ const methodJs = JSON.stringify(method);
75
74
  return (await page.evaluate(`
76
75
  async () => {
77
76
  const urls = ${urlsJs};
78
- const method = "${method}";
77
+ const method = ${methodJs};
79
78
  const headers = ${headersJs};
80
79
  const concurrency = ${concurrency};
81
80
 
@@ -87,9 +86,13 @@ async function fetchBatchInBrowser(
87
86
  const i = idx++;
88
87
  try {
89
88
  const resp = await fetch(urls[i], { method, headers, credentials: "include" });
89
+ if (!resp.ok) {
90
+ throw new Error('HTTP ' + resp.status + ' ' + resp.statusText + ' from ' + urls[i]);
91
+ }
90
92
  results[i] = await resp.json();
91
93
  } catch (e) {
92
- results[i] = { error: e.message };
94
+ results[i] = { error: e instanceof Error ? e.message : String(e) };
95
+ // Note: getErrorMessage() is a Node.js utility — can't use it inside evaluate()
93
96
  }
94
97
  }
95
98
  }
@@ -130,13 +133,26 @@ export async function stepFetch(page: IPage | null, params: unknown, data: unkno
130
133
 
131
134
  // BATCH IPC: if browser is available, batch all fetches into a single evaluate() call
132
135
  if (page !== null) {
133
- return fetchBatchInBrowser(page, urls, method.toUpperCase(), renderedHeaders, concurrency);
136
+ const results = await fetchBatchInBrowser(page, urls, method.toUpperCase(), renderedHeaders, concurrency);
137
+ for (let i = 0; i < results.length; i++) {
138
+ const r = results[i];
139
+ if (r && typeof r === 'object' && 'error' in r) {
140
+ log.warn(`Batch fetch failed for ${urls[i]}: ${(r as { error: string }).error}`);
141
+ }
142
+ }
143
+ return results;
134
144
  }
135
145
 
136
146
  // Non-browser: use concurrent pool (already optimized)
137
147
  return mapConcurrent(data, concurrency, async (item, index) => {
138
148
  const itemUrl = String(render(urlTemplate, { args, data, item, index }));
139
- return fetchSingle(null, itemUrl, method, queryParams, headers, args, data);
149
+ try {
150
+ return await fetchSingle(null, itemUrl, method, queryParams, headers, args, data);
151
+ } catch (error) {
152
+ const message = getErrorMessage(error);
153
+ log.warn(`Batch fetch failed for ${itemUrl}: ${message}`);
154
+ return { error: message };
155
+ }
140
156
  });
141
157
  }
142
158
  const url = render(urlOrObj, { args, data });
@@ -5,9 +5,7 @@
5
5
  import type { IPage } from '../../types.js';
6
6
  import { render, evalExpr } from '../template.js';
7
7
 
8
- function isRecord(value: unknown): value is Record<string, unknown> {
9
- return typeof value === 'object' && value !== null && !Array.isArray(value);
10
- }
8
+ import { isRecord } from '../../utils.js';
11
9
 
12
10
  export async function stepSelect(_page: IPage | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown> {
13
11
  const pathStr = String(render(params, { args, data }));
@@ -62,9 +60,7 @@ export async function stepSort(_page: IPage | null, params: unknown, data: unkno
62
60
  return [...data].sort((a, b) => {
63
61
  const left = isRecord(a) ? a[key] : undefined;
64
62
  const right = isRecord(b) ? b[key] : undefined;
65
- const va = left ?? '';
66
- const vb = right ?? '';
67
- const cmp = va < vb ? -1 : va > vb ? 1 : 0;
63
+ const cmp = String(left ?? '').localeCompare(String(right ?? ''), undefined, { numeric: true });
68
64
  return reverse ? -cmp : cmp;
69
65
  });
70
66
  }
@@ -54,6 +54,31 @@ describe('evalExpr', () => {
54
54
  it('evaluates || with truthy left', () => {
55
55
  expect(evalExpr("item.name || 'N/A'", { item: { name: 'Alice' } })).toBe('Alice');
56
56
  });
57
+ it('evaluates chained || fallback (issue #303)', () => {
58
+ // When first two are falsy, should evaluate through to the string literal
59
+ expect(evalExpr("item.a || item.b || 'default'", { item: {} })).toBe('default');
60
+ });
61
+ it('evaluates chained || with middle value truthy', () => {
62
+ expect(evalExpr("item.a || item.b || 'default'", { item: { b: 'middle' } })).toBe('middle');
63
+ });
64
+ it('evaluates chained || with first value truthy', () => {
65
+ expect(evalExpr("item.a || item.b || 'default'", { item: { a: 'first', b: 'middle' } })).toBe('first');
66
+ });
67
+ it('evaluates || with 0 as falsy left (JS semantics)', () => {
68
+ expect(evalExpr("item.count || 'N/A'", { item: { count: 0 } })).toBe('N/A');
69
+ });
70
+ it('evaluates || with empty string as falsy left', () => {
71
+ expect(evalExpr("item.name || 'unknown'", { item: { name: '' } })).toBe('unknown');
72
+ });
73
+ it('evaluates || with numeric fallback returning number type', () => {
74
+ expect(evalExpr('item.a || 42', { item: {} })).toBe(42);
75
+ });
76
+ it('evaluates 4-way chained ||', () => {
77
+ expect(evalExpr("item.a || item.b || item.c || 'last'", { item: { c: 'third' } })).toBe('third');
78
+ });
79
+ it('handles || combined with pipe filter', () => {
80
+ expect(evalExpr("item.a || item.b | upper", { item: { b: 'hello' } })).toBe('HELLO');
81
+ });
57
82
  it('resolves simple path', () => {
58
83
  expect(evalExpr('item.title', { item: { title: 'Test' } })).toBe('Test');
59
84
  });
@@ -66,6 +91,9 @@ describe('evalExpr', () => {
66
91
  it('evaluates method calls on values', () => {
67
92
  expect(evalExpr("args.username.startsWith('@') ? args.username : '@' + args.username", { args: { username: 'alice' } })).toBe('@alice');
68
93
  });
94
+ it('rejects constructor-based sandbox escapes', () => {
95
+ expect(evalExpr("args['cons' + 'tructor']['constructor']('return process')()", { args: {} })).toBeUndefined();
96
+ });
69
97
  it('applies join filter', () => {
70
98
  expect(evalExpr('item.tags | join(,)', { item: { tags: ['a', 'b', 'c'] } })).toBe('a,b,c');
71
99
  });