@jackwener/opencli 0.9.6 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (307) hide show
  1. package/.github/ISSUE_TEMPLATE/bug_report.yml +83 -0
  2. package/.github/ISSUE_TEMPLATE/config.yml +8 -0
  3. package/.github/ISSUE_TEMPLATE/feature_request.yml +42 -0
  4. package/.github/ISSUE_TEMPLATE/new_site_adapter.yml +57 -0
  5. package/.github/dependabot.yml +27 -0
  6. package/.github/pull_request_template.md +24 -0
  7. package/.github/workflows/ci.yml +14 -8
  8. package/.github/workflows/e2e-headed.yml +6 -2
  9. package/.github/workflows/pkg-pr-new.yml +2 -2
  10. package/.github/workflows/release-please.yml +25 -0
  11. package/.github/workflows/release.yml +2 -2
  12. package/.github/workflows/security.yml +36 -0
  13. package/CDP.md +1 -1
  14. package/CDP.zh-CN.md +1 -1
  15. package/CLI-ELECTRON.md +89 -36
  16. package/CLI-EXPLORER.md +4 -4
  17. package/CONTRIBUTING.md +167 -0
  18. package/README.md +113 -89
  19. package/README.zh-CN.md +114 -91
  20. package/SKILL.md +10 -8
  21. package/TESTING.md +7 -7
  22. package/dist/browser/daemon-client.d.ts +37 -0
  23. package/dist/browser/daemon-client.js +82 -0
  24. package/dist/browser/discover.d.ts +11 -34
  25. package/dist/browser/discover.js +15 -190
  26. package/dist/browser/errors.d.ts +6 -20
  27. package/dist/browser/errors.js +24 -63
  28. package/dist/browser/index.d.ts +2 -11
  29. package/dist/browser/index.js +5 -11
  30. package/dist/browser/mcp.d.ts +9 -18
  31. package/dist/browser/mcp.js +70 -284
  32. package/dist/browser/page.d.ts +28 -6
  33. package/dist/browser/page.js +210 -85
  34. package/dist/browser.test.js +4 -202
  35. package/dist/build-manifest.d.ts +26 -0
  36. package/dist/build-manifest.js +132 -60
  37. package/dist/build-manifest.test.d.ts +1 -0
  38. package/dist/build-manifest.test.js +26 -0
  39. package/dist/cli-manifest.json +1582 -29
  40. package/dist/clis/bilibili/download.d.ts +10 -0
  41. package/dist/clis/bilibili/download.js +135 -0
  42. package/dist/clis/chatwise/ask.d.ts +1 -0
  43. package/dist/clis/chatwise/ask.js +76 -0
  44. package/dist/clis/chatwise/export.d.ts +1 -0
  45. package/dist/clis/chatwise/export.js +46 -0
  46. package/dist/clis/chatwise/history.d.ts +1 -0
  47. package/dist/clis/chatwise/history.js +43 -0
  48. package/dist/clis/chatwise/model.d.ts +1 -0
  49. package/dist/clis/chatwise/model.js +81 -0
  50. package/dist/clis/chatwise/new.d.ts +1 -0
  51. package/dist/clis/chatwise/new.js +18 -0
  52. package/dist/clis/chatwise/read.d.ts +1 -0
  53. package/dist/clis/chatwise/read.js +39 -0
  54. package/dist/clis/chatwise/screenshot.d.ts +1 -0
  55. package/dist/clis/chatwise/screenshot.js +27 -0
  56. package/dist/clis/chatwise/send.d.ts +1 -0
  57. package/dist/clis/chatwise/send.js +45 -0
  58. package/dist/clis/chatwise/status.d.ts +1 -0
  59. package/dist/clis/chatwise/status.js +22 -0
  60. package/dist/clis/discord-app/channels.d.ts +1 -0
  61. package/dist/clis/discord-app/channels.js +45 -0
  62. package/dist/clis/discord-app/members.d.ts +1 -0
  63. package/dist/clis/discord-app/members.js +38 -0
  64. package/dist/clis/discord-app/read.d.ts +1 -0
  65. package/dist/clis/discord-app/read.js +45 -0
  66. package/dist/clis/discord-app/search.d.ts +1 -0
  67. package/dist/clis/discord-app/search.js +56 -0
  68. package/dist/clis/discord-app/send.d.ts +1 -0
  69. package/dist/clis/discord-app/send.js +27 -0
  70. package/dist/clis/discord-app/servers.d.ts +1 -0
  71. package/dist/clis/discord-app/servers.js +36 -0
  72. package/dist/clis/discord-app/status.d.ts +1 -0
  73. package/dist/clis/discord-app/status.js +16 -0
  74. package/dist/clis/feishu/new.d.ts +1 -0
  75. package/dist/clis/feishu/new.js +27 -0
  76. package/dist/clis/feishu/read.d.ts +1 -0
  77. package/dist/clis/feishu/read.js +40 -0
  78. package/dist/clis/feishu/search.d.ts +1 -0
  79. package/dist/clis/feishu/search.js +30 -0
  80. package/dist/clis/feishu/send.d.ts +1 -0
  81. package/dist/clis/feishu/send.js +39 -0
  82. package/dist/clis/feishu/status.d.ts +1 -0
  83. package/dist/clis/feishu/status.js +28 -0
  84. package/dist/clis/grok/ask.d.ts +1 -0
  85. package/dist/clis/grok/ask.js +82 -0
  86. package/dist/clis/grok/debug.d.ts +1 -0
  87. package/dist/clis/grok/debug.js +45 -0
  88. package/dist/clis/jimeng/generate.yaml +84 -0
  89. package/dist/clis/jimeng/history.yaml +47 -0
  90. package/dist/clis/linux-do/categories.yaml +41 -0
  91. package/dist/clis/linux-do/category.yaml +49 -0
  92. package/dist/clis/linux-do/hot.yaml +50 -0
  93. package/dist/clis/linux-do/latest.yaml +40 -0
  94. package/dist/clis/linux-do/search.yaml +45 -0
  95. package/dist/clis/linux-do/topic.yaml +38 -0
  96. package/dist/clis/neteasemusic/like.d.ts +1 -0
  97. package/dist/clis/neteasemusic/like.js +25 -0
  98. package/dist/clis/neteasemusic/lyrics.d.ts +1 -0
  99. package/dist/clis/neteasemusic/lyrics.js +47 -0
  100. package/dist/clis/neteasemusic/next.d.ts +1 -0
  101. package/dist/clis/neteasemusic/next.js +26 -0
  102. package/dist/clis/neteasemusic/play.d.ts +1 -0
  103. package/dist/clis/neteasemusic/play.js +26 -0
  104. package/dist/clis/neteasemusic/playing.d.ts +1 -0
  105. package/dist/clis/neteasemusic/playing.js +59 -0
  106. package/dist/clis/neteasemusic/playlist.d.ts +1 -0
  107. package/dist/clis/neteasemusic/playlist.js +46 -0
  108. package/dist/clis/neteasemusic/prev.d.ts +1 -0
  109. package/dist/clis/neteasemusic/prev.js +25 -0
  110. package/dist/clis/neteasemusic/search.d.ts +1 -0
  111. package/dist/clis/neteasemusic/search.js +52 -0
  112. package/dist/clis/neteasemusic/status.d.ts +1 -0
  113. package/dist/clis/neteasemusic/status.js +16 -0
  114. package/dist/clis/neteasemusic/volume.d.ts +1 -0
  115. package/dist/clis/neteasemusic/volume.js +54 -0
  116. package/dist/clis/notion/export.d.ts +1 -0
  117. package/dist/clis/notion/export.js +31 -0
  118. package/dist/clis/notion/favorites.d.ts +1 -0
  119. package/dist/clis/notion/favorites.js +84 -0
  120. package/dist/clis/notion/new.d.ts +1 -0
  121. package/dist/clis/notion/new.js +34 -0
  122. package/dist/clis/notion/read.d.ts +1 -0
  123. package/dist/clis/notion/read.js +30 -0
  124. package/dist/clis/notion/search.d.ts +1 -0
  125. package/dist/clis/notion/search.js +46 -0
  126. package/dist/clis/notion/sidebar.d.ts +1 -0
  127. package/dist/clis/notion/sidebar.js +41 -0
  128. package/dist/clis/notion/status.d.ts +1 -0
  129. package/dist/clis/notion/status.js +16 -0
  130. package/dist/clis/notion/write.d.ts +1 -0
  131. package/dist/clis/notion/write.js +40 -0
  132. package/dist/clis/twitter/download.d.ts +8 -0
  133. package/dist/clis/twitter/download.js +204 -0
  134. package/dist/clis/wechat/chats.d.ts +1 -0
  135. package/dist/clis/wechat/chats.js +28 -0
  136. package/dist/clis/wechat/contacts.d.ts +1 -0
  137. package/dist/clis/wechat/contacts.js +28 -0
  138. package/dist/clis/wechat/read.d.ts +1 -0
  139. package/dist/clis/wechat/read.js +58 -0
  140. package/dist/clis/wechat/search.d.ts +1 -0
  141. package/dist/clis/wechat/search.js +31 -0
  142. package/dist/clis/wechat/send.d.ts +1 -0
  143. package/dist/clis/wechat/send.js +42 -0
  144. package/dist/clis/wechat/status.d.ts +1 -0
  145. package/dist/clis/wechat/status.js +29 -0
  146. package/dist/clis/xiaohongshu/creator-note-detail.d.ts +10 -0
  147. package/dist/clis/xiaohongshu/creator-note-detail.js +88 -0
  148. package/dist/clis/xiaohongshu/creator-notes.d.ts +11 -0
  149. package/dist/clis/xiaohongshu/creator-notes.js +109 -0
  150. package/dist/clis/xiaohongshu/creator-profile.d.ts +10 -0
  151. package/dist/clis/xiaohongshu/creator-profile.js +54 -0
  152. package/dist/clis/xiaohongshu/creator-stats.d.ts +10 -0
  153. package/dist/clis/xiaohongshu/creator-stats.js +74 -0
  154. package/dist/clis/xiaohongshu/download.d.ts +7 -0
  155. package/dist/clis/xiaohongshu/download.js +155 -0
  156. package/dist/clis/xiaohongshu/search.js +1 -1
  157. package/dist/clis/xiaohongshu/user-helpers.d.ts +15 -0
  158. package/dist/clis/xiaohongshu/user-helpers.js +67 -0
  159. package/dist/clis/xiaohongshu/user-helpers.test.d.ts +1 -0
  160. package/dist/clis/xiaohongshu/user-helpers.test.js +81 -0
  161. package/dist/clis/xiaohongshu/user.js +46 -29
  162. package/dist/clis/zhihu/download.d.ts +11 -0
  163. package/dist/clis/zhihu/download.js +186 -0
  164. package/dist/clis/zhihu/download.test.d.ts +1 -0
  165. package/dist/clis/zhihu/download.test.js +10 -0
  166. package/dist/daemon.d.ts +13 -0
  167. package/dist/daemon.js +187 -0
  168. package/dist/doctor.d.ts +27 -61
  169. package/dist/doctor.js +70 -601
  170. package/dist/doctor.test.js +30 -170
  171. package/dist/download/index.d.ts +79 -0
  172. package/dist/download/index.js +325 -0
  173. package/dist/download/progress.d.ts +36 -0
  174. package/dist/download/progress.js +111 -0
  175. package/dist/engine.test.js +15 -0
  176. package/dist/main.js +22 -28
  177. package/dist/pipeline/executor.test.js +1 -0
  178. package/dist/pipeline/registry.js +2 -0
  179. package/dist/pipeline/steps/browser.js +2 -2
  180. package/dist/pipeline/steps/download.d.ts +34 -0
  181. package/dist/pipeline/steps/download.js +251 -0
  182. package/dist/pipeline/steps/intercept.js +1 -2
  183. package/dist/pipeline/template.js +28 -0
  184. package/dist/setup.d.ts +6 -0
  185. package/dist/setup.js +46 -160
  186. package/dist/types.d.ts +6 -0
  187. package/extension/icons/icon-128.png +0 -0
  188. package/extension/icons/icon-16.png +0 -0
  189. package/extension/icons/icon-32.png +0 -0
  190. package/extension/icons/icon-48.png +0 -0
  191. package/extension/manifest.json +31 -0
  192. package/extension/package.json +16 -0
  193. package/extension/src/background.ts +293 -0
  194. package/extension/src/cdp.ts +125 -0
  195. package/extension/src/protocol.ts +57 -0
  196. package/extension/store-assets/screenshot-1280x800.png +0 -0
  197. package/extension/tsconfig.json +15 -0
  198. package/extension/vite.config.ts +18 -0
  199. package/package.json +8 -7
  200. package/scripts/test-site.mjs +70 -0
  201. package/src/browser/daemon-client.ts +113 -0
  202. package/src/browser/discover.ts +18 -216
  203. package/src/browser/errors.ts +30 -100
  204. package/src/browser/index.ts +6 -12
  205. package/src/browser/mcp.ts +78 -278
  206. package/src/browser/page.ts +222 -88
  207. package/src/browser.test.ts +3 -210
  208. package/src/build-manifest.test.ts +28 -0
  209. package/src/build-manifest.ts +147 -57
  210. package/src/clis/bilibili/download.ts +161 -0
  211. package/src/clis/chatgpt/README.md +1 -1
  212. package/src/clis/chatgpt/README.zh-CN.md +1 -1
  213. package/src/clis/chatwise/README.md +38 -0
  214. package/src/clis/chatwise/README.zh-CN.md +38 -0
  215. package/src/clis/chatwise/ask.ts +87 -0
  216. package/src/clis/chatwise/export.ts +51 -0
  217. package/src/clis/chatwise/history.ts +47 -0
  218. package/src/clis/chatwise/model.ts +87 -0
  219. package/src/clis/chatwise/new.ts +21 -0
  220. package/src/clis/chatwise/read.ts +42 -0
  221. package/src/clis/chatwise/screenshot.ts +33 -0
  222. package/src/clis/chatwise/send.ts +50 -0
  223. package/src/clis/chatwise/status.ts +25 -0
  224. package/src/clis/discord-app/README.md +28 -0
  225. package/src/clis/discord-app/README.zh-CN.md +28 -0
  226. package/src/clis/discord-app/channels.ts +48 -0
  227. package/src/clis/discord-app/members.ts +41 -0
  228. package/src/clis/discord-app/read.ts +49 -0
  229. package/src/clis/discord-app/search.ts +64 -0
  230. package/src/clis/discord-app/send.ts +32 -0
  231. package/src/clis/discord-app/servers.ts +39 -0
  232. package/src/clis/discord-app/status.ts +18 -0
  233. package/src/clis/feishu/README.md +20 -0
  234. package/src/clis/feishu/README.zh-CN.md +20 -0
  235. package/src/clis/feishu/new.ts +32 -0
  236. package/src/clis/feishu/read.ts +48 -0
  237. package/src/clis/feishu/search.ts +35 -0
  238. package/src/clis/feishu/send.ts +46 -0
  239. package/src/clis/feishu/status.ts +34 -0
  240. package/src/clis/grok/ask.ts +90 -0
  241. package/src/clis/grok/debug.ts +49 -0
  242. package/src/clis/jimeng/generate.yaml +84 -0
  243. package/src/clis/jimeng/history.yaml +47 -0
  244. package/src/clis/linux-do/categories.yaml +41 -0
  245. package/src/clis/linux-do/category.yaml +49 -0
  246. package/src/clis/linux-do/hot.yaml +50 -0
  247. package/src/clis/linux-do/latest.yaml +40 -0
  248. package/src/clis/linux-do/search.yaml +45 -0
  249. package/src/clis/linux-do/topic.yaml +38 -0
  250. package/src/clis/neteasemusic/README.md +31 -0
  251. package/src/clis/neteasemusic/README.zh-CN.md +31 -0
  252. package/src/clis/neteasemusic/like.ts +28 -0
  253. package/src/clis/neteasemusic/lyrics.ts +53 -0
  254. package/src/clis/neteasemusic/next.ts +30 -0
  255. package/src/clis/neteasemusic/play.ts +30 -0
  256. package/src/clis/neteasemusic/playing.ts +62 -0
  257. package/src/clis/neteasemusic/playlist.ts +51 -0
  258. package/src/clis/neteasemusic/prev.ts +29 -0
  259. package/src/clis/neteasemusic/search.ts +58 -0
  260. package/src/clis/neteasemusic/status.ts +18 -0
  261. package/src/clis/neteasemusic/volume.ts +61 -0
  262. package/src/clis/notion/README.md +29 -0
  263. package/src/clis/notion/README.zh-CN.md +29 -0
  264. package/src/clis/notion/export.ts +36 -0
  265. package/src/clis/notion/favorites.ts +87 -0
  266. package/src/clis/notion/new.ts +39 -0
  267. package/src/clis/notion/read.ts +33 -0
  268. package/src/clis/notion/search.ts +54 -0
  269. package/src/clis/notion/sidebar.ts +44 -0
  270. package/src/clis/notion/status.ts +18 -0
  271. package/src/clis/notion/write.ts +45 -0
  272. package/src/clis/twitter/download.ts +227 -0
  273. package/src/clis/wechat/README.md +28 -0
  274. package/src/clis/wechat/README.zh-CN.md +28 -0
  275. package/src/clis/wechat/chats.ts +33 -0
  276. package/src/clis/wechat/contacts.ts +33 -0
  277. package/src/clis/wechat/read.ts +72 -0
  278. package/src/clis/wechat/search.ts +36 -0
  279. package/src/clis/wechat/send.ts +49 -0
  280. package/src/clis/wechat/status.ts +35 -0
  281. package/src/clis/xiaohongshu/creator-note-detail.ts +95 -0
  282. package/src/clis/xiaohongshu/creator-notes.ts +116 -0
  283. package/src/clis/xiaohongshu/creator-profile.ts +60 -0
  284. package/src/clis/xiaohongshu/creator-stats.ts +81 -0
  285. package/src/clis/xiaohongshu/download.ts +173 -0
  286. package/src/clis/xiaohongshu/search.ts +1 -1
  287. package/src/clis/xiaohongshu/user-helpers.test.ts +106 -0
  288. package/src/clis/xiaohongshu/user-helpers.ts +85 -0
  289. package/src/clis/xiaohongshu/user.ts +52 -32
  290. package/src/clis/zhihu/download.test.ts +12 -0
  291. package/src/clis/zhihu/download.ts +223 -0
  292. package/src/daemon.ts +217 -0
  293. package/src/doctor.test.ts +32 -193
  294. package/src/doctor.ts +74 -668
  295. package/src/download/index.ts +395 -0
  296. package/src/download/progress.ts +125 -0
  297. package/src/engine.test.ts +17 -0
  298. package/src/main.ts +18 -26
  299. package/src/pipeline/executor.test.ts +1 -0
  300. package/src/pipeline/registry.ts +2 -0
  301. package/src/pipeline/steps/browser.ts +2 -2
  302. package/src/pipeline/steps/download.ts +310 -0
  303. package/src/pipeline/steps/intercept.ts +1 -2
  304. package/src/pipeline/template.ts +26 -0
  305. package/src/setup.ts +47 -183
  306. package/src/types.ts +1 -0
  307. package/tests/e2e/browser-auth.test.ts +25 -0
@@ -0,0 +1,310 @@
1
+ /**
2
+ * Pipeline step: download — file download with concurrency and progress.
3
+ *
4
+ * Supports:
5
+ * - Direct HTTP downloads (images, documents)
6
+ * - yt-dlp integration for video platforms
7
+ * - Browser cookie forwarding for authenticated downloads
8
+ * - Filename templating and deduplication
9
+ */
10
+
11
+ import * as fs from 'node:fs';
12
+ import * as path from 'node:path';
13
+ import * as os from 'node:os';
14
+ import type { IPage } from '../../types.js';
15
+ import { render } from '../template.js';
16
+ import {
17
+ httpDownload,
18
+ ytdlpDownload,
19
+ saveDocument,
20
+ detectContentType,
21
+ requiresYtdlp,
22
+ sanitizeFilename,
23
+ generateFilename,
24
+ exportCookiesToNetscape,
25
+ getTempDir,
26
+ } from '../../download/index.js';
27
+ import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
28
+
29
+ export interface DownloadResult {
30
+ status: 'success' | 'skipped' | 'failed';
31
+ path?: string;
32
+ size?: number;
33
+ error?: string;
34
+ duration?: number;
35
+ }
36
+
37
+ /**
38
+ * Simple async concurrency limiter for downloads.
39
+ */
40
+ async function mapConcurrent<T, R>(
41
+ items: T[],
42
+ limit: number,
43
+ fn: (item: T, index: number) => Promise<R>,
44
+ ): Promise<R[]> {
45
+ const results: R[] = new Array(items.length);
46
+ let index = 0;
47
+
48
+ async function worker() {
49
+ while (index < items.length) {
50
+ const i = index++;
51
+ results[i] = await fn(items[i], i);
52
+ }
53
+ }
54
+
55
+ const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
56
+ await Promise.all(workers);
57
+ return results;
58
+ }
59
+
60
+ /**
61
+ * Extract cookies from browser page.
62
+ */
63
+ async function extractBrowserCookies(page: IPage, domain?: string): Promise<string> {
64
+ try {
65
+ // Use browser evaluate to get document.cookie
66
+ const cookieString = await page.evaluate(`(() => document.cookie)()`);
67
+ return typeof cookieString === 'string' ? cookieString : '';
68
+ } catch {
69
+ return '';
70
+ }
71
+ }
72
+
73
+ /**
74
+ * Extract cookies as array for yt-dlp Netscape format.
75
+ */
76
+ async function extractCookiesArray(
77
+ page: IPage,
78
+ domain: string,
79
+ ): Promise<Array<{ name: string; value: string; domain: string; path: string; secure: boolean; httpOnly: boolean }>> {
80
+ try {
81
+ const cookieString = await extractBrowserCookies(page);
82
+ if (!cookieString) return [];
83
+
84
+ return cookieString.split(';').map((c) => {
85
+ const [name, ...rest] = c.trim().split('=');
86
+ return {
87
+ name: name || '',
88
+ value: rest.join('=') || '',
89
+ domain,
90
+ path: '/',
91
+ secure: true,
92
+ httpOnly: false,
93
+ };
94
+ }).filter((c) => c.name);
95
+ } catch {
96
+ return [];
97
+ }
98
+ }
99
+
100
+ /**
101
+ * Download step handler for YAML pipelines.
102
+ *
103
+ * Usage in YAML:
104
+ * ```yaml
105
+ * pipeline:
106
+ * - download:
107
+ * url: ${{ item.imageUrl }}
108
+ * dir: ./downloads
109
+ * filename: ${{ item.title }}.jpg
110
+ * concurrency: 5
111
+ * skip_existing: true
112
+ * use_ytdlp: false
113
+ * type: auto
114
+ * ```
115
+ */
116
+ export async function stepDownload(
117
+ page: IPage | null,
118
+ params: any,
119
+ data: any,
120
+ args: Record<string, any>,
121
+ ): Promise<any> {
122
+ // Parse parameters with defaults
123
+ const urlTemplate = typeof params === 'string' ? params : (params?.url ?? '');
124
+ const dirTemplate = params?.dir ?? './downloads';
125
+ const filenameTemplate = params?.filename ?? '';
126
+ const concurrency = typeof params?.concurrency === 'number' ? params.concurrency : 3;
127
+ const skipExisting = params?.skip_existing !== false;
128
+ const timeout = typeof params?.timeout === 'number' ? params.timeout * 1000 : 30000;
129
+ const useYtdlp = params?.use_ytdlp ?? false;
130
+ const ytdlpArgs = Array.isArray(params?.ytdlp_args) ? params.ytdlp_args : [];
131
+ const contentType = params?.type ?? 'auto';
132
+ const showProgress = params?.progress !== false;
133
+ const contentTemplate = params?.content;
134
+ const metadataTemplate = params?.metadata;
135
+
136
+ // Resolve output directory
137
+ const dir = String(render(dirTemplate, { args, data }));
138
+ fs.mkdirSync(dir, { recursive: true });
139
+
140
+ // Normalize data to array
141
+ const items: any[] = Array.isArray(data) ? data : data ? [data] : [];
142
+ if (items.length === 0) {
143
+ return [];
144
+ }
145
+
146
+ // Create progress tracker
147
+ const tracker = new DownloadProgressTracker(items.length, showProgress);
148
+
149
+ // Extract cookies if browser is available
150
+ let cookies = '';
151
+ let cookiesFile: string | undefined;
152
+
153
+ if (page) {
154
+ cookies = await extractBrowserCookies(page);
155
+
156
+ // For yt-dlp, we need to export cookies to Netscape format
157
+ if (useYtdlp || items.some((item, index) => {
158
+ const url = String(render(urlTemplate, { args, data, item, index }));
159
+ return requiresYtdlp(url);
160
+ })) {
161
+ try {
162
+ // Try to get domain from first URL
163
+ const firstUrl = String(render(urlTemplate, { args, data, item: items[0], index: 0 }));
164
+ const domain = new URL(firstUrl).hostname;
165
+ const cookiesArray = await extractCookiesArray(page, domain);
166
+
167
+ if (cookiesArray.length > 0) {
168
+ const tempDir = getTempDir();
169
+ fs.mkdirSync(tempDir, { recursive: true });
170
+ cookiesFile = path.join(tempDir, `cookies_${Date.now()}.txt`);
171
+ exportCookiesToNetscape(cookiesArray, cookiesFile);
172
+ }
173
+ } catch {
174
+ // Ignore cookie extraction errors
175
+ }
176
+ }
177
+ }
178
+
179
+ // Process downloads with concurrency
180
+ const results = await mapConcurrent(items, concurrency, async (item, index): Promise<any> => {
181
+ const startTime = Date.now();
182
+
183
+ // Render URL
184
+ const url = String(render(urlTemplate, { args, data, item, index }));
185
+ if (!url) {
186
+ tracker.onFileComplete(false);
187
+ return {
188
+ ...item,
189
+ _download: { status: 'failed', error: 'Empty URL' } as DownloadResult,
190
+ };
191
+ }
192
+
193
+ // Render filename
194
+ let filename: string;
195
+ if (filenameTemplate) {
196
+ filename = String(render(filenameTemplate, { args, data, item, index }));
197
+ } else {
198
+ filename = generateFilename(url, index);
199
+ }
200
+ filename = sanitizeFilename(filename);
201
+
202
+ const destPath = path.join(dir, filename);
203
+
204
+ // Check if file exists and skip_existing is true
205
+ if (skipExisting && fs.existsSync(destPath)) {
206
+ tracker.onFileComplete(true, true);
207
+ return {
208
+ ...item,
209
+ _download: {
210
+ status: 'skipped',
211
+ path: destPath,
212
+ size: fs.statSync(destPath).size,
213
+ } as DownloadResult,
214
+ };
215
+ }
216
+
217
+ // Create progress bar for this file
218
+ const progressBar = tracker.onFileStart(filename, index);
219
+
220
+ // Determine download method
221
+ const detectedType = contentType === 'auto' ? detectContentType(url) : contentType;
222
+ const shouldUseYtdlp = useYtdlp || (detectedType === 'video' && requiresYtdlp(url));
223
+
224
+ let result: { success: boolean; size: number; error?: string };
225
+
226
+ try {
227
+ if (detectedType === 'document' && contentTemplate) {
228
+ // Save extracted content as document
229
+ const content = String(render(contentTemplate, { args, data, item, index }));
230
+ const metadata = metadataTemplate
231
+ ? Object.fromEntries(
232
+ Object.entries(metadataTemplate).map(([k, v]) => [k, render(v, { args, data, item, index })]),
233
+ )
234
+ : undefined;
235
+
236
+ const ext = path.extname(filename).toLowerCase();
237
+ const format = ext === '.json' ? 'json' : ext === '.html' ? 'html' : 'markdown';
238
+ result = await saveDocument(content, destPath, format, metadata);
239
+
240
+ if (progressBar) {
241
+ progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
242
+ }
243
+ } else if (shouldUseYtdlp) {
244
+ // Use yt-dlp for video downloads
245
+ result = await ytdlpDownload(url, destPath, {
246
+ cookiesFile,
247
+ extraArgs: ytdlpArgs,
248
+ onProgress: (percent) => {
249
+ if (progressBar) {
250
+ progressBar.update(percent, 100);
251
+ }
252
+ },
253
+ });
254
+
255
+ if (progressBar) {
256
+ progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
257
+ }
258
+ } else {
259
+ // Direct HTTP download
260
+ result = await httpDownload(url, destPath, {
261
+ cookies,
262
+ timeout,
263
+ onProgress: (received, total) => {
264
+ if (progressBar) {
265
+ progressBar.update(received, total);
266
+ }
267
+ },
268
+ });
269
+
270
+ if (progressBar) {
271
+ progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
272
+ }
273
+ }
274
+ } catch (err: any) {
275
+ result = { success: false, size: 0, error: err.message };
276
+ if (progressBar) {
277
+ progressBar.fail(err.message);
278
+ }
279
+ }
280
+
281
+ tracker.onFileComplete(result.success);
282
+
283
+ const duration = Date.now() - startTime;
284
+
285
+ return {
286
+ ...item,
287
+ _download: {
288
+ status: result.success ? 'success' : 'failed',
289
+ path: result.success ? destPath : undefined,
290
+ size: result.size,
291
+ error: result.error,
292
+ duration,
293
+ } as DownloadResult,
294
+ };
295
+ });
296
+
297
+ // Cleanup temp cookie file
298
+ if (cookiesFile && fs.existsSync(cookiesFile)) {
299
+ try {
300
+ fs.unlinkSync(cookiesFile);
301
+ } catch {
302
+ // Ignore cleanup errors
303
+ }
304
+ }
305
+
306
+ // Show summary
307
+ tracker.finish();
308
+
309
+ return results;
310
+ }
@@ -3,7 +3,7 @@
3
3
  */
4
4
 
5
5
  import type { IPage } from '../../types.js';
6
- import { render } from '../template.js';
6
+ import { render, normalizeEvaluateSource } from '../template.js';
7
7
  import { generateInterceptorJs, generateReadInterceptedJs } from '../../interceptor.js';
8
8
 
9
9
  export async function stepIntercept(page: IPage | null, params: any, data: any, args: Record<string, any>): Promise<any> {
@@ -24,7 +24,6 @@ export async function stepIntercept(page: IPage | null, params: any, data: any,
24
24
  await page!.goto(String(url));
25
25
  } else if (trigger.startsWith('evaluate:')) {
26
26
  const js = trigger.slice('evaluate:'.length);
27
- const { normalizeEvaluateSource } = await import('../template.js');
28
27
  await page!.evaluate(normalizeEvaluateSource(render(js, { args, data }) as string));
29
28
  } else if (trigger.startsWith('click:')) {
30
29
  const ref = render(trigger.slice('click:'.length), { args, data });
@@ -119,6 +119,32 @@ function applyFilter(filterExpr: string, value: any): any {
119
119
  return Array.isArray(value) ? value[value.length - 1] : value;
120
120
  case 'json':
121
121
  return JSON.stringify(value ?? null);
122
+ case 'slugify':
123
+ // Convert to URL-safe slug
124
+ return typeof value === 'string'
125
+ ? value
126
+ .toLowerCase()
127
+ .replace(/[^\p{L}\p{N}]+/gu, '-')
128
+ .replace(/^-|-$/g, '')
129
+ : value;
130
+ case 'sanitize':
131
+ // Remove invalid filename characters
132
+ return typeof value === 'string'
133
+ ? value.replace(/[<>:"/\\|?*\x00-\x1f]/g, '_')
134
+ : value;
135
+ case 'ext': {
136
+ // Extract file extension from URL or path
137
+ if (typeof value !== 'string') return value;
138
+ const lastDot = value.lastIndexOf('.');
139
+ const lastSlash = Math.max(value.lastIndexOf('/'), value.lastIndexOf('\\'));
140
+ return lastDot > lastSlash ? value.slice(lastDot) : '';
141
+ }
142
+ case 'basename': {
143
+ // Extract filename from URL or path
144
+ if (typeof value !== 'string') return value;
145
+ const parts = value.split(/[/\\]/);
146
+ return parts[parts.length - 1] || value;
147
+ }
122
148
  default:
123
149
  return value;
124
150
  }
package/src/setup.ts CHANGED
@@ -1,205 +1,69 @@
1
1
  /**
2
- * setup.ts — Interactive Playwright MCP token setup
2
+ * setup.ts — Interactive browser setup for opencli
3
3
  *
4
- * Discovers the extension token, shows an interactive checkbox
5
- * for selecting which config files to update, and applies changes.
4
+ * Simplified for daemon-based architecture. No more token management.
5
+ * Just verifies daemon + extension connectivity.
6
6
  */
7
- import * as fs from 'node:fs';
7
+
8
8
  import chalk from 'chalk';
9
- import { createInterface } from 'node:readline/promises';
10
- import { stdin as input, stdout as output } from 'node:process';
11
- import {
12
- type DoctorReport,
13
- PLAYWRIGHT_TOKEN_ENV,
14
- checkExtensionInstalled,
15
- checkTokenConnectivity,
16
- discoverExtensionToken,
17
- fileExists,
18
- getDefaultShellRcPath,
19
- runBrowserDoctor,
20
- shortenPath,
21
- toolName,
22
- upsertJsonConfigToken,
23
- upsertShellToken,
24
- upsertTomlConfigToken,
25
- writeFileWithMkdir,
26
- } from './doctor.js';
27
- import { getTokenFingerprint } from './browser/index.js';
28
- import { type CheckboxItem, checkboxPrompt } from './tui.js';
9
+ import { checkDaemonStatus } from './browser/discover.js';
10
+ import { checkConnectivity } from './doctor.js';
11
+ import { PlaywrightMCP } from './browser/index.js';
29
12
 
30
13
  export async function runSetup(opts: { cliVersion?: string; token?: string } = {}) {
31
14
  console.log();
32
- console.log(chalk.bold(' opencli setup') + chalk.dim(' — Playwright MCP token configuration'));
15
+ console.log(chalk.bold(' opencli setup') + chalk.dim(' — browser bridge configuration'));
33
16
  console.log();
34
17
 
35
- // Step 1: Discover token
36
- let token = opts.token ?? null;
37
-
38
- if (!token) {
39
- const extensionToken = discoverExtensionToken();
40
- const envToken = process.env[PLAYWRIGHT_TOKEN_ENV] ?? null;
18
+ // Step 1: Check daemon
19
+ console.log(chalk.dim(' Checking daemon status...'));
20
+ const status = await checkDaemonStatus();
41
21
 
42
- if (extensionToken && envToken && extensionToken === envToken) {
43
- token = extensionToken;
44
- console.log(` ${chalk.green('✓')} Token auto-discovered from Chrome extension`);
45
- console.log(` Fingerprint: ${chalk.bold(getTokenFingerprint(token) ?? 'unknown')}`);
46
- } else if (extensionToken) {
47
- token = extensionToken;
48
- console.log(` ${chalk.green('✓')} Token discovered from Chrome extension ` +
49
- chalk.dim(`(${getTokenFingerprint(token)})`));
50
- if (envToken && envToken !== extensionToken) {
51
- console.log(` ${chalk.yellow('!')} Environment has different token ` +
52
- chalk.dim(`(${getTokenFingerprint(envToken)})`));
53
- }
54
- } else if (envToken) {
55
- token = envToken;
56
- console.log(` ${chalk.green('✓')} Token from environment variable ` +
57
- chalk.dim(`(${getTokenFingerprint(token)})`));
58
- }
22
+ if (status.running) {
23
+ console.log(` ${chalk.green('✓')} Daemon is running on port 19825`);
59
24
  } else {
60
- console.log(` ${chalk.green('')} Using provided token ` +
61
- chalk.dim(`(${getTokenFingerprint(token)})`));
25
+ console.log(` ${chalk.yellow('!')} Daemon is not running`);
26
+ console.log(chalk.dim(' The daemon starts automatically when you run a browser command.'));
27
+ console.log(chalk.dim(' Starting daemon now...'));
28
+
29
+ // Try to spawn daemon
30
+ const mcp = new PlaywrightMCP();
31
+ try {
32
+ await mcp.connect({ timeout: 5 });
33
+ await mcp.close();
34
+ console.log(` ${chalk.green('✓')} Daemon started successfully`);
35
+ } catch {
36
+ console.log(` ${chalk.yellow('!')} Could not start daemon automatically`);
37
+ }
62
38
  }
63
39
 
64
- if (!token) {
65
- // Give precise diagnosis of why token scan failed
66
- const extInstall = checkExtensionInstalled();
67
-
68
- console.log(` ${chalk.red('✗')} Browser token scan failed\n`);
69
- if (!extInstall.installed) {
70
- console.log(chalk.dim(' Cause: Playwright MCP Bridge extension is not installed'));
71
- console.log(chalk.dim(' Fix: Install from https://chromewebstore.google.com/detail/'));
72
- console.log(chalk.dim(' playwright-mcp-bridge/mmlmfjhmonkocbjadbfplnigmagldckm'));
73
- } else {
74
- console.log(chalk.dim(` Cause: Extension is installed (${extInstall.browsers.join(', ')}) but token not found in LevelDB`));
75
- console.log(chalk.dim(' Fix: 1) Open the extension popup and verify the token is generated'));
76
- console.log(chalk.dim(' 2) Close Chrome completely, then re-run setup'));
77
- }
40
+ // Step 2: Check extension
41
+ const statusAfter = await checkDaemonStatus();
42
+ if (statusAfter.extensionConnected) {
43
+ console.log(` ${chalk.green('✓')} Chrome extension connected`);
44
+ } else {
45
+ console.log(` ${chalk.red('✗')} Chrome extension not connected`);
78
46
  console.log();
79
- console.log(` You can enter the token manually, or fix the above and re-run ${chalk.bold('opencli setup')}.`);
47
+ console.log(chalk.dim(' To install the opencli Browser Bridge extension:'));
48
+ console.log(chalk.dim(' 1. Download from GitHub Releases'));
49
+ console.log(chalk.dim(' 2. Open chrome://extensions/ → Enable Developer Mode'));
50
+ console.log(chalk.dim(' 3. Click "Load unpacked" → select the extension folder'));
51
+ console.log(chalk.dim(' 4. Make sure Chrome is running'));
80
52
  console.log();
81
- const rl = createInterface({ input, output });
82
- const answer = await rl.question(' Token (press Enter to abort): ');
83
- rl.close();
84
- token = answer.trim();
85
- if (!token) {
86
- console.log(chalk.red('\n No token provided. Aborting.\n'));
87
- return;
88
- }
89
- }
90
-
91
- const fingerprint = getTokenFingerprint(token) ?? 'unknown';
92
- console.log();
93
-
94
- // Step 2: Scan all config locations
95
- const report = await runBrowserDoctor({ token, cliVersion: opts.cliVersion });
96
-
97
- // Step 3: Build checkbox items
98
- const items: CheckboxItem[] = [];
99
-
100
- // Shell file
101
- const shellPath = report.shellFiles[0]?.path ?? getDefaultShellRcPath();
102
- const shellStatus = report.shellFiles[0];
103
- const shellFp = shellStatus?.fingerprint;
104
- const shellOk = shellFp === fingerprint;
105
- const shellTool = toolName(shellPath) || 'Shell';
106
- items.push({
107
- label: padRight(shortenPath(shellPath), 50) + chalk.dim(` [${shellTool}]`),
108
- value: `shell:${shellPath}`,
109
- checked: !shellOk,
110
- status: shellOk ? `configured (${shellFp})` : shellFp ? `mismatch (${shellFp})` : 'missing',
111
- statusColor: shellOk ? 'green' : shellFp ? 'yellow' : 'red',
112
- });
113
-
114
- // Config files
115
- for (const config of report.configs) {
116
- const fp = config.fingerprint;
117
- const ok = fp === fingerprint;
118
- const tool = toolName(config.path);
119
- items.push({
120
- label: padRight(shortenPath(config.path), 50) + chalk.dim(tool ? ` [${tool}]` : ''),
121
- value: `config:${config.path}`,
122
- checked: false, // let user explicitly select which tools to configure
123
- status: ok ? `configured (${fp})` : !config.exists ? 'will create' : fp ? `mismatch (${fp})` : 'missing',
124
- statusColor: ok ? 'green' : 'yellow',
125
- });
126
- }
127
-
128
- // Step 4: Show interactive checkbox
129
- console.clear();
130
- const selected = await checkboxPrompt(items, {
131
- title: ` ${chalk.bold('opencli setup')} — token ${chalk.cyan(fingerprint)}`,
132
- });
133
-
134
- if (selected.length === 0) {
135
- console.log(chalk.dim(' No changes made.\n'));
136
53
  return;
137
54
  }
138
55
 
139
- // Step 5: Apply changes
140
- const written: string[] = [];
141
- let wroteShell = false;
142
-
143
- for (const sel of selected) {
144
- if (sel.startsWith('shell:')) {
145
- const p = sel.slice('shell:'.length);
146
- const before = fileExists(p) ? fs.readFileSync(p, 'utf-8') : '';
147
- writeFileWithMkdir(p, upsertShellToken(before, token, p));
148
- written.push(p);
149
- wroteShell = true;
150
- } else if (sel.startsWith('config:')) {
151
- const p = sel.slice('config:'.length);
152
- const config = report.configs.find(c => c.path === p);
153
- if (config && config.parseError) continue;
154
- const before = fileExists(p) ? fs.readFileSync(p, 'utf-8') : '';
155
- const format = config?.format ?? (p.endsWith('.toml') ? 'toml' : 'json');
156
- const next = format === 'toml' ? upsertTomlConfigToken(before, token) : upsertJsonConfigToken(before, token, p);
157
- writeFileWithMkdir(p, next);
158
- written.push(p);
159
- }
160
- }
161
-
162
- process.env[PLAYWRIGHT_TOKEN_ENV] = token;
163
-
164
- // Step 6: Summary
165
- if (written.length > 0) {
166
- console.log(chalk.green.bold(` ✓ Updated ${written.length} file(s):`));
167
- for (const p of written) {
168
- const tool = toolName(p);
169
- console.log(` ${chalk.dim('•')} ${shortenPath(p)}${tool ? chalk.dim(` [${tool}]`) : ''}`);
170
- }
171
- if (wroteShell) {
172
- console.log();
173
- console.log(chalk.cyan(` 💡 Run ${chalk.bold(`source ${shortenPath(shellPath)}`)} to apply token to current shell.`));
174
- }
175
- } else {
176
- console.log(chalk.yellow(' No files were changed.'));
177
- }
56
+ // Step 3: Test connectivity
178
57
  console.log();
179
-
180
- // Step 7: Auto-verify browser connectivity
181
- console.log(chalk.dim(' Verifying browser connectivity...'));
182
- try {
183
- const result = await checkTokenConnectivity({ timeout: 5 });
184
- if (result.ok) {
185
- console.log(` ${chalk.green('✓')} Browser connected in ${(result.durationMs / 1000).toFixed(1)}s`);
186
- } else {
187
- console.log(` ${chalk.green('')} Token saved successfully.`);
188
- console.log(` ${chalk.yellow('!')} Browser connectivity test failed: ${result.error ?? 'unknown'}`);
189
- console.log(chalk.dim(' Token configuration is complete. To use opencli, make sure Chrome'));
190
- console.log(chalk.dim(' is running with the Playwright MCP Bridge extension enabled.'));
191
- console.log(chalk.dim(` Run ${chalk.bold('opencli doctor --live')} to re-test connectivity.`));
192
- }
193
- } catch {
194
- console.log(` ${chalk.green('✓')} Token saved successfully.`);
195
- console.log(` ${chalk.yellow('!')} Browser connectivity test skipped (Chrome may not be running).`);
196
- console.log(chalk.dim(' Token configuration is complete. Start Chrome to begin using opencli.'));
197
- console.log(chalk.dim(` Run ${chalk.bold('opencli doctor --live')} to re-test connectivity.`));
58
+ console.log(chalk.dim(' Testing browser connectivity...'));
59
+ const conn = await checkConnectivity({ timeout: 5 });
60
+ if (conn.ok) {
61
+ console.log(` ${chalk.green('✓')} Browser connected in ${(conn.durationMs / 1000).toFixed(1)}s`);
62
+ console.log();
63
+ console.log(chalk.green.bold(' ✓ Setup complete! You can now use opencli browser commands.'));
64
+ } else {
65
+ console.log(` ${chalk.yellow('!')} Connectivity test failed: ${conn.error ?? 'unknown'}`);
66
+ console.log(chalk.dim(` Run ${chalk.bold('opencli doctor --live')} to diagnose.`));
198
67
  }
199
68
  console.log();
200
69
  }
201
-
202
- function padRight(s: string, n: number): string {
203
- const visible = s.replace(/\x1b\[[0-9;]*m/g, '');
204
- return visible.length >= n ? s : s + ' '.repeat(n - visible.length);
205
- }
package/src/types.ts CHANGED
@@ -23,4 +23,5 @@ export interface IPage {
23
23
  autoScroll(options?: { times?: number; delayMs?: number }): Promise<void>;
24
24
  installInterceptor(pattern: string): Promise<void>;
25
25
  getInterceptedRequests(): Promise<any[]>;
26
+ screenshot(options?: { format?: 'png' | 'jpeg'; quality?: number; fullPage?: boolean; path?: string }): Promise<string>;
26
27
  }
@@ -79,6 +79,31 @@ describe('login-required commands — graceful failure', () => {
79
79
  await expectGracefulAuthFailure(['xueqiu', 'watchlist', '-f', 'json'], 'xueqiu watchlist');
80
80
  }, 60_000);
81
81
 
82
+ // ── linux-do (requires login — all endpoints need authentication) ──
83
+ it('linux-do hot fails gracefully without login', async () => {
84
+ await expectGracefulAuthFailure(['linux-do', 'hot', '--limit', '3', '-f', 'json'], 'linux-do hot');
85
+ }, 60_000);
86
+
87
+ it('linux-do latest fails gracefully without login', async () => {
88
+ await expectGracefulAuthFailure(['linux-do', 'latest', '--limit', '3', '-f', 'json'], 'linux-do latest');
89
+ }, 60_000);
90
+
91
+ it('linux-do categories fails gracefully without login', async () => {
92
+ await expectGracefulAuthFailure(['linux-do', 'categories', '--limit', '3', '-f', 'json'], 'linux-do categories');
93
+ }, 60_000);
94
+
95
+ it('linux-do category fails gracefully without login', async () => {
96
+ await expectGracefulAuthFailure(['linux-do', 'category', '--slug', 'general', '--id', '1', '--limit', '3', '-f', 'json'], 'linux-do category');
97
+ }, 60_000);
98
+
99
+ it('linux-do topic fails gracefully without login', async () => {
100
+ await expectGracefulAuthFailure(['linux-do', 'topic', '--id', '1', '-f', 'json'], 'linux-do topic');
101
+ }, 60_000);
102
+
103
+ it('linux-do search fails gracefully without login', async () => {
104
+ await expectGracefulAuthFailure(['linux-do', 'search', '--keyword', 'test', '--limit', '3', '-f', 'json'], 'linux-do search');
105
+ }, 60_000);
106
+
82
107
  // ── xiaohongshu (requires login) ──
83
108
  it('xiaohongshu feed fails gracefully without login', async () => {
84
109
  await expectGracefulAuthFailure(['xiaohongshu', 'feed', '--limit', '3', '-f', 'json'], 'xiaohongshu feed');