@jackwener/opencli 1.0.1 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. package/.github/workflows/build-extension.yml +80 -0
  2. package/.github/workflows/ci.yml +6 -6
  3. package/.github/workflows/docs.yml +52 -0
  4. package/.github/workflows/e2e-headed.yml +2 -2
  5. package/.github/workflows/pkg-pr-new.yml +2 -2
  6. package/.github/workflows/release.yml +2 -5
  7. package/.github/workflows/security.yml +2 -2
  8. package/CDP.md +1 -1
  9. package/CDP.zh-CN.md +1 -1
  10. package/README.md +42 -34
  11. package/README.zh-CN.md +42 -34
  12. package/SKILL.md +3 -5
  13. package/dist/browser/cdp.d.ts +42 -0
  14. package/dist/browser/cdp.js +339 -0
  15. package/dist/browser/daemon-client.d.ts +3 -1
  16. package/dist/browser/daemon-client.js +4 -0
  17. package/dist/browser/dom-helpers.d.ts +20 -0
  18. package/dist/browser/dom-helpers.js +109 -0
  19. package/dist/browser/index.d.ts +3 -0
  20. package/dist/browser/index.js +4 -0
  21. package/dist/browser/mcp.d.ts +1 -0
  22. package/dist/browser/mcp.js +10 -5
  23. package/dist/browser/page.d.ts +7 -0
  24. package/dist/browser/page.js +39 -123
  25. package/dist/browser/utils.d.ts +10 -0
  26. package/dist/browser/utils.js +27 -0
  27. package/dist/browser.test.js +49 -1
  28. package/dist/build-manifest.js +3 -1
  29. package/dist/build-manifest.test.js +34 -0
  30. package/dist/capabilityRouting.d.ts +2 -0
  31. package/dist/capabilityRouting.js +30 -0
  32. package/dist/capabilityRouting.test.d.ts +1 -0
  33. package/dist/capabilityRouting.test.js +42 -0
  34. package/dist/chaoxing.d.ts +58 -0
  35. package/dist/chaoxing.js +225 -0
  36. package/dist/chaoxing.test.d.ts +1 -0
  37. package/dist/chaoxing.test.js +45 -0
  38. package/dist/cli-manifest.json +885 -48
  39. package/dist/cli.d.ts +1 -0
  40. package/dist/cli.js +234 -0
  41. package/dist/clis/antigravity/serve.d.ts +14 -0
  42. package/dist/clis/antigravity/serve.js +263 -0
  43. package/dist/clis/bilibili/download.js +4 -14
  44. package/dist/clis/boss/chatlist.d.ts +1 -0
  45. package/dist/clis/boss/chatlist.js +50 -0
  46. package/dist/clis/boss/chatmsg.d.ts +1 -0
  47. package/dist/clis/boss/chatmsg.js +73 -0
  48. package/dist/clis/boss/resume.d.ts +1 -0
  49. package/dist/clis/boss/resume.js +249 -0
  50. package/dist/clis/boss/send.d.ts +1 -0
  51. package/dist/clis/boss/send.js +176 -0
  52. package/dist/clis/chaoxing/assignments.d.ts +1 -0
  53. package/dist/clis/chaoxing/assignments.js +74 -0
  54. package/dist/clis/chaoxing/exams.d.ts +1 -0
  55. package/dist/clis/chaoxing/exams.js +74 -0
  56. package/dist/clis/chatgpt/ask.js +15 -14
  57. package/dist/clis/chatgpt/ax.d.ts +1 -0
  58. package/dist/clis/chatgpt/ax.js +78 -0
  59. package/dist/clis/chatgpt/read.js +5 -6
  60. package/dist/clis/hf/top.d.ts +1 -0
  61. package/dist/clis/hf/top.js +119 -0
  62. package/dist/clis/jike/comment.d.ts +1 -0
  63. package/dist/clis/jike/comment.js +107 -0
  64. package/dist/clis/jike/create.d.ts +1 -0
  65. package/dist/clis/jike/create.js +106 -0
  66. package/dist/clis/jike/feed.d.ts +1 -0
  67. package/dist/clis/jike/feed.js +67 -0
  68. package/dist/clis/jike/like.d.ts +1 -0
  69. package/dist/clis/jike/like.js +61 -0
  70. package/dist/clis/jike/notifications.d.ts +1 -0
  71. package/dist/clis/jike/notifications.js +169 -0
  72. package/dist/clis/jike/post.yaml +58 -0
  73. package/dist/clis/jike/repost.d.ts +1 -0
  74. package/dist/clis/jike/repost.js +103 -0
  75. package/dist/clis/jike/search.d.ts +1 -0
  76. package/dist/clis/jike/search.js +67 -0
  77. package/dist/clis/jike/shared.d.ts +19 -0
  78. package/dist/clis/jike/shared.js +25 -0
  79. package/dist/clis/jike/topic.yaml +52 -0
  80. package/dist/clis/jike/user.yaml +51 -0
  81. package/dist/clis/smzdm/search.js +28 -39
  82. package/dist/clis/stackoverflow/bounties.yaml +29 -0
  83. package/dist/clis/stackoverflow/hot.yaml +28 -0
  84. package/dist/clis/stackoverflow/search.yaml +32 -0
  85. package/dist/clis/stackoverflow/unanswered.yaml +28 -0
  86. package/dist/clis/twitter/download.js +6 -16
  87. package/dist/clis/twitter/post.js +9 -2
  88. package/dist/clis/twitter/search.js +14 -33
  89. package/dist/clis/xiaohongshu/download.d.ts +1 -1
  90. package/dist/clis/xiaohongshu/download.js +4 -4
  91. package/dist/clis/zhihu/download.js +3 -3
  92. package/dist/doctor.d.ts +7 -0
  93. package/dist/doctor.js +16 -0
  94. package/dist/download/index.d.ts +12 -8
  95. package/dist/download/index.js +11 -3
  96. package/dist/download/index.test.d.ts +1 -0
  97. package/dist/download/index.test.js +14 -0
  98. package/dist/engine.js +25 -14
  99. package/dist/explore.d.ts +1 -0
  100. package/dist/explore.js +48 -103
  101. package/dist/generate.js +1 -0
  102. package/dist/interceptor.js +3 -2
  103. package/dist/main.js +4 -193
  104. package/dist/output.d.ts +2 -1
  105. package/dist/output.js +3 -1
  106. package/dist/pipeline/executor.test.js +1 -0
  107. package/dist/pipeline/steps/download.js +14 -18
  108. package/dist/registry.d.ts +4 -3
  109. package/dist/registry.js +5 -2
  110. package/dist/runtime.d.ts +4 -1
  111. package/dist/runtime.js +2 -2
  112. package/dist/scripts/framework.d.ts +4 -0
  113. package/dist/scripts/framework.js +21 -0
  114. package/dist/scripts/interact.d.ts +4 -0
  115. package/dist/scripts/interact.js +20 -0
  116. package/dist/scripts/store.d.ts +9 -0
  117. package/dist/scripts/store.js +44 -0
  118. package/dist/synthesize.js +1 -1
  119. package/dist/types.d.ts +12 -0
  120. package/dist/verify.d.ts +6 -1
  121. package/dist/verify.js +54 -2
  122. package/docs/.vitepress/config.mts +193 -0
  123. package/docs/adapters/browser/apple-podcasts.md +28 -0
  124. package/docs/adapters/browser/bbc.md +26 -0
  125. package/docs/adapters/browser/bilibili.md +38 -0
  126. package/docs/adapters/browser/boss.md +28 -0
  127. package/docs/adapters/browser/coupang.md +28 -0
  128. package/docs/adapters/browser/ctrip.md +27 -0
  129. package/docs/adapters/browser/github.md +26 -0
  130. package/docs/adapters/browser/hackernews.md +26 -0
  131. package/docs/adapters/browser/linkedin.md +27 -0
  132. package/docs/adapters/browser/reddit.md +41 -0
  133. package/docs/adapters/browser/reuters.md +27 -0
  134. package/docs/adapters/browser/smzdm.md +27 -0
  135. package/docs/adapters/browser/twitter.md +47 -0
  136. package/docs/adapters/browser/v2ex.md +32 -0
  137. package/docs/adapters/browser/weibo.md +27 -0
  138. package/docs/adapters/browser/xiaohongshu.md +32 -0
  139. package/docs/adapters/browser/xiaoyuzhou.md +28 -0
  140. package/docs/adapters/browser/xueqiu.md +32 -0
  141. package/docs/adapters/browser/yahoo-finance.md +26 -0
  142. package/docs/adapters/browser/youtube.md +29 -0
  143. package/docs/adapters/browser/zhihu.md +30 -0
  144. package/docs/adapters/desktop/antigravity.md +46 -0
  145. package/docs/adapters/desktop/chatgpt.md +43 -0
  146. package/docs/adapters/desktop/chatwise.md +38 -0
  147. package/docs/adapters/desktop/codex.md +32 -0
  148. package/docs/adapters/desktop/cursor.md +33 -0
  149. package/docs/adapters/desktop/discord.md +28 -0
  150. package/docs/adapters/desktop/feishu.md +20 -0
  151. package/docs/adapters/desktop/neteasemusic.md +31 -0
  152. package/docs/adapters/desktop/notion.md +29 -0
  153. package/docs/adapters/desktop/wechat.md +28 -0
  154. package/docs/adapters/index.md +49 -0
  155. package/docs/advanced/cdp.md +103 -0
  156. package/docs/advanced/download.md +63 -0
  157. package/docs/advanced/electron.md +125 -0
  158. package/docs/advanced/remote-chrome.md +72 -0
  159. package/docs/developer/ai-workflow.md +66 -0
  160. package/docs/developer/architecture.md +90 -0
  161. package/docs/developer/contributing.md +136 -0
  162. package/docs/developer/testing.md +237 -0
  163. package/docs/developer/ts-adapter.md +87 -0
  164. package/docs/developer/yaml-adapter.md +108 -0
  165. package/docs/guide/browser-bridge.md +38 -0
  166. package/docs/guide/getting-started.md +56 -0
  167. package/docs/guide/installation.md +37 -0
  168. package/docs/guide/troubleshooting.md +56 -0
  169. package/docs/index.md +35 -0
  170. package/docs/zh/adapters/index.md +5 -0
  171. package/docs/zh/advanced/cdp.md +3 -0
  172. package/docs/zh/developer/contributing.md +24 -0
  173. package/docs/zh/guide/browser-bridge.md +25 -0
  174. package/docs/zh/guide/getting-started.md +40 -0
  175. package/docs/zh/guide/installation.md +37 -0
  176. package/docs/zh/index.md +29 -0
  177. package/extension/dist/background.js +386 -438
  178. package/extension/manifest.json +2 -2
  179. package/extension/package-lock.json +1156 -0
  180. package/extension/src/background.test.ts +151 -0
  181. package/extension/src/background.ts +124 -53
  182. package/extension/src/protocol.ts +3 -1
  183. package/package.json +7 -3
  184. package/src/browser/cdp.ts +367 -0
  185. package/src/browser/daemon-client.ts +7 -1
  186. package/src/browser/dom-helpers.ts +116 -0
  187. package/src/browser/index.ts +4 -0
  188. package/src/browser/mcp.ts +14 -6
  189. package/src/browser/page.ts +47 -124
  190. package/src/browser/utils.ts +27 -0
  191. package/src/browser.test.ts +56 -0
  192. package/src/build-manifest.test.ts +36 -0
  193. package/src/build-manifest.ts +2 -1
  194. package/src/capabilityRouting.test.ts +47 -0
  195. package/src/capabilityRouting.ts +28 -0
  196. package/src/chaoxing.test.ts +53 -0
  197. package/src/chaoxing.ts +268 -0
  198. package/src/cli.ts +205 -0
  199. package/src/clis/antigravity/SKILL.md +5 -0
  200. package/src/clis/antigravity/serve.ts +329 -0
  201. package/src/clis/bilibili/download.ts +4 -15
  202. package/src/clis/boss/chatlist.ts +50 -0
  203. package/src/clis/boss/chatmsg.ts +70 -0
  204. package/src/clis/boss/resume.ts +262 -0
  205. package/src/clis/boss/send.ts +193 -0
  206. package/src/clis/chaoxing/README.md +36 -0
  207. package/src/clis/chaoxing/README.zh-CN.md +35 -0
  208. package/src/clis/chaoxing/assignments.ts +88 -0
  209. package/src/clis/chaoxing/exams.ts +88 -0
  210. package/src/clis/chatgpt/ask.ts +14 -15
  211. package/src/clis/chatgpt/ax.ts +81 -0
  212. package/src/clis/chatgpt/read.ts +5 -7
  213. package/src/clis/hf/top.ts +141 -0
  214. package/src/clis/jike/comment.ts +113 -0
  215. package/src/clis/jike/create.ts +113 -0
  216. package/src/clis/jike/feed.ts +74 -0
  217. package/src/clis/jike/like.ts +65 -0
  218. package/src/clis/jike/notifications.ts +185 -0
  219. package/src/clis/jike/post.yaml +58 -0
  220. package/src/clis/jike/repost.ts +114 -0
  221. package/src/clis/jike/search.ts +74 -0
  222. package/src/clis/jike/shared.ts +36 -0
  223. package/src/clis/jike/topic.yaml +52 -0
  224. package/src/clis/jike/user.yaml +51 -0
  225. package/src/clis/smzdm/search.ts +30 -39
  226. package/src/clis/stackoverflow/bounties.yaml +29 -0
  227. package/src/clis/stackoverflow/hot.yaml +28 -0
  228. package/src/clis/stackoverflow/search.yaml +32 -0
  229. package/src/clis/stackoverflow/unanswered.yaml +28 -0
  230. package/src/clis/twitter/download.ts +6 -17
  231. package/src/clis/twitter/post.ts +9 -2
  232. package/src/clis/twitter/search.ts +15 -33
  233. package/src/clis/xiaohongshu/download.ts +4 -4
  234. package/src/clis/zhihu/download.ts +3 -3
  235. package/src/doctor.ts +18 -2
  236. package/src/download/index.test.ts +16 -0
  237. package/src/download/index.ts +22 -4
  238. package/src/engine.ts +20 -13
  239. package/src/explore.ts +54 -103
  240. package/src/generate.ts +1 -0
  241. package/src/interceptor.ts +3 -2
  242. package/src/main.ts +4 -180
  243. package/src/output.ts +15 -13
  244. package/src/pipeline/executor.test.ts +1 -0
  245. package/src/pipeline/steps/download.ts +14 -17
  246. package/src/registry.ts +9 -5
  247. package/src/runtime.ts +3 -2
  248. package/src/scripts/framework.ts +20 -0
  249. package/src/scripts/interact.ts +22 -0
  250. package/src/scripts/store.ts +40 -0
  251. package/src/synthesize.ts +1 -1
  252. package/src/types.ts +9 -0
  253. package/src/verify.ts +64 -3
package/dist/explore.js CHANGED
@@ -9,6 +9,9 @@ import * as fs from 'node:fs';
9
9
  import * as path from 'node:path';
10
10
  import { DEFAULT_BROWSER_EXPLORE_TIMEOUT, browserSession, runWithTimeout } from './runtime.js';
11
11
  import { VOLATILE_PARAMS, SEARCH_PARAMS, PAGINATION_PARAMS, LIMIT_PARAMS, FIELD_ROLES } from './constants.js';
12
+ import { detectFramework } from './scripts/framework.js';
13
+ import { discoverStores } from './scripts/store.js';
14
+ import { interactFuzz } from './scripts/interact.js';
12
15
  // ── Site name detection ────────────────────────────────────────────────────
13
16
  const KNOWN_SITE_ALIASES = {
14
17
  'x.com': 'twitter', 'twitter.com': 'twitter',
@@ -134,11 +137,13 @@ function flattenFields(obj, prefix, maxDepth) {
134
137
  if (maxDepth <= 0 || !obj || typeof obj !== 'object')
135
138
  return [];
136
139
  const names = [];
137
- for (const key of Object.keys(obj)) {
140
+ const record = obj;
141
+ for (const key of Object.keys(record)) {
138
142
  const full = prefix ? `${prefix}.${key}` : key;
139
143
  names.push(full);
140
- if (obj[key] && typeof obj[key] === 'object' && !Array.isArray(obj[key]))
141
- names.push(...flattenFields(obj[key], full, maxDepth - 1));
144
+ const val = record[key];
145
+ if (val && typeof val === 'object' && !Array.isArray(val))
146
+ names.push(...flattenFields(val, full, maxDepth - 1));
142
147
  }
143
148
  return names;
144
149
  }
@@ -200,83 +205,11 @@ function inferStrategy(authIndicators) {
200
205
  return 'cookie';
201
206
  }
202
207
  // ── Framework detection ────────────────────────────────────────────────────
203
- const FRAMEWORK_DETECT_JS = `
204
- () => {
205
- const r = {};
206
- try {
207
- const app = document.querySelector('#app');
208
- r.vue3 = !!(app && app.__vue_app__);
209
- r.vue2 = !!(app && app.__vue__);
210
- r.react = !!window.__REACT_DEVTOOLS_GLOBAL_HOOK__ || !!document.querySelector('[data-reactroot]');
211
- r.nextjs = !!window.__NEXT_DATA__;
212
- r.nuxt = !!window.__NUXT__;
213
- if (r.vue3 && app.__vue_app__) { const gp = app.__vue_app__.config?.globalProperties; r.pinia = !!(gp && gp.$pinia); r.vuex = !!(gp && gp.$store); }
214
- } catch {}
215
- return r;
216
- }
217
- `;
208
+ const FRAMEWORK_DETECT_JS = detectFramework.toString();
218
209
  // ── Store discovery ────────────────────────────────────────────────────────
219
- const STORE_DISCOVER_JS = `
220
- () => {
221
- const stores = [];
222
- try {
223
- const app = document.querySelector('#app');
224
- if (!app?.__vue_app__) return stores;
225
- const gp = app.__vue_app__.config?.globalProperties;
226
-
227
- // Pinia stores
228
- const pinia = gp?.$pinia;
229
- if (pinia?._s) {
230
- pinia._s.forEach((store, id) => {
231
- const actions = [];
232
- const stateKeys = [];
233
- for (const k in store) {
234
- try {
235
- if (k.startsWith('$') || k.startsWith('_')) continue;
236
- if (typeof store[k] === 'function') actions.push(k);
237
- else stateKeys.push(k);
238
- } catch {}
239
- }
240
- stores.push({ type: 'pinia', id, actions: actions.slice(0, 20), stateKeys: stateKeys.slice(0, 15) });
241
- });
242
- }
243
-
244
- // Vuex store modules
245
- const vuex = gp?.$store;
246
- if (vuex?._modules?.root?._children) {
247
- const children = vuex._modules.root._children;
248
- for (const [modName, mod] of Object.entries(children)) {
249
- const actions = Object.keys(mod._rawModule?.actions ?? {}).slice(0, 20);
250
- const stateKeys = Object.keys(mod.state ?? {}).slice(0, 15);
251
- stores.push({ type: 'vuex', id: modName, actions, stateKeys });
252
- }
253
- }
254
- } catch {}
255
- return stores;
256
- }
257
- `;
210
+ const STORE_DISCOVER_JS = discoverStores.toString();
258
211
  // ── Auto-Interaction (Fuzzing) ─────────────────────────────────────────────
259
- const INTERACT_FUZZ_JS = `
260
- async () => {
261
- const sleep = ms => new Promise(r => setTimeout(r, ms));
262
- const clickables = Array.from(document.querySelectorAll(
263
- 'button, [role="button"], [role="tab"], .tab, .btn, a[href="javascript:void(0)"], a[href="#"]'
264
- )).slice(0, 15); // limit to 15 to avoid endless loops
265
-
266
- let clicked = 0;
267
- for (const el of clickables) {
268
- try {
269
- const rect = el.getBoundingClientRect();
270
- if (rect.width > 0 && rect.height > 0) {
271
- el.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
272
- clicked++;
273
- await sleep(300); // give it time to trigger network
274
- }
275
- } catch {}
276
- }
277
- return clicked;
278
- }
279
- `;
212
+ const INTERACT_FUZZ_JS = interactFuzz.toString();
280
213
  // ── Main explore function ──────────────────────────────────────────────────
281
214
  export async function exploreUrl(url, opts) {
282
215
  const waitSeconds = opts.waitSeconds ?? 3.0;
@@ -286,25 +219,19 @@ export async function exploreUrl(url, opts) {
286
219
  // Step 1: Navigate
287
220
  await page.goto(url);
288
221
  await page.wait(waitSeconds);
289
- // Step 2: Auto-scroll to trigger lazy loading (use keyboard since page.scroll may not exist)
290
- for (let i = 0; i < 3; i++) {
291
- try {
292
- await page.pressKey('End');
293
- }
294
- catch { }
295
- await page.wait(1);
296
- }
222
+ // Step 2: Auto-scroll to trigger lazy loading intelligently
223
+ await page.autoScroll({ times: 3, delayMs: 1500 }).catch(() => { });
297
224
  // Step 2.5: Interactive Fuzzing (if requested)
298
225
  if (opts.auto) {
299
226
  try {
300
227
  // First: targeted clicks by label (e.g. "字幕", "CC", "评论")
301
228
  if (opts.clickLabels?.length) {
302
229
  for (const label of opts.clickLabels) {
303
- const safeLabel = label.replace(/'/g, "\\'");
230
+ const safeLabel = JSON.stringify(label);
304
231
  await page.evaluate(`
305
232
  (() => {
306
233
  const el = [...document.querySelectorAll('button, [role="button"], [role="tab"], a, span')]
307
- .find(e => e.textContent && e.textContent.trim().includes('${safeLabel}'));
234
+ .find(e => e.textContent && e.textContent.trim().includes(${safeLabel}));
308
235
  if (el) el.click();
309
236
  })()
310
237
  `);
@@ -324,11 +251,27 @@ export async function exploreUrl(url, opts) {
324
251
  // Step 4: Capture network traffic
325
252
  const rawNetwork = await page.networkRequests(false);
326
253
  const networkEntries = parseNetworkRequests(rawNetwork);
327
- // Step 5: For JSON endpoints, re-fetch response body in-browser
328
- const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200);
329
- for (const ep of jsonEndpoints.slice(0, 10)) {
254
+ // Step 5: For JSON endpoints missing a body, carefully re-fetch in-browser via a pristine iframe
255
+ const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200 && !e.responseBody);
256
+ await Promise.allSettled(jsonEndpoints.slice(0, 5).map(async (ep) => {
330
257
  try {
331
- const body = await page.evaluate(`async () => { try { const r = await fetch(${JSON.stringify(ep.url)}, {credentials:'include'}); if (!r.ok) return null; const d = await r.json(); return JSON.stringify(d).slice(0,10000); } catch { return null; } }`);
258
+ const body = await page.evaluate(`async () => {
259
+ let iframe = null;
260
+ try {
261
+ iframe = document.createElement('iframe');
262
+ iframe.style.display = 'none';
263
+ document.body.appendChild(iframe);
264
+ const cleanFetch = iframe.contentWindow.fetch || window.fetch;
265
+ const r = await cleanFetch(${JSON.stringify(ep.url)}, { credentials: 'include' });
266
+ if (!r.ok) return null;
267
+ const d = await r.json();
268
+ return JSON.stringify(d).slice(0, 10000);
269
+ } catch {
270
+ return null;
271
+ } finally {
272
+ if (iframe && iframe.parentNode) iframe.parentNode.removeChild(iframe);
273
+ }
274
+ }`);
332
275
  if (body && typeof body === 'string') {
333
276
  try {
334
277
  ep.responseBody = JSON.parse(body);
@@ -339,7 +282,7 @@ export async function exploreUrl(url, opts) {
339
282
  ep.responseBody = body;
340
283
  }
341
284
  catch { }
342
- }
285
+ }));
343
286
  // Step 6: Detect framework
344
287
  let framework = {};
345
288
  try {
@@ -443,7 +386,7 @@ export async function exploreUrl(url, opts) {
443
386
  const topStrategy = allAuth.has('signature') ? 'intercept' : allAuth.has('bearer') || allAuth.has('csrf') ? 'header' : allAuth.size === 0 ? 'public' : 'cookie';
444
387
  const siteName = opts.site ?? detectSiteName(metadata.url || url);
445
388
  const targetDir = opts.outDir ?? path.join('.opencli', 'explore', siteName);
446
- fs.mkdirSync(targetDir, { recursive: true });
389
+ await fs.promises.mkdir(targetDir, { recursive: true });
447
390
  const result = {
448
391
  site: siteName, target_url: url, final_url: metadata.url, title: metadata.title,
449
392
  framework, stores, top_strategy: topStrategy,
@@ -452,27 +395,29 @@ export async function exploreUrl(url, opts) {
452
395
  capabilities, auth_indicators: [...allAuth],
453
396
  };
454
397
  // Write artifacts
455
- fs.writeFileSync(path.join(targetDir, 'manifest.json'), JSON.stringify({
398
+ const writeTasks = [];
399
+ writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'manifest.json'), JSON.stringify({
456
400
  site: siteName, target_url: url, final_url: metadata.url, title: metadata.title,
457
401
  framework, stores: stores.map(s => ({ type: s.type, id: s.id, actions: s.actions })),
458
402
  top_strategy: topStrategy, explored_at: new Date().toISOString(),
459
- }, null, 2));
460
- fs.writeFileSync(path.join(targetDir, 'endpoints.json'), JSON.stringify(analyzedEndpoints.map(ep => ({
403
+ }, null, 2)));
404
+ writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'endpoints.json'), JSON.stringify(analyzedEndpoints.map(ep => ({
461
405
  pattern: ep.pattern, method: ep.method, url: ep.url, status: ep.status,
462
406
  contentType: ep.contentType, score: ep.score, queryParams: ep.queryParams,
463
407
  itemPath: ep.responseAnalysis?.itemPath ?? null, itemCount: ep.responseAnalysis?.itemCount ?? 0,
464
408
  detectedFields: ep.responseAnalysis?.detectedFields ?? {}, authIndicators: ep.authIndicators,
465
- })), null, 2));
466
- fs.writeFileSync(path.join(targetDir, 'capabilities.json'), JSON.stringify(capabilities, null, 2));
467
- fs.writeFileSync(path.join(targetDir, 'auth.json'), JSON.stringify({
409
+ })), null, 2)));
410
+ writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'capabilities.json'), JSON.stringify(capabilities, null, 2)));
411
+ writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'auth.json'), JSON.stringify({
468
412
  top_strategy: topStrategy, indicators: [...allAuth], framework,
469
- }, null, 2));
413
+ }, null, 2)));
470
414
  if (stores.length > 0) {
471
- fs.writeFileSync(path.join(targetDir, 'stores.json'), JSON.stringify(stores, null, 2));
415
+ writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'stores.json'), JSON.stringify(stores, null, 2)));
472
416
  }
417
+ await Promise.all(writeTasks);
473
418
  return { ...result, out_dir: targetDir };
474
419
  })(), { timeout: exploreTimeout, label: `Explore ${url}` });
475
- });
420
+ }, { workspace: opts.workspace });
476
421
  }
477
422
  export function renderExploreSummary(result) {
478
423
  const lines = [
package/dist/generate.js CHANGED
@@ -60,6 +60,7 @@ export async function generateCliFromUrl(opts) {
60
60
  site: opts.site,
61
61
  goal: normalizeGoal(opts.goal) ?? opts.goal,
62
62
  waitSeconds: opts.waitSeconds ?? 3,
63
+ workspace: opts.workspace,
63
64
  });
64
65
  // Step 2: Synthesize candidates
65
66
  const synthesizeResult = synthesizeFromExplore(exploreResult.out_dir, {
@@ -22,6 +22,7 @@ export function generateInterceptorJs(patternExpr, opts = {}) {
22
22
  return `
23
23
  () => {
24
24
  window.${arr} = window.${arr} || [];
25
+ window.${arr}_errors = window.${arr}_errors || [];
25
26
  const __pattern = ${patternExpr};
26
27
 
27
28
  if (!window.${guard}) {
@@ -38,7 +39,7 @@ export function generateInterceptorJs(patternExpr, opts = {}) {
38
39
  const clone = response.clone();
39
40
  const json = await clone.json();
40
41
  window.${arr}.push(json);
41
- } catch(e) {}
42
+ } catch(e) { window.${arr}_errors.push({ url: reqUrl, error: String(e) }); }
42
43
  }
43
44
  return response;
44
45
  };
@@ -56,7 +57,7 @@ export function generateInterceptorJs(patternExpr, opts = {}) {
56
57
  this.addEventListener('load', function() {
57
58
  try {
58
59
  window.${arr}.push(JSON.parse(this.responseText));
59
- } catch(e) {}
60
+ } catch(e) { window.${arr}_errors.push({ url: this.__opencli_url, error: String(e) }); }
60
61
  });
61
62
  }
62
63
  return __origSend.apply(this, arguments);
package/dist/main.js CHANGED
@@ -5,16 +5,9 @@
5
5
  import * as os from 'node:os';
6
6
  import * as path from 'node:path';
7
7
  import { fileURLToPath } from 'node:url';
8
- import { Command } from 'commander';
9
- import chalk from 'chalk';
10
- import { discoverClis, executeCommand } from './engine.js';
11
- import { fullName, getRegistry, strategyLabel } from './registry.js';
12
- import { render as renderOutput } from './output.js';
13
- import { BrowserBridge } from './browser/index.js';
14
- import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
15
- import { PKG_VERSION } from './version.js';
16
- import { getCompletions, printCompletionScript } from './completion.js';
17
- import { CliError } from './errors.js';
8
+ import { discoverClis } from './engine.js';
9
+ import { getCompletions } from './completion.js';
10
+ import { runCli } from './cli.js';
18
11
  const __filename = fileURLToPath(import.meta.url);
19
12
  const __dirname = path.dirname(__filename);
20
13
  const BUILTIN_CLIS = path.resolve(__dirname, 'clis');
@@ -42,186 +35,4 @@ if (getCompIdx !== -1) {
42
35
  process.stdout.write(candidates.join('\n') + '\n');
43
36
  process.exit(0);
44
37
  }
45
- const program = new Command();
46
- program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
47
- // ── Built-in commands ──────────────────────────────────────────────────────
48
- program.command('list').description('List all available CLI commands').option('-f, --format <fmt>', 'Output format: table, json, yaml, md, csv', 'table').option('--json', 'JSON output (deprecated)')
49
- .action((opts) => {
50
- const registry = getRegistry();
51
- const commands = [...registry.values()].sort((a, b) => fullName(a).localeCompare(fullName(b)));
52
- const rows = commands.map(c => ({
53
- command: fullName(c),
54
- site: c.site,
55
- name: c.name,
56
- description: c.description,
57
- strategy: strategyLabel(c),
58
- browser: c.browser,
59
- args: c.args.map(a => a.name).join(', '),
60
- }));
61
- const fmt = opts.json && opts.format === 'table' ? 'json' : opts.format;
62
- if (fmt !== 'table') {
63
- renderOutput(rows, {
64
- fmt,
65
- columns: ['command', 'site', 'name', 'description', 'strategy', 'browser', 'args'],
66
- title: 'opencli/list',
67
- source: 'opencli list',
68
- });
69
- return;
70
- }
71
- const sites = new Map();
72
- for (const cmd of commands) {
73
- const g = sites.get(cmd.site) ?? [];
74
- g.push(cmd);
75
- sites.set(cmd.site, g);
76
- }
77
- console.log();
78
- console.log(chalk.bold(' opencli') + chalk.dim(' — available commands'));
79
- console.log();
80
- for (const [site, cmds] of sites) {
81
- console.log(chalk.bold.cyan(` ${site}`));
82
- for (const cmd of cmds) {
83
- const tag = strategyLabel(cmd) === 'public' ? chalk.green('[public]') : chalk.yellow(`[${strategyLabel(cmd)}]`);
84
- console.log(` ${cmd.name} ${tag}${cmd.description ? chalk.dim(` — ${cmd.description}`) : ''}`);
85
- }
86
- console.log();
87
- }
88
- console.log(chalk.dim(` ${commands.length} commands across ${sites.size} sites`));
89
- console.log();
90
- });
91
- program.command('validate').description('Validate CLI definitions').argument('[target]', 'site or site/name')
92
- .action(async (target) => {
93
- const { validateClisWithTarget, renderValidationReport } = await import('./validate.js');
94
- console.log(renderValidationReport(validateClisWithTarget([BUILTIN_CLIS, USER_CLIS], target)));
95
- });
96
- program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false)
97
- .action(async (target, opts) => {
98
- const { verifyClis, renderVerifyReport } = await import('./verify.js');
99
- const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke });
100
- console.log(renderVerifyReport(r));
101
- process.exitCode = r.ok ? 0 : 1;
102
- });
103
- program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3').option('--auto', 'Enable interactive fuzzing (simulate clicks to trigger lazy APIs)').option('--click <labels>', 'Comma-separated labels to click before fuzzing (e.g. "字幕,CC,评论")')
104
- .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); const clickLabels = opts.click ? opts.click.split(',').map((s) => s.trim()) : undefined; console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: BrowserBridge, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait), auto: opts.auto, clickLabels }))); });
105
- program.command('synthesize').description('Synthesize CLIs from explore').argument('<target>').option('--top <n>', '', '3')
106
- .action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); });
107
- program.command('generate').description('One-shot: explore → synthesize → register').argument('<url>').option('--goal <text>').option('--site <name>')
108
- .action(async (url, opts) => { const { generateCliFromUrl, renderGenerateSummary } = await import('./generate.js'); const r = await generateCliFromUrl({ url, BrowserFactory: BrowserBridge, builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, goal: opts.goal, site: opts.site }); console.log(renderGenerateSummary(r)); process.exitCode = r.ok ? 0 : 1; });
109
- program.command('cascade').description('Strategy cascade: find simplest working strategy').argument('<url>').option('--site <name>')
110
- .action(async (url, opts) => {
111
- const { cascadeProbe, renderCascadeResult } = await import('./cascade.js');
112
- const result = await browserSession(BrowserBridge, async (page) => {
113
- // Navigate to the site first for cookie context
114
- try {
115
- const siteUrl = new URL(url);
116
- await page.goto(`${siteUrl.protocol}//${siteUrl.host}`);
117
- await page.wait(2);
118
- }
119
- catch { }
120
- return cascadeProbe(page, url);
121
- });
122
- console.log(renderCascadeResult(result));
123
- });
124
- program.command('doctor')
125
- .description('Diagnose opencli browser bridge connectivity')
126
- .option('--live', 'Test browser connectivity (requires Chrome running)', false)
127
- .action(async (opts) => {
128
- const { runBrowserDoctor, renderBrowserDoctorReport } = await import('./doctor.js');
129
- const report = await runBrowserDoctor({ live: opts.live, cliVersion: PKG_VERSION });
130
- console.log(renderBrowserDoctorReport(report));
131
- });
132
- program.command('setup')
133
- .description('Interactive setup: verify browser bridge connectivity')
134
- .action(async () => {
135
- const { runSetup } = await import('./setup.js');
136
- await runSetup({ cliVersion: PKG_VERSION });
137
- });
138
- program.command('completion')
139
- .description('Output shell completion script')
140
- .argument('<shell>', 'Shell type: bash, zsh, or fish')
141
- .action((shell) => {
142
- printCompletionScript(shell);
143
- });
144
- // ── Dynamic site commands ──────────────────────────────────────────────────
145
- const registry = getRegistry();
146
- const siteGroups = new Map();
147
- for (const [, cmd] of registry) {
148
- let siteCmd = siteGroups.get(cmd.site);
149
- if (!siteCmd) {
150
- siteCmd = program.command(cmd.site).description(`${cmd.site} commands`);
151
- siteGroups.set(cmd.site, siteCmd);
152
- }
153
- const subCmd = siteCmd.command(cmd.name).description(cmd.description);
154
- // Register positional args first, then named options
155
- const positionalArgs = [];
156
- for (const arg of cmd.args) {
157
- if (arg.positional) {
158
- const bracket = arg.required ? `<${arg.name}>` : `[${arg.name}]`;
159
- subCmd.argument(bracket, arg.help ?? '');
160
- positionalArgs.push(arg);
161
- }
162
- else {
163
- const flag = arg.required ? `--${arg.name} <value>` : `--${arg.name} [value]`;
164
- if (arg.required)
165
- subCmd.requiredOption(flag, arg.help ?? '');
166
- else if (arg.default != null)
167
- subCmd.option(flag, arg.help ?? '', String(arg.default));
168
- else
169
- subCmd.option(flag, arg.help ?? '');
170
- }
171
- }
172
- subCmd.option('-f, --format <fmt>', 'Output format: table, json, yaml, md, csv', 'table').option('-v, --verbose', 'Debug output', false);
173
- subCmd.action(async (...actionArgs) => {
174
- // Commander passes positional args first, then options object, then the Command
175
- const actionOpts = actionArgs[positionalArgs.length] ?? {};
176
- const startTime = Date.now();
177
- const kwargs = {};
178
- // Collect positional args
179
- for (let i = 0; i < positionalArgs.length; i++) {
180
- const arg = positionalArgs[i];
181
- const v = actionArgs[i];
182
- if (v !== undefined)
183
- kwargs[arg.name] = v;
184
- }
185
- // Collect named options
186
- for (const arg of cmd.args) {
187
- if (arg.positional)
188
- continue;
189
- const camelName = arg.name.replace(/-([a-z])/g, (_m, ch) => ch.toUpperCase());
190
- const v = actionOpts[arg.name] ?? actionOpts[camelName];
191
- if (v !== undefined)
192
- kwargs[arg.name] = v;
193
- }
194
- try {
195
- if (actionOpts.verbose)
196
- process.env.OPENCLI_VERBOSE = '1';
197
- let result;
198
- if (cmd.browser) {
199
- result = await browserSession(BrowserBridge, async (page) => {
200
- return runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) });
201
- });
202
- }
203
- else {
204
- result = await executeCommand(cmd, null, kwargs, actionOpts.verbose);
205
- }
206
- if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) {
207
- console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`));
208
- }
209
- renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) });
210
- }
211
- catch (err) {
212
- if (err instanceof CliError) {
213
- console.error(chalk.red(`Error [${err.code}]: ${err.message}`));
214
- if (err.hint)
215
- console.error(chalk.yellow(`Hint: ${err.hint}`));
216
- }
217
- else if (actionOpts.verbose && err.stack) {
218
- console.error(chalk.red(err.stack));
219
- }
220
- else {
221
- console.error(chalk.red(`Error: ${err.message ?? err}`));
222
- }
223
- process.exitCode = 1;
224
- }
225
- });
226
- }
227
- program.parse();
38
+ runCli(BUILTIN_CLIS, USER_CLIS);
package/dist/output.d.ts CHANGED
@@ -7,5 +7,6 @@ export interface RenderOptions {
7
7
  title?: string;
8
8
  elapsed?: number;
9
9
  source?: string;
10
+ footerExtra?: string;
10
11
  }
11
- export declare function render(data: any, opts?: RenderOptions): void;
12
+ export declare function render(data: unknown, opts?: RenderOptions): void;
package/dist/output.js CHANGED
@@ -60,6 +60,8 @@ function renderTable(data, opts) {
60
60
  footer.push(`${opts.elapsed.toFixed(1)}s`);
61
61
  if (opts.source)
62
62
  footer.push(opts.source);
63
+ if (opts.footerExtra)
64
+ footer.push(opts.footerExtra);
63
65
  console.log(chalk.dim(footer.join(' · ')));
64
66
  }
65
67
  function renderJson(data) {
@@ -85,7 +87,7 @@ function renderCsv(data, opts) {
85
87
  for (const row of rows) {
86
88
  console.log(columns.map(c => {
87
89
  const v = String(row[c] ?? '');
88
- return v.includes(',') || v.includes('"') || v.includes('\n')
90
+ return v.includes(',') || v.includes('"') || v.includes('\n') || v.includes('\r')
89
91
  ? `"${v.replace(/"/g, '""')}"` : v;
90
92
  }).join(','));
91
93
  }
@@ -8,6 +8,7 @@ function createMockPage(overrides = {}) {
8
8
  return {
9
9
  goto: vi.fn(),
10
10
  evaluate: vi.fn().mockResolvedValue(null),
11
+ getCookies: vi.fn().mockResolvedValue([]),
11
12
  snapshot: vi.fn().mockResolvedValue(''),
12
13
  click: vi.fn(),
13
14
  typeText: vi.fn(),
@@ -10,7 +10,7 @@
10
10
  import * as fs from 'node:fs';
11
11
  import * as path from 'node:path';
12
12
  import { render } from '../template.js';
13
- import { httpDownload, ytdlpDownload, saveDocument, detectContentType, requiresYtdlp, sanitizeFilename, generateFilename, exportCookiesToNetscape, getTempDir, } from '../../download/index.js';
13
+ import { httpDownload, ytdlpDownload, saveDocument, detectContentType, requiresYtdlp, sanitizeFilename, generateFilename, exportCookiesToNetscape, getTempDir, formatCookieHeader, } from '../../download/index.js';
14
14
  import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
15
15
  /**
16
16
  * Simple async concurrency limiter for downloads.
@@ -33,9 +33,8 @@ async function mapConcurrent(items, limit, fn) {
33
33
  */
34
34
  async function extractBrowserCookies(page, domain) {
35
35
  try {
36
- // Use browser evaluate to get document.cookie
37
- const cookieString = await page.evaluate(`(() => document.cookie)()`);
38
- return typeof cookieString === 'string' ? cookieString : '';
36
+ const cookies = await page.getCookies(domain ? { domain } : {});
37
+ return formatCookieHeader(cookies);
39
38
  }
40
39
  catch {
41
40
  return '';
@@ -46,20 +45,17 @@ async function extractBrowserCookies(page, domain) {
46
45
  */
47
46
  async function extractCookiesArray(page, domain) {
48
47
  try {
49
- const cookieString = await extractBrowserCookies(page);
50
- if (!cookieString)
51
- return [];
52
- return cookieString.split(';').map((c) => {
53
- const [name, ...rest] = c.trim().split('=');
54
- return {
55
- name: name || '',
56
- value: rest.join('=') || '',
57
- domain,
58
- path: '/',
59
- secure: true,
60
- httpOnly: false,
61
- };
62
- }).filter((c) => c.name);
48
+ const cookies = await page.getCookies({ domain });
49
+ return cookies
50
+ .filter((cookie) => cookie.name)
51
+ .map((cookie) => ({
52
+ name: cookie.name,
53
+ value: cookie.value,
54
+ domain: cookie.domain,
55
+ path: cookie.path ?? '/',
56
+ secure: cookie.secure ?? false,
57
+ httpOnly: cookie.httpOnly ?? false,
58
+ }));
63
59
  }
64
60
  catch {
65
61
  return [];
@@ -12,7 +12,7 @@ export declare enum Strategy {
12
12
  export interface Arg {
13
13
  name: string;
14
14
  type?: string;
15
- default?: any;
15
+ default?: unknown;
16
16
  required?: boolean;
17
17
  positional?: boolean;
18
18
  help?: string;
@@ -27,10 +27,11 @@ export interface CliCommand {
27
27
  browser?: boolean;
28
28
  args: Arg[];
29
29
  columns?: string[];
30
- func?: (page: IPage, kwargs: Record<string, any>, debug?: boolean) => Promise<any>;
31
- pipeline?: any[];
30
+ func?: (page: IPage, kwargs: Record<string, any>, debug?: boolean) => Promise<unknown>;
31
+ pipeline?: Record<string, unknown>[];
32
32
  timeoutSeconds?: number;
33
33
  source?: string;
34
+ footerExtra?: (kwargs: Record<string, any>) => string | undefined;
34
35
  }
35
36
  /** Internal extension for lazy-loaded TS modules (not exposed in public API) */
36
37
  export interface InternalCliCommand extends CliCommand {
package/dist/registry.js CHANGED
@@ -11,18 +11,21 @@ export var Strategy;
11
11
  })(Strategy || (Strategy = {}));
12
12
  const _registry = new Map();
13
13
  export function cli(opts) {
14
+ const strategy = opts.strategy ?? (opts.browser === false ? Strategy.PUBLIC : Strategy.COOKIE);
15
+ const browser = opts.browser ?? (strategy !== Strategy.PUBLIC);
14
16
  const cmd = {
15
17
  site: opts.site,
16
18
  name: opts.name,
17
19
  description: opts.description ?? '',
18
20
  domain: opts.domain,
19
- strategy: opts.strategy ?? (opts.browser === false ? Strategy.PUBLIC : Strategy.COOKIE),
20
- browser: opts.browser ?? (opts.strategy === Strategy.PUBLIC ? false : true),
21
+ strategy,
22
+ browser,
21
23
  args: opts.args ?? [],
22
24
  columns: opts.columns,
23
25
  func: opts.func,
24
26
  pipeline: opts.pipeline,
25
27
  timeoutSeconds: opts.timeoutSeconds,
28
+ footerExtra: opts.footerExtra,
26
29
  };
27
30
  const key = fullName(cmd);
28
31
  _registry.set(key, cmd);
package/dist/runtime.d.ts CHANGED
@@ -18,7 +18,10 @@ export declare function withTimeoutMs<T>(promise: Promise<T>, timeoutMs: number,
18
18
  export interface IBrowserFactory {
19
19
  connect(opts?: {
20
20
  timeout?: number;
21
+ workspace?: string;
21
22
  }): Promise<IPage>;
22
23
  close(): Promise<void>;
23
24
  }
24
- export declare function browserSession<T>(BrowserFactory: new () => IBrowserFactory, fn: (page: IPage) => Promise<T>): Promise<T>;
25
+ export declare function browserSession<T>(BrowserFactory: new () => IBrowserFactory, fn: (page: IPage) => Promise<T>, opts?: {
26
+ workspace?: string;
27
+ }): Promise<T>;
package/dist/runtime.js CHANGED
@@ -17,10 +17,10 @@ export function withTimeoutMs(promise, timeoutMs, message) {
17
17
  promise.then((value) => { clearTimeout(timer); resolve(value); }, (error) => { clearTimeout(timer); reject(error); });
18
18
  });
19
19
  }
20
- export async function browserSession(BrowserFactory, fn) {
20
+ export async function browserSession(BrowserFactory, fn, opts = {}) {
21
21
  const mcp = new BrowserFactory();
22
22
  try {
23
- const page = await mcp.connect({ timeout: DEFAULT_BROWSER_CONNECT_TIMEOUT });
23
+ const page = await mcp.connect({ timeout: DEFAULT_BROWSER_CONNECT_TIMEOUT, workspace: opts.workspace });
24
24
  return await fn(page);
25
25
  }
26
26
  finally {
@@ -0,0 +1,4 @@
1
+ /**
2
+ * Injected script for detecting frontend frameworks (Vue, React, Next, Nuxt, etc.)
3
+ */
4
+ export declare function detectFramework(): Record<string, boolean>;