@jackwener/opencli 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/CLI-CREATOR.md +151 -75
  2. package/README.md +11 -8
  3. package/README.zh-CN.md +11 -8
  4. package/SKILL.md +42 -15
  5. package/dist/browser.d.ts +11 -1
  6. package/dist/browser.js +95 -3
  7. package/dist/clis/bilibili/dynamic.d.ts +1 -0
  8. package/dist/clis/bilibili/dynamic.js +33 -0
  9. package/dist/clis/bilibili/ranking.d.ts +1 -0
  10. package/dist/clis/bilibili/ranking.js +24 -0
  11. package/dist/clis/bilibili/subtitle.d.ts +1 -0
  12. package/dist/clis/bilibili/subtitle.js +86 -0
  13. package/dist/clis/reddit/frontpage.yaml +30 -0
  14. package/dist/clis/reddit/hot.yaml +3 -2
  15. package/dist/clis/reddit/search.yaml +34 -0
  16. package/dist/clis/reddit/subreddit.yaml +39 -0
  17. package/dist/clis/twitter/bookmarks.yaml +85 -0
  18. package/dist/clis/twitter/profile.d.ts +1 -0
  19. package/dist/clis/twitter/profile.js +56 -0
  20. package/dist/clis/twitter/search.d.ts +1 -0
  21. package/dist/clis/twitter/search.js +60 -0
  22. package/dist/clis/twitter/timeline.d.ts +1 -0
  23. package/dist/clis/twitter/timeline.js +47 -0
  24. package/dist/clis/xiaohongshu/user.d.ts +1 -0
  25. package/dist/clis/xiaohongshu/user.js +40 -0
  26. package/dist/clis/xueqiu/feed.yaml +53 -0
  27. package/dist/clis/xueqiu/hot-stock.yaml +49 -0
  28. package/dist/clis/xueqiu/hot.yaml +46 -0
  29. package/dist/clis/xueqiu/search.yaml +53 -0
  30. package/dist/clis/xueqiu/stock.yaml +67 -0
  31. package/dist/clis/xueqiu/watchlist.yaml +46 -0
  32. package/dist/clis/zhihu/hot.yaml +6 -2
  33. package/dist/clis/zhihu/search.yaml +3 -1
  34. package/dist/engine.d.ts +1 -1
  35. package/dist/engine.js +9 -1
  36. package/dist/explore.js +50 -0
  37. package/dist/main.d.ts +1 -1
  38. package/dist/main.js +12 -5
  39. package/dist/pipeline/steps/browser.js +4 -8
  40. package/dist/pipeline/steps/fetch.js +19 -6
  41. package/dist/pipeline/steps/intercept.js +56 -29
  42. package/dist/pipeline/steps/tap.js +8 -6
  43. package/dist/pipeline/template.js +3 -1
  44. package/dist/pipeline/template.test.js +6 -0
  45. package/dist/types.d.ts +11 -1
  46. package/package.json +1 -1
  47. package/src/browser.ts +101 -6
  48. package/src/clis/bilibili/dynamic.ts +34 -0
  49. package/src/clis/bilibili/ranking.ts +25 -0
  50. package/src/clis/bilibili/subtitle.ts +100 -0
  51. package/src/clis/reddit/frontpage.yaml +30 -0
  52. package/src/clis/reddit/hot.yaml +3 -2
  53. package/src/clis/reddit/search.yaml +34 -0
  54. package/src/clis/reddit/subreddit.yaml +39 -0
  55. package/src/clis/twitter/bookmarks.yaml +85 -0
  56. package/src/clis/twitter/profile.ts +61 -0
  57. package/src/clis/twitter/search.ts +65 -0
  58. package/src/clis/twitter/timeline.ts +50 -0
  59. package/src/clis/xiaohongshu/user.ts +45 -0
  60. package/src/clis/xueqiu/feed.yaml +53 -0
  61. package/src/clis/xueqiu/hot-stock.yaml +49 -0
  62. package/src/clis/xueqiu/hot.yaml +46 -0
  63. package/src/clis/xueqiu/search.yaml +53 -0
  64. package/src/clis/xueqiu/stock.yaml +67 -0
  65. package/src/clis/xueqiu/watchlist.yaml +46 -0
  66. package/src/clis/zhihu/hot.yaml +6 -2
  67. package/src/clis/zhihu/search.yaml +3 -1
  68. package/src/engine.ts +10 -1
  69. package/src/explore.ts +51 -0
  70. package/src/main.ts +11 -5
  71. package/src/pipeline/steps/browser.ts +4 -7
  72. package/src/pipeline/steps/fetch.ts +22 -6
  73. package/src/pipeline/steps/intercept.ts +58 -28
  74. package/src/pipeline/steps/tap.ts +8 -6
  75. package/src/pipeline/template.test.ts +6 -0
  76. package/src/pipeline/template.ts +3 -1
  77. package/src/types.ts +4 -1
  78. package/dist/clis/index.d.ts +0 -22
  79. package/dist/clis/index.js +0 -34
  80. package/src/clis/index.ts +0 -46
@@ -0,0 +1,49 @@
1
+ site: xueqiu
2
+ name: hot-stock
3
+ description: 获取雪球热门股票榜
4
+ domain: xueqiu.com
5
+ browser: true
6
+
7
+ args:
8
+ limit:
9
+ type: int
10
+ default: 20
11
+ description: 返回数量,默认 20,最大 50
12
+ type:
13
+ type: str
14
+ default: "10"
15
+ description: 榜单类型 10=人气榜(默认) 12=关注榜
16
+
17
+ pipeline:
18
+ - navigate: https://xueqiu.com
19
+ - evaluate: |
20
+ (async () => {
21
+ const count = ${{ args.limit }};
22
+ const type = ${{ args.type | json }};
23
+ const resp = await fetch(`https://stock.xueqiu.com/v5/stock/hot_stock/list.json?size=${count}&type=${type}`, {credentials: 'include'});
24
+ if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
25
+ const d = await resp.json();
26
+ if (!d.data || !d.data.items) throw new Error('获取失败');
27
+ return d.data.items.map((s, i) => ({
28
+ rank: i + 1,
29
+ symbol: s.symbol,
30
+ name: s.name,
31
+ price: s.current,
32
+ changePercent: s.percent != null ? s.percent.toFixed(2) + '%' : null,
33
+ heat: s.value,
34
+ rank_change: s.rank_change,
35
+ url: 'https://xueqiu.com/S/' + s.symbol
36
+ }));
37
+ })()
38
+
39
+ - map:
40
+ rank: ${{ item.rank }}
41
+ symbol: ${{ item.symbol }}
42
+ name: ${{ item.name }}
43
+ price: ${{ item.price }}
44
+ changePercent: ${{ item.changePercent }}
45
+ heat: ${{ item.heat }}
46
+
47
+ - limit: ${{ args.limit }}
48
+
49
+ columns: [rank, symbol, name, price, changePercent, heat]
@@ -0,0 +1,46 @@
1
+ site: xueqiu
2
+ name: hot
3
+ description: 获取雪球热门动态
4
+ domain: xueqiu.com
5
+ browser: true
6
+
7
+ args:
8
+ limit:
9
+ type: int
10
+ default: 20
11
+ description: 返回数量,默认 20,最大 50
12
+
13
+ pipeline:
14
+ - navigate: https://xueqiu.com
15
+ - evaluate: |
16
+ (async () => {
17
+ const resp = await fetch('https://xueqiu.com/statuses/hot/listV3.json?source=hot&page=1', {credentials: 'include'});
18
+ if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
19
+ const d = await resp.json();
20
+ const list = d.list || [];
21
+
22
+ const strip = (html) => (html || '').replace(/<[^>]+>/g, '').replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').trim();
23
+ return list.map((item, i) => {
24
+ const user = item.user || {};
25
+ return {
26
+ rank: i + 1,
27
+ text: strip(item.description).substring(0, 200),
28
+ url: 'https://xueqiu.com/' + user.id + '/' + item.id,
29
+ author: user.screen_name,
30
+ likes: item.fav_count,
31
+ retweets: item.retweet_count,
32
+ replies: item.reply_count
33
+ };
34
+ });
35
+ })()
36
+
37
+ - map:
38
+ rank: ${{ item.rank }}
39
+ author: ${{ item.author }}
40
+ text: ${{ item.text }}
41
+ likes: ${{ item.likes }}
42
+ url: ${{ item.url }}
43
+
44
+ - limit: ${{ args.limit }}
45
+
46
+ columns: [rank, author, text, likes, url]
@@ -0,0 +1,53 @@
1
+ site: xueqiu
2
+ name: search
3
+ description: 搜索雪球股票(代码或名称)
4
+ domain: xueqiu.com
5
+ browser: true
6
+
7
+ args:
8
+ query:
9
+ type: str
10
+ description: 搜索关键词,如 茅台、AAPL、腾讯
11
+ limit:
12
+ type: int
13
+ default: 10
14
+ description: 返回数量,默认 10
15
+
16
+ pipeline:
17
+ - navigate: https://xueqiu.com
18
+ - evaluate: |
19
+ (async () => {
20
+ const query = ${{ args.query | json }};
21
+ const count = ${{ args.limit }};
22
+ if (!query) throw new Error('Missing argument: query');
23
+ const resp = await fetch(`https://xueqiu.com/stock/search.json?code=${encodeURIComponent(query)}&size=${count}`, {credentials: 'include'});
24
+ if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
25
+ const d = await resp.json();
26
+ return (d.stocks || []).map(s => {
27
+ let symbol = '';
28
+ if (s.exchange === 'SH' || s.exchange === 'SZ' || s.exchange === 'BJ') {
29
+ symbol = s.code.startsWith(s.exchange) ? s.code : s.exchange + s.code;
30
+ } else {
31
+ symbol = s.code;
32
+ }
33
+ return {
34
+ symbol: symbol,
35
+ name: s.name,
36
+ exchange: s.exchange,
37
+ price: s.current,
38
+ changePercent: s.percentage != null ? s.percentage.toFixed(2) + '%' : null,
39
+ url: 'https://xueqiu.com/S/' + symbol
40
+ };
41
+ });
42
+ })()
43
+
44
+ - map:
45
+ symbol: ${{ item.symbol }}
46
+ name: ${{ item.name }}
47
+ exchange: ${{ item.exchange }}
48
+ price: ${{ item.price }}
49
+ changePercent: ${{ item.changePercent }}
50
+
51
+ - limit: ${{ args.limit }}
52
+
53
+ columns: [symbol, name, exchange, price, changePercent]
@@ -0,0 +1,67 @@
1
+ site: xueqiu
2
+ name: stock
3
+ description: 获取雪球股票实时行情
4
+ domain: xueqiu.com
5
+ browser: true
6
+
7
+ args:
8
+ symbol:
9
+ type: str
10
+ description: 股票代码,如 SH600519、SZ000858、AAPL、00700
11
+
12
+ pipeline:
13
+ - navigate: https://xueqiu.com
14
+ - evaluate: |
15
+ (async () => {
16
+ const symbol = (${{ args.symbol | json }} || '').toUpperCase();
17
+ if (!symbol) throw new Error('Missing argument: symbol');
18
+ const resp = await fetch(`https://stock.xueqiu.com/v5/stock/batch/quote.json?symbol=${encodeURIComponent(symbol)}`, {credentials: 'include'});
19
+ if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
20
+ const d = await resp.json();
21
+ if (!d.data || !d.data.items || d.data.items.length === 0) throw new Error('未找到股票: ' + symbol);
22
+
23
+ function fmtAmount(v) {
24
+ if (v == null) return null;
25
+ if (Math.abs(v) >= 1e12) return (v / 1e12).toFixed(2) + '万亿';
26
+ if (Math.abs(v) >= 1e8) return (v / 1e8).toFixed(2) + '亿';
27
+ if (Math.abs(v) >= 1e4) return (v / 1e4).toFixed(2) + '万';
28
+ return v.toString();
29
+ }
30
+
31
+ const item = d.data.items[0];
32
+ const q = item.quote || {};
33
+ const m = item.market || {};
34
+
35
+ return [{
36
+ name: q.name,
37
+ symbol: q.symbol,
38
+ exchange: q.exchange,
39
+ currency: q.currency,
40
+ price: q.current,
41
+ change: q.chg,
42
+ changePercent: q.percent != null ? q.percent.toFixed(2) + '%' : null,
43
+ open: q.open,
44
+ high: q.high,
45
+ low: q.low,
46
+ prevClose: q.last_close,
47
+ amplitude: q.amplitude != null ? q.amplitude.toFixed(2) + '%' : null,
48
+ volume: q.volume,
49
+ amount: fmtAmount(q.amount),
50
+ turnover_rate: q.turnover_rate != null ? q.turnover_rate.toFixed(2) + '%' : null,
51
+ marketCap: fmtAmount(q.market_capital),
52
+ floatMarketCap: fmtAmount(q.float_market_capital),
53
+ ytdPercent: q.current_year_percent != null ? q.current_year_percent.toFixed(2) + '%' : null,
54
+ market_status: m.status || null,
55
+ time: q.timestamp ? new Date(q.timestamp).toISOString() : null,
56
+ url: 'https://xueqiu.com/S/' + q.symbol
57
+ }];
58
+ })()
59
+
60
+ - map:
61
+ name: ${{ item.name }}
62
+ symbol: ${{ item.symbol }}
63
+ price: ${{ item.price }}
64
+ changePercent: ${{ item.changePercent }}
65
+ marketCap: ${{ item.marketCap }}
66
+
67
+ columns: [name, symbol, price, changePercent, marketCap]
@@ -0,0 +1,46 @@
1
+ site: xueqiu
2
+ name: watchlist
3
+ description: 获取雪球自选股列表
4
+ domain: xueqiu.com
5
+ browser: true
6
+
7
+ args:
8
+ category:
9
+ type: str # using str to prevent parsing issues like 01
10
+ default: "1"
11
+ description: "分类:1=自选(默认) 2=持仓 3=关注"
12
+ limit:
13
+ type: int
14
+ default: 100
15
+ description: 默认 100
16
+
17
+ pipeline:
18
+ - navigate: https://xueqiu.com
19
+ - evaluate: |
20
+ (async () => {
21
+ const category = parseInt(${{ args.category | json }}) || 1;
22
+ const resp = await fetch(`https://stock.xueqiu.com/v5/stock/portfolio/stock/list.json?size=100&category=${category}&pid=-1`, {credentials: 'include'});
23
+ if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
24
+ const d = await resp.json();
25
+ if (!d.data || !d.data.stocks) throw new Error('获取失败,可能未登录');
26
+
27
+ return d.data.stocks.map(s => ({
28
+ symbol: s.symbol,
29
+ name: s.name,
30
+ price: s.current,
31
+ change: s.chg,
32
+ changePercent: s.percent != null ? s.percent.toFixed(2) + '%' : null,
33
+ volume: s.volume,
34
+ url: 'https://xueqiu.com/S/' + s.symbol
35
+ }));
36
+ })()
37
+
38
+ - map:
39
+ symbol: ${{ item.symbol }}
40
+ name: ${{ item.name }}
41
+ price: ${{ item.price }}
42
+ changePercent: ${{ item.changePercent }}
43
+
44
+ - limit: ${{ args.limit }}
45
+
46
+ columns: [symbol, name, price, changePercent]
@@ -17,12 +17,16 @@ pipeline:
17
17
  const res = await fetch('https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total?limit=50', {
18
18
  credentials: 'include'
19
19
  });
20
- const d = await res.json();
20
+ const text = await res.text();
21
+ const d = JSON.parse(
22
+ text.replace(/("id"\s*:\s*)(\d{16,})/g, '$1"$2"')
23
+ );
21
24
  return (d?.data || []).map((item) => {
22
25
  const t = item.target || {};
26
+ const questionId = t.id == null ? '' : String(t.id);
23
27
  return {
24
28
  title: t.title,
25
- url: 'https://www.zhihu.com/question/' + t.id,
29
+ url: 'https://www.zhihu.com/question/' + questionId,
26
30
  answer_count: t.answer_count,
27
31
  follower_count: t.follower_count,
28
32
  heat: item.detail_text || '',
@@ -19,7 +19,9 @@ pipeline:
19
19
  - evaluate: |
20
20
  (async () => {
21
21
  const strip = (html) => (html || '').replace(/<[^>]+>/g, '').replace(/&nbsp;/g, ' ').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&amp;/g, '&').replace(/<em>/g, '').replace(/<\/em>/g, '').trim();
22
- const res = await fetch('https://www.zhihu.com/api/v4/search_v3?q=' + encodeURIComponent('${{ args.keyword }}') + '&t=general&offset=0&limit=${{ args.limit }}', {
22
+ const keyword = ${{ args.keyword | json }};
23
+ const limit = ${{ args.limit }};
24
+ const res = await fetch('https://www.zhihu.com/api/v4/search_v3?q=' + encodeURIComponent(keyword) + '&t=general&offset=0&limit=' + limit, {
23
25
  credentials: 'include'
24
26
  });
25
27
  const d = await res.json();
package/dist/engine.d.ts CHANGED
@@ -3,5 +3,5 @@
3
3
  */
4
4
  import { type CliCommand } from './registry.js';
5
5
  import type { IPage } from './types.js';
6
- export declare function discoverClis(...dirs: string[]): void;
6
+ export declare function discoverClis(...dirs: string[]): Promise<void>;
7
7
  export declare function executeCommand(cmd: CliCommand, page: IPage | null, kwargs: Record<string, any>, debug?: boolean): Promise<any>;
package/dist/engine.js CHANGED
@@ -6,7 +6,8 @@ import * as path from 'node:path';
6
6
  import yaml from 'js-yaml';
7
7
  import { Strategy, registerCommand } from './registry.js';
8
8
  import { executePipeline } from './pipeline.js';
9
- export function discoverClis(...dirs) {
9
+ export async function discoverClis(...dirs) {
10
+ const promises = [];
10
11
  for (const dir of dirs) {
11
12
  if (!fs.existsSync(dir))
12
13
  continue;
@@ -19,9 +20,16 @@ export function discoverClis(...dirs) {
19
20
  if (file.endsWith('.yaml') || file.endsWith('.yml')) {
20
21
  registerYamlCli(filePath, site);
21
22
  }
23
+ else if (file.endsWith('.js')) {
24
+ // Dynamic import of compiled adapter modules
25
+ promises.push(import(`file://${filePath}`).catch((err) => {
26
+ process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`);
27
+ }));
28
+ }
22
29
  }
23
30
  }
24
31
  }
32
+ await Promise.all(promises);
25
33
  }
26
34
  function registerYamlCli(filePath, defaultSite) {
27
35
  try {
package/dist/explore.js CHANGED
@@ -175,6 +175,9 @@ function scoreEndpoint(ep) {
175
175
  s += 2;
176
176
  if (ep.status === 200)
177
177
  s += 2;
178
+ // Anti-Bot Empty Value Detection: penalize JSON endpoints returning empty data
179
+ if (ep.responseAnalysis && ep.responseAnalysis.itemCount === 0 && ep.contentType.includes('json'))
180
+ s -= 3;
178
181
  return s;
179
182
  }
180
183
  function inferCapabilityName(url, goal) {
@@ -266,6 +269,28 @@ const STORE_DISCOVER_JS = `
266
269
  return stores;
267
270
  }
268
271
  `;
272
+ // ── Auto-Interaction (Fuzzing) ─────────────────────────────────────────────
273
+ const INTERACT_FUZZ_JS = `
274
+ async () => {
275
+ const sleep = ms => new Promise(r => setTimeout(r, ms));
276
+ const clickables = Array.from(document.querySelectorAll(
277
+ 'button, [role="button"], [role="tab"], .tab, .btn, a[href="javascript:void(0)"], a[href="#"]'
278
+ )).slice(0, 15); // limit to 15 to avoid endless loops
279
+
280
+ let clicked = 0;
281
+ for (const el of clickables) {
282
+ try {
283
+ const rect = el.getBoundingClientRect();
284
+ if (rect.width > 0 && rect.height > 0) {
285
+ el.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
286
+ clicked++;
287
+ await sleep(300); // give it time to trigger network
288
+ }
289
+ } catch {}
290
+ }
291
+ return clicked;
292
+ }
293
+ `;
269
294
  // ── Main explore function ──────────────────────────────────────────────────
270
295
  export async function exploreUrl(url, opts) {
271
296
  const waitSeconds = opts.waitSeconds ?? 3.0;
@@ -283,6 +308,31 @@ export async function exploreUrl(url, opts) {
283
308
  catch { }
284
309
  await page.wait(1);
285
310
  }
311
+ // Step 2.5: Interactive Fuzzing (if requested)
312
+ if (opts.auto) {
313
+ try {
314
+ // First: targeted clicks by label (e.g. "字幕", "CC", "评论")
315
+ if (opts.clickLabels?.length) {
316
+ for (const label of opts.clickLabels) {
317
+ const safeLabel = label.replace(/'/g, "\\'");
318
+ await page.evaluate(`
319
+ (() => {
320
+ const el = [...document.querySelectorAll('button, [role="button"], [role="tab"], a, span')]
321
+ .find(e => e.textContent && e.textContent.trim().includes('${safeLabel}'));
322
+ if (el) el.click();
323
+ })()
324
+ `);
325
+ await page.wait(1);
326
+ }
327
+ }
328
+ // Then: blind fuzzing on generic interactive elements
329
+ const clicks = await page.evaluate(INTERACT_FUZZ_JS);
330
+ await page.wait(2); // wait for XHRs to settle
331
+ }
332
+ catch (e) {
333
+ // fuzzing is best-effort, don't fail the whole explore
334
+ }
335
+ }
286
336
  // Step 3: Read page metadata
287
337
  const metadata = await readPageMetadata(page);
288
338
  // Step 4: Capture network traffic
package/dist/main.d.ts CHANGED
@@ -2,4 +2,4 @@
2
2
  /**
3
3
  * opencli — Make any website your CLI. AI-powered.
4
4
  */
5
- import './clis/index.js';
5
+ export {};
package/dist/main.js CHANGED
@@ -11,7 +11,6 @@ import chalk from 'chalk';
11
11
  import { discoverClis, executeCommand } from './engine.js';
12
12
  import { fullName, getRegistry, strategyLabel } from './registry.js';
13
13
  import { render as renderOutput } from './output.js';
14
- import './clis/index.js';
15
14
  import { PlaywrightMCP } from './browser.js';
16
15
  import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
17
16
  const __filename = fileURLToPath(import.meta.url);
@@ -21,7 +20,7 @@ const USER_CLIS = path.join(os.homedir(), '.opencli', 'clis');
21
20
  // Read version from package.json (single source of truth)
22
21
  const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
23
22
  const PKG_VERSION = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version ?? '0.0.0';
24
- discoverClis(BUILTIN_CLIS, USER_CLIS);
23
+ await discoverClis(BUILTIN_CLIS, USER_CLIS);
25
24
  const program = new Command();
26
25
  program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
27
26
  // ── Built-in commands ──────────────────────────────────────────────────────
@@ -57,8 +56,8 @@ program.command('validate').description('Validate CLI definitions').argument('[t
57
56
  .action(async (target) => { const { validateClisWithTarget, renderValidationReport } = await import('./validate.js'); console.log(renderValidationReport(validateClisWithTarget([BUILTIN_CLIS, USER_CLIS], target))); });
58
57
  program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false)
59
58
  .action(async (target, opts) => { const { verifyClis, renderVerifyReport } = await import('./verify.js'); const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke }); console.log(renderVerifyReport(r)); process.exitCode = r.ok ? 0 : 1; });
60
- program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3')
61
- .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait) }))); });
59
+ program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3').option('--auto', 'Enable interactive fuzzing (simulate clicks to trigger lazy APIs)').option('--click <labels>', 'Comma-separated labels to click before fuzzing (e.g. "字幕,CC,评论")')
60
+ .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); const clickLabels = opts.click ? opts.click.split(',').map((s) => s.trim()) : undefined; console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait), auto: opts.auto, clickLabels }))); });
62
61
  program.command('synthesize').description('Synthesize CLIs from explore').argument('<target>').option('--top <n>', '', '3')
63
62
  .action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); });
64
63
  program.command('generate').description('One-shot: explore → synthesize → register').argument('<url>').option('--goal <text>').option('--site <name>')
@@ -116,10 +115,18 @@ for (const [, cmd] of registry) {
116
115
  else {
117
116
  result = await executeCommand(cmd, null, kwargs, actionOpts.verbose);
118
117
  }
118
+ if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) {
119
+ console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`));
120
+ }
119
121
  renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) });
120
122
  }
121
123
  catch (err) {
122
- console.error(chalk.red(`Error: ${err.message ?? err}`));
124
+ if (actionOpts.verbose && err.stack) {
125
+ console.error(chalk.red(err.stack));
126
+ }
127
+ else {
128
+ console.error(chalk.red(`Error: ${err.message ?? err}`));
129
+ }
123
130
  process.exitCode = 1;
124
131
  }
125
132
  });
@@ -27,14 +27,10 @@ export async function stepWait(page, params, data, args) {
27
27
  await page.wait(params);
28
28
  else if (typeof params === 'object' && params) {
29
29
  if ('text' in params) {
30
- const timeout = params.timeout ?? 10;
31
- const start = Date.now();
32
- while ((Date.now() - start) / 1000 < timeout) {
33
- const snap = await page.snapshot({ raw: true });
34
- if (typeof snap === 'string' && snap.includes(params.text))
35
- break;
36
- await page.wait(0.5);
37
- }
30
+ await page.wait({
31
+ text: String(render(params.text, { args, data })),
32
+ timeout: params.timeout
33
+ });
38
34
  }
39
35
  else if ('time' in params)
40
36
  await page.wait(Number(params.time));
@@ -2,6 +2,20 @@
2
2
  * Pipeline step: fetch — HTTP API requests.
3
3
  */
4
4
  import { render } from '../template.js';
5
+ /** Simple async concurrency limiter */
6
+ async function mapConcurrent(items, limit, fn) {
7
+ const results = new Array(items.length);
8
+ let index = 0;
9
+ async function worker() {
10
+ while (index < items.length) {
11
+ const i = index++;
12
+ results[i] = await fn(items[i], i);
13
+ }
14
+ }
15
+ const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
16
+ await Promise.all(workers);
17
+ return results;
18
+ }
5
19
  /** Single URL fetch helper */
6
20
  async function fetchSingle(page, url, method, queryParams, headers, args, data) {
7
21
  const renderedParams = {};
@@ -38,12 +52,11 @@ export async function stepFetch(page, params, data, args) {
38
52
  const urlTemplate = String(urlOrObj);
39
53
  // Per-item fetch when data is array and URL references item
40
54
  if (Array.isArray(data) && urlTemplate.includes('item')) {
41
- const results = [];
42
- for (let i = 0; i < data.length; i++) {
43
- const itemUrl = String(render(urlTemplate, { args, data, item: data[i], index: i }));
44
- results.push(await fetchSingle(page, itemUrl, method, queryParams, headers, args, data));
45
- }
46
- return results;
55
+ const concurrency = typeof params?.concurrency === 'number' ? params.concurrency : 5;
56
+ return mapConcurrent(data, concurrency, async (item, index) => {
57
+ const itemUrl = String(render(urlTemplate, { args, data, item, index }));
58
+ return fetchSingle(page, itemUrl, method, queryParams, headers, args, data);
59
+ });
47
60
  }
48
61
  const url = render(urlOrObj, { args, data });
49
62
  return fetchSingle(page, String(url), method, queryParams, headers, args, data);
@@ -10,7 +10,54 @@ export async function stepIntercept(page, params, data, args) {
10
10
  const selectPath = cfg.select ?? null;
11
11
  if (!capturePattern)
12
12
  return data;
13
- // Step 1: Execute the trigger action
13
+ // Step 1: Inject fetch/XHR interceptor BEFORE trigger
14
+ await page.evaluate(`
15
+ () => {
16
+ window.__opencli_intercepted = window.__opencli_intercepted || [];
17
+ const pattern = ${JSON.stringify(capturePattern)};
18
+
19
+ if (!window.__opencli_fetch_patched) {
20
+ const origFetch = window.fetch;
21
+ window.fetch = async function(...args) {
22
+ const reqUrl = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
23
+ const response = await origFetch.apply(this, args);
24
+ setTimeout(async () => {
25
+ try {
26
+ if (reqUrl.includes(pattern)) {
27
+ const clone = response.clone();
28
+ const json = await clone.json();
29
+ window.__opencli_intercepted.push(json);
30
+ }
31
+ } catch(e) {}
32
+ }, 0);
33
+ return response;
34
+ };
35
+ window.__opencli_fetch_patched = true;
36
+ }
37
+
38
+ if (!window.__opencli_xhr_patched) {
39
+ const XHR = XMLHttpRequest.prototype;
40
+ const open = XHR.open;
41
+ const send = XHR.send;
42
+ XHR.open = function(method, url, ...args) {
43
+ this._reqUrl = url;
44
+ return open.call(this, method, url, ...args);
45
+ };
46
+ XHR.send = function(...args) {
47
+ this.addEventListener('load', function() {
48
+ try {
49
+ if (this._reqUrl && this._reqUrl.includes(pattern)) {
50
+ window.__opencli_intercepted.push(JSON.parse(this.responseText));
51
+ }
52
+ } catch(e) {}
53
+ });
54
+ return send.apply(this, args);
55
+ };
56
+ window.__opencli_xhr_patched = true;
57
+ }
58
+ }
59
+ `);
60
+ // Step 2: Execute the trigger action
14
61
  if (trigger.startsWith('navigate:')) {
15
62
  const url = render(trigger.slice('navigate:'.length), { args, data });
16
63
  await page.goto(String(url));
@@ -27,36 +74,16 @@ export async function stepIntercept(page, params, data, args) {
27
74
  else if (trigger === 'scroll') {
28
75
  await page.scroll('down');
29
76
  }
30
- // Step 2: Wait a bit for network requests to fire
77
+ // Step 3: Wait a bit for network requests to fire
31
78
  await page.wait(Math.min(timeout, 3));
32
- // Step 3: Get network requests and find matching ones
33
- const rawNetwork = await page.networkRequests(false);
34
- const matchingResponses = [];
35
- if (typeof rawNetwork === 'string') {
36
- const lines = rawNetwork.split('\n');
37
- for (const line of lines) {
38
- const match = line.match(/\[?(GET|POST)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i);
39
- if (match) {
40
- const [, , url, status] = match;
41
- if (url.includes(capturePattern) && status === '200') {
42
- try {
43
- const body = await page.evaluate(`
44
- async () => {
45
- try {
46
- const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
47
- if (!resp.ok) return null;
48
- return await resp.json();
49
- } catch { return null; }
50
- }
51
- `);
52
- if (body)
53
- matchingResponses.push(body);
54
- }
55
- catch { }
56
- }
57
- }
58
- }
79
+ // Step 4: Retrieve captured data
80
+ const matchingResponses = await page.evaluate(`
81
+ () => {
82
+ const data = window.__opencli_intercepted || [];
83
+ window.__opencli_intercepted = []; // clear after reading
84
+ return data;
59
85
  }
86
+ `);
60
87
  // Step 4: Select from response if specified
61
88
  let result = matchingResponses.length === 1 ? matchingResponses[0] :
62
89
  matchingResponses.length > 1 ? matchingResponses : data;