@jackwener/opencli 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI-CREATOR.md +151 -75
- package/README.md +11 -8
- package/README.zh-CN.md +11 -8
- package/SKILL.md +42 -15
- package/dist/browser.d.ts +11 -1
- package/dist/browser.js +95 -3
- package/dist/clis/bilibili/dynamic.d.ts +1 -0
- package/dist/clis/bilibili/dynamic.js +33 -0
- package/dist/clis/bilibili/ranking.d.ts +1 -0
- package/dist/clis/bilibili/ranking.js +24 -0
- package/dist/clis/bilibili/subtitle.d.ts +1 -0
- package/dist/clis/bilibili/subtitle.js +86 -0
- package/dist/clis/reddit/frontpage.yaml +30 -0
- package/dist/clis/reddit/hot.yaml +3 -2
- package/dist/clis/reddit/search.yaml +34 -0
- package/dist/clis/reddit/subreddit.yaml +39 -0
- package/dist/clis/twitter/bookmarks.yaml +85 -0
- package/dist/clis/twitter/profile.d.ts +1 -0
- package/dist/clis/twitter/profile.js +56 -0
- package/dist/clis/twitter/search.d.ts +1 -0
- package/dist/clis/twitter/search.js +60 -0
- package/dist/clis/twitter/timeline.d.ts +1 -0
- package/dist/clis/twitter/timeline.js +47 -0
- package/dist/clis/xiaohongshu/user.d.ts +1 -0
- package/dist/clis/xiaohongshu/user.js +40 -0
- package/dist/clis/xueqiu/feed.yaml +53 -0
- package/dist/clis/xueqiu/hot-stock.yaml +49 -0
- package/dist/clis/xueqiu/hot.yaml +46 -0
- package/dist/clis/xueqiu/search.yaml +53 -0
- package/dist/clis/xueqiu/stock.yaml +67 -0
- package/dist/clis/xueqiu/watchlist.yaml +46 -0
- package/dist/clis/zhihu/hot.yaml +6 -2
- package/dist/clis/zhihu/search.yaml +3 -1
- package/dist/engine.d.ts +1 -1
- package/dist/engine.js +9 -1
- package/dist/explore.js +50 -0
- package/dist/main.d.ts +1 -1
- package/dist/main.js +12 -5
- package/dist/pipeline/steps/browser.js +4 -8
- package/dist/pipeline/steps/fetch.js +19 -6
- package/dist/pipeline/steps/intercept.js +56 -29
- package/dist/pipeline/steps/tap.js +8 -6
- package/dist/pipeline/template.js +3 -1
- package/dist/pipeline/template.test.js +6 -0
- package/dist/types.d.ts +11 -1
- package/package.json +1 -1
- package/src/browser.ts +101 -6
- package/src/clis/bilibili/dynamic.ts +34 -0
- package/src/clis/bilibili/ranking.ts +25 -0
- package/src/clis/bilibili/subtitle.ts +100 -0
- package/src/clis/reddit/frontpage.yaml +30 -0
- package/src/clis/reddit/hot.yaml +3 -2
- package/src/clis/reddit/search.yaml +34 -0
- package/src/clis/reddit/subreddit.yaml +39 -0
- package/src/clis/twitter/bookmarks.yaml +85 -0
- package/src/clis/twitter/profile.ts +61 -0
- package/src/clis/twitter/search.ts +65 -0
- package/src/clis/twitter/timeline.ts +50 -0
- package/src/clis/xiaohongshu/user.ts +45 -0
- package/src/clis/xueqiu/feed.yaml +53 -0
- package/src/clis/xueqiu/hot-stock.yaml +49 -0
- package/src/clis/xueqiu/hot.yaml +46 -0
- package/src/clis/xueqiu/search.yaml +53 -0
- package/src/clis/xueqiu/stock.yaml +67 -0
- package/src/clis/xueqiu/watchlist.yaml +46 -0
- package/src/clis/zhihu/hot.yaml +6 -2
- package/src/clis/zhihu/search.yaml +3 -1
- package/src/engine.ts +10 -1
- package/src/explore.ts +51 -0
- package/src/main.ts +11 -5
- package/src/pipeline/steps/browser.ts +4 -7
- package/src/pipeline/steps/fetch.ts +22 -6
- package/src/pipeline/steps/intercept.ts +58 -28
- package/src/pipeline/steps/tap.ts +8 -6
- package/src/pipeline/template.test.ts +6 -0
- package/src/pipeline/template.ts +3 -1
- package/src/types.ts +4 -1
- package/dist/clis/index.d.ts +0 -22
- package/dist/clis/index.js +0 -34
- package/src/clis/index.ts +0 -46
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
site: xueqiu
|
|
2
|
+
name: hot-stock
|
|
3
|
+
description: 获取雪球热门股票榜
|
|
4
|
+
domain: xueqiu.com
|
|
5
|
+
browser: true
|
|
6
|
+
|
|
7
|
+
args:
|
|
8
|
+
limit:
|
|
9
|
+
type: int
|
|
10
|
+
default: 20
|
|
11
|
+
description: 返回数量,默认 20,最大 50
|
|
12
|
+
type:
|
|
13
|
+
type: str
|
|
14
|
+
default: "10"
|
|
15
|
+
description: 榜单类型 10=人气榜(默认) 12=关注榜
|
|
16
|
+
|
|
17
|
+
pipeline:
|
|
18
|
+
- navigate: https://xueqiu.com
|
|
19
|
+
- evaluate: |
|
|
20
|
+
(async () => {
|
|
21
|
+
const count = ${{ args.limit }};
|
|
22
|
+
const type = ${{ args.type | json }};
|
|
23
|
+
const resp = await fetch(`https://stock.xueqiu.com/v5/stock/hot_stock/list.json?size=${count}&type=${type}`, {credentials: 'include'});
|
|
24
|
+
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
|
25
|
+
const d = await resp.json();
|
|
26
|
+
if (!d.data || !d.data.items) throw new Error('获取失败');
|
|
27
|
+
return d.data.items.map((s, i) => ({
|
|
28
|
+
rank: i + 1,
|
|
29
|
+
symbol: s.symbol,
|
|
30
|
+
name: s.name,
|
|
31
|
+
price: s.current,
|
|
32
|
+
changePercent: s.percent != null ? s.percent.toFixed(2) + '%' : null,
|
|
33
|
+
heat: s.value,
|
|
34
|
+
rank_change: s.rank_change,
|
|
35
|
+
url: 'https://xueqiu.com/S/' + s.symbol
|
|
36
|
+
}));
|
|
37
|
+
})()
|
|
38
|
+
|
|
39
|
+
- map:
|
|
40
|
+
rank: ${{ item.rank }}
|
|
41
|
+
symbol: ${{ item.symbol }}
|
|
42
|
+
name: ${{ item.name }}
|
|
43
|
+
price: ${{ item.price }}
|
|
44
|
+
changePercent: ${{ item.changePercent }}
|
|
45
|
+
heat: ${{ item.heat }}
|
|
46
|
+
|
|
47
|
+
- limit: ${{ args.limit }}
|
|
48
|
+
|
|
49
|
+
columns: [rank, symbol, name, price, changePercent, heat]
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
site: xueqiu
|
|
2
|
+
name: hot
|
|
3
|
+
description: 获取雪球热门动态
|
|
4
|
+
domain: xueqiu.com
|
|
5
|
+
browser: true
|
|
6
|
+
|
|
7
|
+
args:
|
|
8
|
+
limit:
|
|
9
|
+
type: int
|
|
10
|
+
default: 20
|
|
11
|
+
description: 返回数量,默认 20,最大 50
|
|
12
|
+
|
|
13
|
+
pipeline:
|
|
14
|
+
- navigate: https://xueqiu.com
|
|
15
|
+
- evaluate: |
|
|
16
|
+
(async () => {
|
|
17
|
+
const resp = await fetch('https://xueqiu.com/statuses/hot/listV3.json?source=hot&page=1', {credentials: 'include'});
|
|
18
|
+
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
|
19
|
+
const d = await resp.json();
|
|
20
|
+
const list = d.list || [];
|
|
21
|
+
|
|
22
|
+
const strip = (html) => (html || '').replace(/<[^>]+>/g, '').replace(/ /g, ' ').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').trim();
|
|
23
|
+
return list.map((item, i) => {
|
|
24
|
+
const user = item.user || {};
|
|
25
|
+
return {
|
|
26
|
+
rank: i + 1,
|
|
27
|
+
text: strip(item.description).substring(0, 200),
|
|
28
|
+
url: 'https://xueqiu.com/' + user.id + '/' + item.id,
|
|
29
|
+
author: user.screen_name,
|
|
30
|
+
likes: item.fav_count,
|
|
31
|
+
retweets: item.retweet_count,
|
|
32
|
+
replies: item.reply_count
|
|
33
|
+
};
|
|
34
|
+
});
|
|
35
|
+
})()
|
|
36
|
+
|
|
37
|
+
- map:
|
|
38
|
+
rank: ${{ item.rank }}
|
|
39
|
+
author: ${{ item.author }}
|
|
40
|
+
text: ${{ item.text }}
|
|
41
|
+
likes: ${{ item.likes }}
|
|
42
|
+
url: ${{ item.url }}
|
|
43
|
+
|
|
44
|
+
- limit: ${{ args.limit }}
|
|
45
|
+
|
|
46
|
+
columns: [rank, author, text, likes, url]
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
site: xueqiu
|
|
2
|
+
name: search
|
|
3
|
+
description: 搜索雪球股票(代码或名称)
|
|
4
|
+
domain: xueqiu.com
|
|
5
|
+
browser: true
|
|
6
|
+
|
|
7
|
+
args:
|
|
8
|
+
query:
|
|
9
|
+
type: str
|
|
10
|
+
description: 搜索关键词,如 茅台、AAPL、腾讯
|
|
11
|
+
limit:
|
|
12
|
+
type: int
|
|
13
|
+
default: 10
|
|
14
|
+
description: 返回数量,默认 10
|
|
15
|
+
|
|
16
|
+
pipeline:
|
|
17
|
+
- navigate: https://xueqiu.com
|
|
18
|
+
- evaluate: |
|
|
19
|
+
(async () => {
|
|
20
|
+
const query = ${{ args.query | json }};
|
|
21
|
+
const count = ${{ args.limit }};
|
|
22
|
+
if (!query) throw new Error('Missing argument: query');
|
|
23
|
+
const resp = await fetch(`https://xueqiu.com/stock/search.json?code=${encodeURIComponent(query)}&size=${count}`, {credentials: 'include'});
|
|
24
|
+
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
|
25
|
+
const d = await resp.json();
|
|
26
|
+
return (d.stocks || []).map(s => {
|
|
27
|
+
let symbol = '';
|
|
28
|
+
if (s.exchange === 'SH' || s.exchange === 'SZ' || s.exchange === 'BJ') {
|
|
29
|
+
symbol = s.code.startsWith(s.exchange) ? s.code : s.exchange + s.code;
|
|
30
|
+
} else {
|
|
31
|
+
symbol = s.code;
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
symbol: symbol,
|
|
35
|
+
name: s.name,
|
|
36
|
+
exchange: s.exchange,
|
|
37
|
+
price: s.current,
|
|
38
|
+
changePercent: s.percentage != null ? s.percentage.toFixed(2) + '%' : null,
|
|
39
|
+
url: 'https://xueqiu.com/S/' + symbol
|
|
40
|
+
};
|
|
41
|
+
});
|
|
42
|
+
})()
|
|
43
|
+
|
|
44
|
+
- map:
|
|
45
|
+
symbol: ${{ item.symbol }}
|
|
46
|
+
name: ${{ item.name }}
|
|
47
|
+
exchange: ${{ item.exchange }}
|
|
48
|
+
price: ${{ item.price }}
|
|
49
|
+
changePercent: ${{ item.changePercent }}
|
|
50
|
+
|
|
51
|
+
- limit: ${{ args.limit }}
|
|
52
|
+
|
|
53
|
+
columns: [symbol, name, exchange, price, changePercent]
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
site: xueqiu
|
|
2
|
+
name: stock
|
|
3
|
+
description: 获取雪球股票实时行情
|
|
4
|
+
domain: xueqiu.com
|
|
5
|
+
browser: true
|
|
6
|
+
|
|
7
|
+
args:
|
|
8
|
+
symbol:
|
|
9
|
+
type: str
|
|
10
|
+
description: 股票代码,如 SH600519、SZ000858、AAPL、00700
|
|
11
|
+
|
|
12
|
+
pipeline:
|
|
13
|
+
- navigate: https://xueqiu.com
|
|
14
|
+
- evaluate: |
|
|
15
|
+
(async () => {
|
|
16
|
+
const symbol = (${{ args.symbol | json }} || '').toUpperCase();
|
|
17
|
+
if (!symbol) throw new Error('Missing argument: symbol');
|
|
18
|
+
const resp = await fetch(`https://stock.xueqiu.com/v5/stock/batch/quote.json?symbol=${encodeURIComponent(symbol)}`, {credentials: 'include'});
|
|
19
|
+
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
|
20
|
+
const d = await resp.json();
|
|
21
|
+
if (!d.data || !d.data.items || d.data.items.length === 0) throw new Error('未找到股票: ' + symbol);
|
|
22
|
+
|
|
23
|
+
function fmtAmount(v) {
|
|
24
|
+
if (v == null) return null;
|
|
25
|
+
if (Math.abs(v) >= 1e12) return (v / 1e12).toFixed(2) + '万亿';
|
|
26
|
+
if (Math.abs(v) >= 1e8) return (v / 1e8).toFixed(2) + '亿';
|
|
27
|
+
if (Math.abs(v) >= 1e4) return (v / 1e4).toFixed(2) + '万';
|
|
28
|
+
return v.toString();
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const item = d.data.items[0];
|
|
32
|
+
const q = item.quote || {};
|
|
33
|
+
const m = item.market || {};
|
|
34
|
+
|
|
35
|
+
return [{
|
|
36
|
+
name: q.name,
|
|
37
|
+
symbol: q.symbol,
|
|
38
|
+
exchange: q.exchange,
|
|
39
|
+
currency: q.currency,
|
|
40
|
+
price: q.current,
|
|
41
|
+
change: q.chg,
|
|
42
|
+
changePercent: q.percent != null ? q.percent.toFixed(2) + '%' : null,
|
|
43
|
+
open: q.open,
|
|
44
|
+
high: q.high,
|
|
45
|
+
low: q.low,
|
|
46
|
+
prevClose: q.last_close,
|
|
47
|
+
amplitude: q.amplitude != null ? q.amplitude.toFixed(2) + '%' : null,
|
|
48
|
+
volume: q.volume,
|
|
49
|
+
amount: fmtAmount(q.amount),
|
|
50
|
+
turnover_rate: q.turnover_rate != null ? q.turnover_rate.toFixed(2) + '%' : null,
|
|
51
|
+
marketCap: fmtAmount(q.market_capital),
|
|
52
|
+
floatMarketCap: fmtAmount(q.float_market_capital),
|
|
53
|
+
ytdPercent: q.current_year_percent != null ? q.current_year_percent.toFixed(2) + '%' : null,
|
|
54
|
+
market_status: m.status || null,
|
|
55
|
+
time: q.timestamp ? new Date(q.timestamp).toISOString() : null,
|
|
56
|
+
url: 'https://xueqiu.com/S/' + q.symbol
|
|
57
|
+
}];
|
|
58
|
+
})()
|
|
59
|
+
|
|
60
|
+
- map:
|
|
61
|
+
name: ${{ item.name }}
|
|
62
|
+
symbol: ${{ item.symbol }}
|
|
63
|
+
price: ${{ item.price }}
|
|
64
|
+
changePercent: ${{ item.changePercent }}
|
|
65
|
+
marketCap: ${{ item.marketCap }}
|
|
66
|
+
|
|
67
|
+
columns: [name, symbol, price, changePercent, marketCap]
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
site: xueqiu
|
|
2
|
+
name: watchlist
|
|
3
|
+
description: 获取雪球自选股列表
|
|
4
|
+
domain: xueqiu.com
|
|
5
|
+
browser: true
|
|
6
|
+
|
|
7
|
+
args:
|
|
8
|
+
category:
|
|
9
|
+
type: str # using str to prevent parsing issues like 01
|
|
10
|
+
default: "1"
|
|
11
|
+
description: "分类:1=自选(默认) 2=持仓 3=关注"
|
|
12
|
+
limit:
|
|
13
|
+
type: int
|
|
14
|
+
default: 100
|
|
15
|
+
description: 默认 100
|
|
16
|
+
|
|
17
|
+
pipeline:
|
|
18
|
+
- navigate: https://xueqiu.com
|
|
19
|
+
- evaluate: |
|
|
20
|
+
(async () => {
|
|
21
|
+
const category = parseInt(${{ args.category | json }}) || 1;
|
|
22
|
+
const resp = await fetch(`https://stock.xueqiu.com/v5/stock/portfolio/stock/list.json?size=100&category=${category}&pid=-1`, {credentials: 'include'});
|
|
23
|
+
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
|
24
|
+
const d = await resp.json();
|
|
25
|
+
if (!d.data || !d.data.stocks) throw new Error('获取失败,可能未登录');
|
|
26
|
+
|
|
27
|
+
return d.data.stocks.map(s => ({
|
|
28
|
+
symbol: s.symbol,
|
|
29
|
+
name: s.name,
|
|
30
|
+
price: s.current,
|
|
31
|
+
change: s.chg,
|
|
32
|
+
changePercent: s.percent != null ? s.percent.toFixed(2) + '%' : null,
|
|
33
|
+
volume: s.volume,
|
|
34
|
+
url: 'https://xueqiu.com/S/' + s.symbol
|
|
35
|
+
}));
|
|
36
|
+
})()
|
|
37
|
+
|
|
38
|
+
- map:
|
|
39
|
+
symbol: ${{ item.symbol }}
|
|
40
|
+
name: ${{ item.name }}
|
|
41
|
+
price: ${{ item.price }}
|
|
42
|
+
changePercent: ${{ item.changePercent }}
|
|
43
|
+
|
|
44
|
+
- limit: ${{ args.limit }}
|
|
45
|
+
|
|
46
|
+
columns: [symbol, name, price, changePercent]
|
package/dist/clis/zhihu/hot.yaml
CHANGED
|
@@ -17,12 +17,16 @@ pipeline:
|
|
|
17
17
|
const res = await fetch('https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total?limit=50', {
|
|
18
18
|
credentials: 'include'
|
|
19
19
|
});
|
|
20
|
-
const
|
|
20
|
+
const text = await res.text();
|
|
21
|
+
const d = JSON.parse(
|
|
22
|
+
text.replace(/("id"\s*:\s*)(\d{16,})/g, '$1"$2"')
|
|
23
|
+
);
|
|
21
24
|
return (d?.data || []).map((item) => {
|
|
22
25
|
const t = item.target || {};
|
|
26
|
+
const questionId = t.id == null ? '' : String(t.id);
|
|
23
27
|
return {
|
|
24
28
|
title: t.title,
|
|
25
|
-
url: 'https://www.zhihu.com/question/' +
|
|
29
|
+
url: 'https://www.zhihu.com/question/' + questionId,
|
|
26
30
|
answer_count: t.answer_count,
|
|
27
31
|
follower_count: t.follower_count,
|
|
28
32
|
heat: item.detail_text || '',
|
|
@@ -19,7 +19,9 @@ pipeline:
|
|
|
19
19
|
- evaluate: |
|
|
20
20
|
(async () => {
|
|
21
21
|
const strip = (html) => (html || '').replace(/<[^>]+>/g, '').replace(/ /g, ' ').replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&').replace(/<em>/g, '').replace(/<\/em>/g, '').trim();
|
|
22
|
-
const
|
|
22
|
+
const keyword = ${{ args.keyword | json }};
|
|
23
|
+
const limit = ${{ args.limit }};
|
|
24
|
+
const res = await fetch('https://www.zhihu.com/api/v4/search_v3?q=' + encodeURIComponent(keyword) + '&t=general&offset=0&limit=' + limit, {
|
|
23
25
|
credentials: 'include'
|
|
24
26
|
});
|
|
25
27
|
const d = await res.json();
|
package/dist/engine.d.ts
CHANGED
|
@@ -3,5 +3,5 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { type CliCommand } from './registry.js';
|
|
5
5
|
import type { IPage } from './types.js';
|
|
6
|
-
export declare function discoverClis(...dirs: string[]): void
|
|
6
|
+
export declare function discoverClis(...dirs: string[]): Promise<void>;
|
|
7
7
|
export declare function executeCommand(cmd: CliCommand, page: IPage | null, kwargs: Record<string, any>, debug?: boolean): Promise<any>;
|
package/dist/engine.js
CHANGED
|
@@ -6,7 +6,8 @@ import * as path from 'node:path';
|
|
|
6
6
|
import yaml from 'js-yaml';
|
|
7
7
|
import { Strategy, registerCommand } from './registry.js';
|
|
8
8
|
import { executePipeline } from './pipeline.js';
|
|
9
|
-
export function discoverClis(...dirs) {
|
|
9
|
+
export async function discoverClis(...dirs) {
|
|
10
|
+
const promises = [];
|
|
10
11
|
for (const dir of dirs) {
|
|
11
12
|
if (!fs.existsSync(dir))
|
|
12
13
|
continue;
|
|
@@ -19,9 +20,16 @@ export function discoverClis(...dirs) {
|
|
|
19
20
|
if (file.endsWith('.yaml') || file.endsWith('.yml')) {
|
|
20
21
|
registerYamlCli(filePath, site);
|
|
21
22
|
}
|
|
23
|
+
else if (file.endsWith('.js')) {
|
|
24
|
+
// Dynamic import of compiled adapter modules
|
|
25
|
+
promises.push(import(`file://${filePath}`).catch((err) => {
|
|
26
|
+
process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`);
|
|
27
|
+
}));
|
|
28
|
+
}
|
|
22
29
|
}
|
|
23
30
|
}
|
|
24
31
|
}
|
|
32
|
+
await Promise.all(promises);
|
|
25
33
|
}
|
|
26
34
|
function registerYamlCli(filePath, defaultSite) {
|
|
27
35
|
try {
|
package/dist/explore.js
CHANGED
|
@@ -175,6 +175,9 @@ function scoreEndpoint(ep) {
|
|
|
175
175
|
s += 2;
|
|
176
176
|
if (ep.status === 200)
|
|
177
177
|
s += 2;
|
|
178
|
+
// Anti-Bot Empty Value Detection: penalize JSON endpoints returning empty data
|
|
179
|
+
if (ep.responseAnalysis && ep.responseAnalysis.itemCount === 0 && ep.contentType.includes('json'))
|
|
180
|
+
s -= 3;
|
|
178
181
|
return s;
|
|
179
182
|
}
|
|
180
183
|
function inferCapabilityName(url, goal) {
|
|
@@ -266,6 +269,28 @@ const STORE_DISCOVER_JS = `
|
|
|
266
269
|
return stores;
|
|
267
270
|
}
|
|
268
271
|
`;
|
|
272
|
+
// ── Auto-Interaction (Fuzzing) ─────────────────────────────────────────────
|
|
273
|
+
const INTERACT_FUZZ_JS = `
|
|
274
|
+
async () => {
|
|
275
|
+
const sleep = ms => new Promise(r => setTimeout(r, ms));
|
|
276
|
+
const clickables = Array.from(document.querySelectorAll(
|
|
277
|
+
'button, [role="button"], [role="tab"], .tab, .btn, a[href="javascript:void(0)"], a[href="#"]'
|
|
278
|
+
)).slice(0, 15); // limit to 15 to avoid endless loops
|
|
279
|
+
|
|
280
|
+
let clicked = 0;
|
|
281
|
+
for (const el of clickables) {
|
|
282
|
+
try {
|
|
283
|
+
const rect = el.getBoundingClientRect();
|
|
284
|
+
if (rect.width > 0 && rect.height > 0) {
|
|
285
|
+
el.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
|
|
286
|
+
clicked++;
|
|
287
|
+
await sleep(300); // give it time to trigger network
|
|
288
|
+
}
|
|
289
|
+
} catch {}
|
|
290
|
+
}
|
|
291
|
+
return clicked;
|
|
292
|
+
}
|
|
293
|
+
`;
|
|
269
294
|
// ── Main explore function ──────────────────────────────────────────────────
|
|
270
295
|
export async function exploreUrl(url, opts) {
|
|
271
296
|
const waitSeconds = opts.waitSeconds ?? 3.0;
|
|
@@ -283,6 +308,31 @@ export async function exploreUrl(url, opts) {
|
|
|
283
308
|
catch { }
|
|
284
309
|
await page.wait(1);
|
|
285
310
|
}
|
|
311
|
+
// Step 2.5: Interactive Fuzzing (if requested)
|
|
312
|
+
if (opts.auto) {
|
|
313
|
+
try {
|
|
314
|
+
// First: targeted clicks by label (e.g. "字幕", "CC", "评论")
|
|
315
|
+
if (opts.clickLabels?.length) {
|
|
316
|
+
for (const label of opts.clickLabels) {
|
|
317
|
+
const safeLabel = label.replace(/'/g, "\\'");
|
|
318
|
+
await page.evaluate(`
|
|
319
|
+
(() => {
|
|
320
|
+
const el = [...document.querySelectorAll('button, [role="button"], [role="tab"], a, span')]
|
|
321
|
+
.find(e => e.textContent && e.textContent.trim().includes('${safeLabel}'));
|
|
322
|
+
if (el) el.click();
|
|
323
|
+
})()
|
|
324
|
+
`);
|
|
325
|
+
await page.wait(1);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
// Then: blind fuzzing on generic interactive elements
|
|
329
|
+
const clicks = await page.evaluate(INTERACT_FUZZ_JS);
|
|
330
|
+
await page.wait(2); // wait for XHRs to settle
|
|
331
|
+
}
|
|
332
|
+
catch (e) {
|
|
333
|
+
// fuzzing is best-effort, don't fail the whole explore
|
|
334
|
+
}
|
|
335
|
+
}
|
|
286
336
|
// Step 3: Read page metadata
|
|
287
337
|
const metadata = await readPageMetadata(page);
|
|
288
338
|
// Step 4: Capture network traffic
|
package/dist/main.d.ts
CHANGED
package/dist/main.js
CHANGED
|
@@ -11,7 +11,6 @@ import chalk from 'chalk';
|
|
|
11
11
|
import { discoverClis, executeCommand } from './engine.js';
|
|
12
12
|
import { fullName, getRegistry, strategyLabel } from './registry.js';
|
|
13
13
|
import { render as renderOutput } from './output.js';
|
|
14
|
-
import './clis/index.js';
|
|
15
14
|
import { PlaywrightMCP } from './browser.js';
|
|
16
15
|
import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
|
|
17
16
|
const __filename = fileURLToPath(import.meta.url);
|
|
@@ -21,7 +20,7 @@ const USER_CLIS = path.join(os.homedir(), '.opencli', 'clis');
|
|
|
21
20
|
// Read version from package.json (single source of truth)
|
|
22
21
|
const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
|
|
23
22
|
const PKG_VERSION = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version ?? '0.0.0';
|
|
24
|
-
discoverClis(BUILTIN_CLIS, USER_CLIS);
|
|
23
|
+
await discoverClis(BUILTIN_CLIS, USER_CLIS);
|
|
25
24
|
const program = new Command();
|
|
26
25
|
program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
|
|
27
26
|
// ── Built-in commands ──────────────────────────────────────────────────────
|
|
@@ -57,8 +56,8 @@ program.command('validate').description('Validate CLI definitions').argument('[t
|
|
|
57
56
|
.action(async (target) => { const { validateClisWithTarget, renderValidationReport } = await import('./validate.js'); console.log(renderValidationReport(validateClisWithTarget([BUILTIN_CLIS, USER_CLIS], target))); });
|
|
58
57
|
program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false)
|
|
59
58
|
.action(async (target, opts) => { const { verifyClis, renderVerifyReport } = await import('./verify.js'); const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke }); console.log(renderVerifyReport(r)); process.exitCode = r.ok ? 0 : 1; });
|
|
60
|
-
program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3')
|
|
61
|
-
.action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait) }))); });
|
|
59
|
+
program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3').option('--auto', 'Enable interactive fuzzing (simulate clicks to trigger lazy APIs)').option('--click <labels>', 'Comma-separated labels to click before fuzzing (e.g. "字幕,CC,评论")')
|
|
60
|
+
.action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); const clickLabels = opts.click ? opts.click.split(',').map((s) => s.trim()) : undefined; console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait), auto: opts.auto, clickLabels }))); });
|
|
62
61
|
program.command('synthesize').description('Synthesize CLIs from explore').argument('<target>').option('--top <n>', '', '3')
|
|
63
62
|
.action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); });
|
|
64
63
|
program.command('generate').description('One-shot: explore → synthesize → register').argument('<url>').option('--goal <text>').option('--site <name>')
|
|
@@ -116,10 +115,18 @@ for (const [, cmd] of registry) {
|
|
|
116
115
|
else {
|
|
117
116
|
result = await executeCommand(cmd, null, kwargs, actionOpts.verbose);
|
|
118
117
|
}
|
|
118
|
+
if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) {
|
|
119
|
+
console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`));
|
|
120
|
+
}
|
|
119
121
|
renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) });
|
|
120
122
|
}
|
|
121
123
|
catch (err) {
|
|
122
|
-
|
|
124
|
+
if (actionOpts.verbose && err.stack) {
|
|
125
|
+
console.error(chalk.red(err.stack));
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
console.error(chalk.red(`Error: ${err.message ?? err}`));
|
|
129
|
+
}
|
|
123
130
|
process.exitCode = 1;
|
|
124
131
|
}
|
|
125
132
|
});
|
|
@@ -27,14 +27,10 @@ export async function stepWait(page, params, data, args) {
|
|
|
27
27
|
await page.wait(params);
|
|
28
28
|
else if (typeof params === 'object' && params) {
|
|
29
29
|
if ('text' in params) {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
if (typeof snap === 'string' && snap.includes(params.text))
|
|
35
|
-
break;
|
|
36
|
-
await page.wait(0.5);
|
|
37
|
-
}
|
|
30
|
+
await page.wait({
|
|
31
|
+
text: String(render(params.text, { args, data })),
|
|
32
|
+
timeout: params.timeout
|
|
33
|
+
});
|
|
38
34
|
}
|
|
39
35
|
else if ('time' in params)
|
|
40
36
|
await page.wait(Number(params.time));
|
|
@@ -2,6 +2,20 @@
|
|
|
2
2
|
* Pipeline step: fetch — HTTP API requests.
|
|
3
3
|
*/
|
|
4
4
|
import { render } from '../template.js';
|
|
5
|
+
/** Simple async concurrency limiter */
|
|
6
|
+
async function mapConcurrent(items, limit, fn) {
|
|
7
|
+
const results = new Array(items.length);
|
|
8
|
+
let index = 0;
|
|
9
|
+
async function worker() {
|
|
10
|
+
while (index < items.length) {
|
|
11
|
+
const i = index++;
|
|
12
|
+
results[i] = await fn(items[i], i);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
|
|
16
|
+
await Promise.all(workers);
|
|
17
|
+
return results;
|
|
18
|
+
}
|
|
5
19
|
/** Single URL fetch helper */
|
|
6
20
|
async function fetchSingle(page, url, method, queryParams, headers, args, data) {
|
|
7
21
|
const renderedParams = {};
|
|
@@ -38,12 +52,11 @@ export async function stepFetch(page, params, data, args) {
|
|
|
38
52
|
const urlTemplate = String(urlOrObj);
|
|
39
53
|
// Per-item fetch when data is array and URL references item
|
|
40
54
|
if (Array.isArray(data) && urlTemplate.includes('item')) {
|
|
41
|
-
const
|
|
42
|
-
|
|
43
|
-
const itemUrl = String(render(urlTemplate, { args, data, item
|
|
44
|
-
|
|
45
|
-
}
|
|
46
|
-
return results;
|
|
55
|
+
const concurrency = typeof params?.concurrency === 'number' ? params.concurrency : 5;
|
|
56
|
+
return mapConcurrent(data, concurrency, async (item, index) => {
|
|
57
|
+
const itemUrl = String(render(urlTemplate, { args, data, item, index }));
|
|
58
|
+
return fetchSingle(page, itemUrl, method, queryParams, headers, args, data);
|
|
59
|
+
});
|
|
47
60
|
}
|
|
48
61
|
const url = render(urlOrObj, { args, data });
|
|
49
62
|
return fetchSingle(page, String(url), method, queryParams, headers, args, data);
|
|
@@ -10,7 +10,54 @@ export async function stepIntercept(page, params, data, args) {
|
|
|
10
10
|
const selectPath = cfg.select ?? null;
|
|
11
11
|
if (!capturePattern)
|
|
12
12
|
return data;
|
|
13
|
-
// Step 1:
|
|
13
|
+
// Step 1: Inject fetch/XHR interceptor BEFORE trigger
|
|
14
|
+
await page.evaluate(`
|
|
15
|
+
() => {
|
|
16
|
+
window.__opencli_intercepted = window.__opencli_intercepted || [];
|
|
17
|
+
const pattern = ${JSON.stringify(capturePattern)};
|
|
18
|
+
|
|
19
|
+
if (!window.__opencli_fetch_patched) {
|
|
20
|
+
const origFetch = window.fetch;
|
|
21
|
+
window.fetch = async function(...args) {
|
|
22
|
+
const reqUrl = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
|
|
23
|
+
const response = await origFetch.apply(this, args);
|
|
24
|
+
setTimeout(async () => {
|
|
25
|
+
try {
|
|
26
|
+
if (reqUrl.includes(pattern)) {
|
|
27
|
+
const clone = response.clone();
|
|
28
|
+
const json = await clone.json();
|
|
29
|
+
window.__opencli_intercepted.push(json);
|
|
30
|
+
}
|
|
31
|
+
} catch(e) {}
|
|
32
|
+
}, 0);
|
|
33
|
+
return response;
|
|
34
|
+
};
|
|
35
|
+
window.__opencli_fetch_patched = true;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
if (!window.__opencli_xhr_patched) {
|
|
39
|
+
const XHR = XMLHttpRequest.prototype;
|
|
40
|
+
const open = XHR.open;
|
|
41
|
+
const send = XHR.send;
|
|
42
|
+
XHR.open = function(method, url, ...args) {
|
|
43
|
+
this._reqUrl = url;
|
|
44
|
+
return open.call(this, method, url, ...args);
|
|
45
|
+
};
|
|
46
|
+
XHR.send = function(...args) {
|
|
47
|
+
this.addEventListener('load', function() {
|
|
48
|
+
try {
|
|
49
|
+
if (this._reqUrl && this._reqUrl.includes(pattern)) {
|
|
50
|
+
window.__opencli_intercepted.push(JSON.parse(this.responseText));
|
|
51
|
+
}
|
|
52
|
+
} catch(e) {}
|
|
53
|
+
});
|
|
54
|
+
return send.apply(this, args);
|
|
55
|
+
};
|
|
56
|
+
window.__opencli_xhr_patched = true;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
`);
|
|
60
|
+
// Step 2: Execute the trigger action
|
|
14
61
|
if (trigger.startsWith('navigate:')) {
|
|
15
62
|
const url = render(trigger.slice('navigate:'.length), { args, data });
|
|
16
63
|
await page.goto(String(url));
|
|
@@ -27,36 +74,16 @@ export async function stepIntercept(page, params, data, args) {
|
|
|
27
74
|
else if (trigger === 'scroll') {
|
|
28
75
|
await page.scroll('down');
|
|
29
76
|
}
|
|
30
|
-
// Step
|
|
77
|
+
// Step 3: Wait a bit for network requests to fire
|
|
31
78
|
await page.wait(Math.min(timeout, 3));
|
|
32
|
-
// Step
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
const match = line.match(/\[?(GET|POST)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i);
|
|
39
|
-
if (match) {
|
|
40
|
-
const [, , url, status] = match;
|
|
41
|
-
if (url.includes(capturePattern) && status === '200') {
|
|
42
|
-
try {
|
|
43
|
-
const body = await page.evaluate(`
|
|
44
|
-
async () => {
|
|
45
|
-
try {
|
|
46
|
-
const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
|
|
47
|
-
if (!resp.ok) return null;
|
|
48
|
-
return await resp.json();
|
|
49
|
-
} catch { return null; }
|
|
50
|
-
}
|
|
51
|
-
`);
|
|
52
|
-
if (body)
|
|
53
|
-
matchingResponses.push(body);
|
|
54
|
-
}
|
|
55
|
-
catch { }
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
}
|
|
79
|
+
// Step 4: Retrieve captured data
|
|
80
|
+
const matchingResponses = await page.evaluate(`
|
|
81
|
+
() => {
|
|
82
|
+
const data = window.__opencli_intercepted || [];
|
|
83
|
+
window.__opencli_intercepted = []; // clear after reading
|
|
84
|
+
return data;
|
|
59
85
|
}
|
|
86
|
+
`);
|
|
60
87
|
// Step 4: Select from response if specified
|
|
61
88
|
let result = matchingResponses.length === 1 ? matchingResponses[0] :
|
|
62
89
|
matchingResponses.length > 1 ? matchingResponses : data;
|