@jackwener/opencli 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI-CREATOR.md +151 -75
- package/README.md +11 -8
- package/README.zh-CN.md +11 -8
- package/SKILL.md +42 -15
- package/dist/browser.d.ts +11 -1
- package/dist/browser.js +95 -3
- package/dist/clis/bilibili/dynamic.d.ts +1 -0
- package/dist/clis/bilibili/dynamic.js +33 -0
- package/dist/clis/bilibili/ranking.d.ts +1 -0
- package/dist/clis/bilibili/ranking.js +24 -0
- package/dist/clis/bilibili/subtitle.d.ts +1 -0
- package/dist/clis/bilibili/subtitle.js +86 -0
- package/dist/clis/reddit/frontpage.yaml +30 -0
- package/dist/clis/reddit/hot.yaml +3 -2
- package/dist/clis/reddit/search.yaml +34 -0
- package/dist/clis/reddit/subreddit.yaml +39 -0
- package/dist/clis/twitter/bookmarks.yaml +85 -0
- package/dist/clis/twitter/profile.d.ts +1 -0
- package/dist/clis/twitter/profile.js +56 -0
- package/dist/clis/twitter/search.d.ts +1 -0
- package/dist/clis/twitter/search.js +60 -0
- package/dist/clis/twitter/timeline.d.ts +1 -0
- package/dist/clis/twitter/timeline.js +47 -0
- package/dist/clis/xiaohongshu/user.d.ts +1 -0
- package/dist/clis/xiaohongshu/user.js +40 -0
- package/dist/clis/xueqiu/feed.yaml +53 -0
- package/dist/clis/xueqiu/hot-stock.yaml +49 -0
- package/dist/clis/xueqiu/hot.yaml +46 -0
- package/dist/clis/xueqiu/search.yaml +53 -0
- package/dist/clis/xueqiu/stock.yaml +67 -0
- package/dist/clis/xueqiu/watchlist.yaml +46 -0
- package/dist/clis/zhihu/hot.yaml +6 -2
- package/dist/clis/zhihu/search.yaml +3 -1
- package/dist/engine.d.ts +1 -1
- package/dist/engine.js +9 -1
- package/dist/explore.js +50 -0
- package/dist/main.d.ts +1 -1
- package/dist/main.js +12 -5
- package/dist/pipeline/steps/browser.js +4 -8
- package/dist/pipeline/steps/fetch.js +19 -6
- package/dist/pipeline/steps/intercept.js +56 -29
- package/dist/pipeline/steps/tap.js +8 -6
- package/dist/pipeline/template.js +3 -1
- package/dist/pipeline/template.test.js +6 -0
- package/dist/types.d.ts +11 -1
- package/package.json +1 -1
- package/src/browser.ts +101 -6
- package/src/clis/bilibili/dynamic.ts +34 -0
- package/src/clis/bilibili/ranking.ts +25 -0
- package/src/clis/bilibili/subtitle.ts +100 -0
- package/src/clis/reddit/frontpage.yaml +30 -0
- package/src/clis/reddit/hot.yaml +3 -2
- package/src/clis/reddit/search.yaml +34 -0
- package/src/clis/reddit/subreddit.yaml +39 -0
- package/src/clis/twitter/bookmarks.yaml +85 -0
- package/src/clis/twitter/profile.ts +61 -0
- package/src/clis/twitter/search.ts +65 -0
- package/src/clis/twitter/timeline.ts +50 -0
- package/src/clis/xiaohongshu/user.ts +45 -0
- package/src/clis/xueqiu/feed.yaml +53 -0
- package/src/clis/xueqiu/hot-stock.yaml +49 -0
- package/src/clis/xueqiu/hot.yaml +46 -0
- package/src/clis/xueqiu/search.yaml +53 -0
- package/src/clis/xueqiu/stock.yaml +67 -0
- package/src/clis/xueqiu/watchlist.yaml +46 -0
- package/src/clis/zhihu/hot.yaml +6 -2
- package/src/clis/zhihu/search.yaml +3 -1
- package/src/engine.ts +10 -1
- package/src/explore.ts +51 -0
- package/src/main.ts +11 -5
- package/src/pipeline/steps/browser.ts +4 -7
- package/src/pipeline/steps/fetch.ts +22 -6
- package/src/pipeline/steps/intercept.ts +58 -28
- package/src/pipeline/steps/tap.ts +8 -6
- package/src/pipeline/template.test.ts +6 -0
- package/src/pipeline/template.ts +3 -1
- package/src/types.ts +4 -1
- package/dist/clis/index.d.ts +0 -22
- package/dist/clis/index.js +0 -34
- package/src/clis/index.ts +0 -46
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { cli, Strategy } from '../../registry.js';
|
|
2
|
+
|
|
3
|
+
cli({
|
|
4
|
+
site: 'twitter',
|
|
5
|
+
name: 'search',
|
|
6
|
+
description: 'Search Twitter/X for tweets',
|
|
7
|
+
domain: 'x.com',
|
|
8
|
+
strategy: Strategy.INTERCEPT, // Use intercept strategy
|
|
9
|
+
browser: true,
|
|
10
|
+
args: [
|
|
11
|
+
{ name: 'query', type: 'string', required: true },
|
|
12
|
+
{ name: 'limit', type: 'int', default: 15 },
|
|
13
|
+
],
|
|
14
|
+
columns: ['id', 'author', 'text', 'likes', 'views', 'url'],
|
|
15
|
+
func: async (page, kwargs) => {
|
|
16
|
+
// 1. Navigate to the search page
|
|
17
|
+
const q = encodeURIComponent(kwargs.query);
|
|
18
|
+
await page.goto(`https://x.com/search?q=${q}&f=top`);
|
|
19
|
+
await page.wait(5);
|
|
20
|
+
|
|
21
|
+
// 2. Inject XHR interceptor
|
|
22
|
+
await page.installInterceptor('SearchTimeline');
|
|
23
|
+
|
|
24
|
+
// 3. Trigger API by scrolling
|
|
25
|
+
await page.autoScroll({ times: 3, delayMs: 2000 });
|
|
26
|
+
|
|
27
|
+
// 4. Retrieve data
|
|
28
|
+
const requests = await page.getInterceptedRequests();
|
|
29
|
+
if (!requests || requests.length === 0) return [];
|
|
30
|
+
|
|
31
|
+
let results: any[] = [];
|
|
32
|
+
for (const req of requests) {
|
|
33
|
+
try {
|
|
34
|
+
const insts = req.data.data.search_by_raw_query.search_timeline.timeline.instructions;
|
|
35
|
+
const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries');
|
|
36
|
+
if (!addEntries) continue;
|
|
37
|
+
|
|
38
|
+
for (const entry of addEntries.entries) {
|
|
39
|
+
if (!entry.entryId.startsWith('tweet-')) continue;
|
|
40
|
+
|
|
41
|
+
let tweet = entry.content?.itemContent?.tweet_results?.result;
|
|
42
|
+
if (!tweet) continue;
|
|
43
|
+
|
|
44
|
+
// Handle retweet wrapping
|
|
45
|
+
if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) {
|
|
46
|
+
tweet = tweet.tweet;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
results.push({
|
|
50
|
+
id: tweet.rest_id,
|
|
51
|
+
author: tweet.core?.user_results?.result?.legacy?.screen_name || 'unknown',
|
|
52
|
+
text: tweet.legacy?.full_text || '',
|
|
53
|
+
likes: tweet.legacy?.favorite_count || 0,
|
|
54
|
+
views: tweet.views?.count || '0',
|
|
55
|
+
url: `https://x.com/i/status/${tweet.rest_id}`
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
} catch (e) {
|
|
59
|
+
// ignore parsing errors for individual payloads
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return results.slice(0, kwargs.limit);
|
|
64
|
+
}
|
|
65
|
+
});
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { cli, Strategy } from '../../registry.js';
|
|
2
|
+
|
|
3
|
+
cli({
|
|
4
|
+
site: 'twitter',
|
|
5
|
+
name: 'timeline',
|
|
6
|
+
description: 'Twitter Home Timeline',
|
|
7
|
+
domain: 'x.com',
|
|
8
|
+
strategy: Strategy.COOKIE,
|
|
9
|
+
args: [
|
|
10
|
+
{ name: 'limit', type: 'int', default: 20 },
|
|
11
|
+
],
|
|
12
|
+
columns: ['responseType', 'first'],
|
|
13
|
+
func: async (page, kwargs) => {
|
|
14
|
+
await page.goto('https://x.com/home');
|
|
15
|
+
await page.wait(5);
|
|
16
|
+
// Inject the fetch interceptor manually to see exactly what happens
|
|
17
|
+
await page.evaluate(`
|
|
18
|
+
() => {
|
|
19
|
+
window.__intercept_data = [];
|
|
20
|
+
const origFetch = window.fetch;
|
|
21
|
+
window.fetch = async function(...args) {
|
|
22
|
+
let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
|
|
23
|
+
const res = await origFetch.apply(this, args);
|
|
24
|
+
setTimeout(async () => {
|
|
25
|
+
try {
|
|
26
|
+
if (u.includes('HomeTimeline')) {
|
|
27
|
+
const clone = res.clone();
|
|
28
|
+
const j = await clone.json();
|
|
29
|
+
window.__intercept_data.push(j);
|
|
30
|
+
}
|
|
31
|
+
} catch(e) {}
|
|
32
|
+
}, 0);
|
|
33
|
+
return res;
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
`);
|
|
37
|
+
|
|
38
|
+
// trigger scroll
|
|
39
|
+
for(let i=0; i<3; i++) {
|
|
40
|
+
await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
|
|
41
|
+
await page.wait(2);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// extract
|
|
45
|
+
const data = await page.evaluate('() => window.__intercept_data');
|
|
46
|
+
if (!data || data.length === 0) return [{responseType: 'no data captured'}];
|
|
47
|
+
|
|
48
|
+
return [{responseType: `captured ${data.length} responses`, first: JSON.stringify(data[0]).substring(0,300)}];
|
|
49
|
+
}
|
|
50
|
+
});
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { cli, Strategy } from '../../registry.js';
|
|
2
|
+
|
|
3
|
+
cli({
|
|
4
|
+
site: 'xiaohongshu',
|
|
5
|
+
name: 'user',
|
|
6
|
+
description: 'Get user notes from Xiaohongshu',
|
|
7
|
+
domain: 'xiaohongshu.com',
|
|
8
|
+
strategy: Strategy.INTERCEPT,
|
|
9
|
+
browser: true,
|
|
10
|
+
args: [
|
|
11
|
+
{ name: 'id', type: 'string', required: true },
|
|
12
|
+
{ name: 'limit', type: 'int', default: 15 },
|
|
13
|
+
],
|
|
14
|
+
columns: ['id', 'title', 'type', 'likes', 'url'],
|
|
15
|
+
func: async (page, kwargs) => {
|
|
16
|
+
await page.goto(`https://www.xiaohongshu.com/user/profile/${kwargs.id}`);
|
|
17
|
+
await page.wait(5);
|
|
18
|
+
|
|
19
|
+
await page.installInterceptor('v1/user/posted');
|
|
20
|
+
|
|
21
|
+
// Trigger API by scrolling
|
|
22
|
+
await page.autoScroll({ times: 2, delayMs: 2000 });
|
|
23
|
+
|
|
24
|
+
// Retrieve data
|
|
25
|
+
const requests = await page.getInterceptedRequests();
|
|
26
|
+
if (!requests || requests.length === 0) return [];
|
|
27
|
+
|
|
28
|
+
let results: any[] = [];
|
|
29
|
+
for (const req of requests) {
|
|
30
|
+
if (req.data && req.data.data && req.data.data.notes) {
|
|
31
|
+
for (const note of req.data.data.notes) {
|
|
32
|
+
results.push({
|
|
33
|
+
id: note.note_id || note.id,
|
|
34
|
+
title: note.display_title || '',
|
|
35
|
+
type: note.type || '',
|
|
36
|
+
likes: note.interact_info?.liked_count || '0',
|
|
37
|
+
url: `https://www.xiaohongshu.com/explore/${note.note_id || note.id}`
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return results.slice(0, kwargs.limit);
|
|
44
|
+
}
|
|
45
|
+
});
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
site: xueqiu
|
|
2
|
+
name: feed
|
|
3
|
+
description: 获取雪球首页时间线(关注用户的动态)
|
|
4
|
+
domain: xueqiu.com
|
|
5
|
+
browser: true
|
|
6
|
+
|
|
7
|
+
args:
|
|
8
|
+
page:
|
|
9
|
+
type: int
|
|
10
|
+
default: 1
|
|
11
|
+
description: 页码,默认 1
|
|
12
|
+
limit:
|
|
13
|
+
type: int
|
|
14
|
+
default: 20
|
|
15
|
+
description: 每页数量,默认 20
|
|
16
|
+
|
|
17
|
+
pipeline:
|
|
18
|
+
- navigate: https://xueqiu.com
|
|
19
|
+
- evaluate: |
|
|
20
|
+
(async () => {
|
|
21
|
+
const page = ${{ args.page }};
|
|
22
|
+
const count = ${{ args.limit }};
|
|
23
|
+
const resp = await fetch(`https://xueqiu.com/v4/statuses/home_timeline.json?page=${page}&count=${count}`, {credentials: 'include'});
|
|
24
|
+
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
|
25
|
+
const d = await resp.json();
|
|
26
|
+
|
|
27
|
+
const strip = (html) => (html || '').replace(/<[^>]+>/g, '').replace(/ /g, ' ').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').trim();
|
|
28
|
+
const list = d.home_timeline || d.list || [];
|
|
29
|
+
return list.map(item => {
|
|
30
|
+
const user = item.user || {};
|
|
31
|
+
return {
|
|
32
|
+
id: item.id,
|
|
33
|
+
text: strip(item.description).substring(0, 200),
|
|
34
|
+
url: 'https://xueqiu.com/' + user.id + '/' + item.id,
|
|
35
|
+
author: user.screen_name,
|
|
36
|
+
likes: item.fav_count,
|
|
37
|
+
retweets: item.retweet_count,
|
|
38
|
+
replies: item.reply_count,
|
|
39
|
+
created_at: item.created_at ? new Date(item.created_at).toISOString() : null
|
|
40
|
+
};
|
|
41
|
+
});
|
|
42
|
+
})()
|
|
43
|
+
|
|
44
|
+
- map:
|
|
45
|
+
author: ${{ item.author }}
|
|
46
|
+
text: ${{ item.text }}
|
|
47
|
+
likes: ${{ item.likes }}
|
|
48
|
+
replies: ${{ item.replies }}
|
|
49
|
+
url: ${{ item.url }}
|
|
50
|
+
|
|
51
|
+
- limit: ${{ args.limit }}
|
|
52
|
+
|
|
53
|
+
columns: [author, text, likes, replies, url]
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
site: xueqiu
|
|
2
|
+
name: hot-stock
|
|
3
|
+
description: 获取雪球热门股票榜
|
|
4
|
+
domain: xueqiu.com
|
|
5
|
+
browser: true
|
|
6
|
+
|
|
7
|
+
args:
|
|
8
|
+
limit:
|
|
9
|
+
type: int
|
|
10
|
+
default: 20
|
|
11
|
+
description: 返回数量,默认 20,最大 50
|
|
12
|
+
type:
|
|
13
|
+
type: str
|
|
14
|
+
default: "10"
|
|
15
|
+
description: 榜单类型 10=人气榜(默认) 12=关注榜
|
|
16
|
+
|
|
17
|
+
pipeline:
|
|
18
|
+
- navigate: https://xueqiu.com
|
|
19
|
+
- evaluate: |
|
|
20
|
+
(async () => {
|
|
21
|
+
const count = ${{ args.limit }};
|
|
22
|
+
const type = ${{ args.type | json }};
|
|
23
|
+
const resp = await fetch(`https://stock.xueqiu.com/v5/stock/hot_stock/list.json?size=${count}&type=${type}`, {credentials: 'include'});
|
|
24
|
+
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
|
25
|
+
const d = await resp.json();
|
|
26
|
+
if (!d.data || !d.data.items) throw new Error('获取失败');
|
|
27
|
+
return d.data.items.map((s, i) => ({
|
|
28
|
+
rank: i + 1,
|
|
29
|
+
symbol: s.symbol,
|
|
30
|
+
name: s.name,
|
|
31
|
+
price: s.current,
|
|
32
|
+
changePercent: s.percent != null ? s.percent.toFixed(2) + '%' : null,
|
|
33
|
+
heat: s.value,
|
|
34
|
+
rank_change: s.rank_change,
|
|
35
|
+
url: 'https://xueqiu.com/S/' + s.symbol
|
|
36
|
+
}));
|
|
37
|
+
})()
|
|
38
|
+
|
|
39
|
+
- map:
|
|
40
|
+
rank: ${{ item.rank }}
|
|
41
|
+
symbol: ${{ item.symbol }}
|
|
42
|
+
name: ${{ item.name }}
|
|
43
|
+
price: ${{ item.price }}
|
|
44
|
+
changePercent: ${{ item.changePercent }}
|
|
45
|
+
heat: ${{ item.heat }}
|
|
46
|
+
|
|
47
|
+
- limit: ${{ args.limit }}
|
|
48
|
+
|
|
49
|
+
columns: [rank, symbol, name, price, changePercent, heat]
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
site: xueqiu
|
|
2
|
+
name: hot
|
|
3
|
+
description: 获取雪球热门动态
|
|
4
|
+
domain: xueqiu.com
|
|
5
|
+
browser: true
|
|
6
|
+
|
|
7
|
+
args:
|
|
8
|
+
limit:
|
|
9
|
+
type: int
|
|
10
|
+
default: 20
|
|
11
|
+
description: 返回数量,默认 20,最大 50
|
|
12
|
+
|
|
13
|
+
pipeline:
|
|
14
|
+
- navigate: https://xueqiu.com
|
|
15
|
+
- evaluate: |
|
|
16
|
+
(async () => {
|
|
17
|
+
const resp = await fetch('https://xueqiu.com/statuses/hot/listV3.json?source=hot&page=1', {credentials: 'include'});
|
|
18
|
+
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
|
19
|
+
const d = await resp.json();
|
|
20
|
+
const list = d.list || [];
|
|
21
|
+
|
|
22
|
+
const strip = (html) => (html || '').replace(/<[^>]+>/g, '').replace(/ /g, ' ').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').trim();
|
|
23
|
+
return list.map((item, i) => {
|
|
24
|
+
const user = item.user || {};
|
|
25
|
+
return {
|
|
26
|
+
rank: i + 1,
|
|
27
|
+
text: strip(item.description).substring(0, 200),
|
|
28
|
+
url: 'https://xueqiu.com/' + user.id + '/' + item.id,
|
|
29
|
+
author: user.screen_name,
|
|
30
|
+
likes: item.fav_count,
|
|
31
|
+
retweets: item.retweet_count,
|
|
32
|
+
replies: item.reply_count
|
|
33
|
+
};
|
|
34
|
+
});
|
|
35
|
+
})()
|
|
36
|
+
|
|
37
|
+
- map:
|
|
38
|
+
rank: ${{ item.rank }}
|
|
39
|
+
author: ${{ item.author }}
|
|
40
|
+
text: ${{ item.text }}
|
|
41
|
+
likes: ${{ item.likes }}
|
|
42
|
+
url: ${{ item.url }}
|
|
43
|
+
|
|
44
|
+
- limit: ${{ args.limit }}
|
|
45
|
+
|
|
46
|
+
columns: [rank, author, text, likes, url]
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
site: xueqiu
|
|
2
|
+
name: search
|
|
3
|
+
description: 搜索雪球股票(代码或名称)
|
|
4
|
+
domain: xueqiu.com
|
|
5
|
+
browser: true
|
|
6
|
+
|
|
7
|
+
args:
|
|
8
|
+
query:
|
|
9
|
+
type: str
|
|
10
|
+
description: 搜索关键词,如 茅台、AAPL、腾讯
|
|
11
|
+
limit:
|
|
12
|
+
type: int
|
|
13
|
+
default: 10
|
|
14
|
+
description: 返回数量,默认 10
|
|
15
|
+
|
|
16
|
+
pipeline:
|
|
17
|
+
- navigate: https://xueqiu.com
|
|
18
|
+
- evaluate: |
|
|
19
|
+
(async () => {
|
|
20
|
+
const query = ${{ args.query | json }};
|
|
21
|
+
const count = ${{ args.limit }};
|
|
22
|
+
if (!query) throw new Error('Missing argument: query');
|
|
23
|
+
const resp = await fetch(`https://xueqiu.com/stock/search.json?code=${encodeURIComponent(query)}&size=${count}`, {credentials: 'include'});
|
|
24
|
+
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
|
25
|
+
const d = await resp.json();
|
|
26
|
+
return (d.stocks || []).map(s => {
|
|
27
|
+
let symbol = '';
|
|
28
|
+
if (s.exchange === 'SH' || s.exchange === 'SZ' || s.exchange === 'BJ') {
|
|
29
|
+
symbol = s.code.startsWith(s.exchange) ? s.code : s.exchange + s.code;
|
|
30
|
+
} else {
|
|
31
|
+
symbol = s.code;
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
symbol: symbol,
|
|
35
|
+
name: s.name,
|
|
36
|
+
exchange: s.exchange,
|
|
37
|
+
price: s.current,
|
|
38
|
+
changePercent: s.percentage != null ? s.percentage.toFixed(2) + '%' : null,
|
|
39
|
+
url: 'https://xueqiu.com/S/' + symbol
|
|
40
|
+
};
|
|
41
|
+
});
|
|
42
|
+
})()
|
|
43
|
+
|
|
44
|
+
- map:
|
|
45
|
+
symbol: ${{ item.symbol }}
|
|
46
|
+
name: ${{ item.name }}
|
|
47
|
+
exchange: ${{ item.exchange }}
|
|
48
|
+
price: ${{ item.price }}
|
|
49
|
+
changePercent: ${{ item.changePercent }}
|
|
50
|
+
|
|
51
|
+
- limit: ${{ args.limit }}
|
|
52
|
+
|
|
53
|
+
columns: [symbol, name, exchange, price, changePercent]
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
site: xueqiu
|
|
2
|
+
name: stock
|
|
3
|
+
description: 获取雪球股票实时行情
|
|
4
|
+
domain: xueqiu.com
|
|
5
|
+
browser: true
|
|
6
|
+
|
|
7
|
+
args:
|
|
8
|
+
symbol:
|
|
9
|
+
type: str
|
|
10
|
+
description: 股票代码,如 SH600519、SZ000858、AAPL、00700
|
|
11
|
+
|
|
12
|
+
pipeline:
|
|
13
|
+
- navigate: https://xueqiu.com
|
|
14
|
+
- evaluate: |
|
|
15
|
+
(async () => {
|
|
16
|
+
const symbol = (${{ args.symbol | json }} || '').toUpperCase();
|
|
17
|
+
if (!symbol) throw new Error('Missing argument: symbol');
|
|
18
|
+
const resp = await fetch(`https://stock.xueqiu.com/v5/stock/batch/quote.json?symbol=${encodeURIComponent(symbol)}`, {credentials: 'include'});
|
|
19
|
+
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
|
20
|
+
const d = await resp.json();
|
|
21
|
+
if (!d.data || !d.data.items || d.data.items.length === 0) throw new Error('未找到股票: ' + symbol);
|
|
22
|
+
|
|
23
|
+
function fmtAmount(v) {
|
|
24
|
+
if (v == null) return null;
|
|
25
|
+
if (Math.abs(v) >= 1e12) return (v / 1e12).toFixed(2) + '万亿';
|
|
26
|
+
if (Math.abs(v) >= 1e8) return (v / 1e8).toFixed(2) + '亿';
|
|
27
|
+
if (Math.abs(v) >= 1e4) return (v / 1e4).toFixed(2) + '万';
|
|
28
|
+
return v.toString();
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const item = d.data.items[0];
|
|
32
|
+
const q = item.quote || {};
|
|
33
|
+
const m = item.market || {};
|
|
34
|
+
|
|
35
|
+
return [{
|
|
36
|
+
name: q.name,
|
|
37
|
+
symbol: q.symbol,
|
|
38
|
+
exchange: q.exchange,
|
|
39
|
+
currency: q.currency,
|
|
40
|
+
price: q.current,
|
|
41
|
+
change: q.chg,
|
|
42
|
+
changePercent: q.percent != null ? q.percent.toFixed(2) + '%' : null,
|
|
43
|
+
open: q.open,
|
|
44
|
+
high: q.high,
|
|
45
|
+
low: q.low,
|
|
46
|
+
prevClose: q.last_close,
|
|
47
|
+
amplitude: q.amplitude != null ? q.amplitude.toFixed(2) + '%' : null,
|
|
48
|
+
volume: q.volume,
|
|
49
|
+
amount: fmtAmount(q.amount),
|
|
50
|
+
turnover_rate: q.turnover_rate != null ? q.turnover_rate.toFixed(2) + '%' : null,
|
|
51
|
+
marketCap: fmtAmount(q.market_capital),
|
|
52
|
+
floatMarketCap: fmtAmount(q.float_market_capital),
|
|
53
|
+
ytdPercent: q.current_year_percent != null ? q.current_year_percent.toFixed(2) + '%' : null,
|
|
54
|
+
market_status: m.status || null,
|
|
55
|
+
time: q.timestamp ? new Date(q.timestamp).toISOString() : null,
|
|
56
|
+
url: 'https://xueqiu.com/S/' + q.symbol
|
|
57
|
+
}];
|
|
58
|
+
})()
|
|
59
|
+
|
|
60
|
+
- map:
|
|
61
|
+
name: ${{ item.name }}
|
|
62
|
+
symbol: ${{ item.symbol }}
|
|
63
|
+
price: ${{ item.price }}
|
|
64
|
+
changePercent: ${{ item.changePercent }}
|
|
65
|
+
marketCap: ${{ item.marketCap }}
|
|
66
|
+
|
|
67
|
+
columns: [name, symbol, price, changePercent, marketCap]
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
site: xueqiu
|
|
2
|
+
name: watchlist
|
|
3
|
+
description: 获取雪球自选股列表
|
|
4
|
+
domain: xueqiu.com
|
|
5
|
+
browser: true
|
|
6
|
+
|
|
7
|
+
args:
|
|
8
|
+
category:
|
|
9
|
+
type: str # using str to prevent parsing issues like 01
|
|
10
|
+
default: "1"
|
|
11
|
+
description: "分类:1=自选(默认) 2=持仓 3=关注"
|
|
12
|
+
limit:
|
|
13
|
+
type: int
|
|
14
|
+
default: 100
|
|
15
|
+
description: 默认 100
|
|
16
|
+
|
|
17
|
+
pipeline:
|
|
18
|
+
- navigate: https://xueqiu.com
|
|
19
|
+
- evaluate: |
|
|
20
|
+
(async () => {
|
|
21
|
+
const category = parseInt(${{ args.category | json }}) || 1;
|
|
22
|
+
const resp = await fetch(`https://stock.xueqiu.com/v5/stock/portfolio/stock/list.json?size=100&category=${category}&pid=-1`, {credentials: 'include'});
|
|
23
|
+
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
|
24
|
+
const d = await resp.json();
|
|
25
|
+
if (!d.data || !d.data.stocks) throw new Error('获取失败,可能未登录');
|
|
26
|
+
|
|
27
|
+
return d.data.stocks.map(s => ({
|
|
28
|
+
symbol: s.symbol,
|
|
29
|
+
name: s.name,
|
|
30
|
+
price: s.current,
|
|
31
|
+
change: s.chg,
|
|
32
|
+
changePercent: s.percent != null ? s.percent.toFixed(2) + '%' : null,
|
|
33
|
+
volume: s.volume,
|
|
34
|
+
url: 'https://xueqiu.com/S/' + s.symbol
|
|
35
|
+
}));
|
|
36
|
+
})()
|
|
37
|
+
|
|
38
|
+
- map:
|
|
39
|
+
symbol: ${{ item.symbol }}
|
|
40
|
+
name: ${{ item.name }}
|
|
41
|
+
price: ${{ item.price }}
|
|
42
|
+
changePercent: ${{ item.changePercent }}
|
|
43
|
+
|
|
44
|
+
- limit: ${{ args.limit }}
|
|
45
|
+
|
|
46
|
+
columns: [symbol, name, price, changePercent]
|
package/src/clis/zhihu/hot.yaml
CHANGED
|
@@ -17,12 +17,16 @@ pipeline:
|
|
|
17
17
|
const res = await fetch('https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total?limit=50', {
|
|
18
18
|
credentials: 'include'
|
|
19
19
|
});
|
|
20
|
-
const
|
|
20
|
+
const text = await res.text();
|
|
21
|
+
const d = JSON.parse(
|
|
22
|
+
text.replace(/("id"\s*:\s*)(\d{16,})/g, '$1"$2"')
|
|
23
|
+
);
|
|
21
24
|
return (d?.data || []).map((item) => {
|
|
22
25
|
const t = item.target || {};
|
|
26
|
+
const questionId = t.id == null ? '' : String(t.id);
|
|
23
27
|
return {
|
|
24
28
|
title: t.title,
|
|
25
|
-
url: 'https://www.zhihu.com/question/' +
|
|
29
|
+
url: 'https://www.zhihu.com/question/' + questionId,
|
|
26
30
|
answer_count: t.answer_count,
|
|
27
31
|
follower_count: t.follower_count,
|
|
28
32
|
heat: item.detail_text || '',
|
|
@@ -19,7 +19,9 @@ pipeline:
|
|
|
19
19
|
- evaluate: |
|
|
20
20
|
(async () => {
|
|
21
21
|
const strip = (html) => (html || '').replace(/<[^>]+>/g, '').replace(/ /g, ' ').replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&').replace(/<em>/g, '').replace(/<\/em>/g, '').trim();
|
|
22
|
-
const
|
|
22
|
+
const keyword = ${{ args.keyword | json }};
|
|
23
|
+
const limit = ${{ args.limit }};
|
|
24
|
+
const res = await fetch('https://www.zhihu.com/api/v4/search_v3?q=' + encodeURIComponent(keyword) + '&t=general&offset=0&limit=' + limit, {
|
|
23
25
|
credentials: 'include'
|
|
24
26
|
});
|
|
25
27
|
const d = await res.json();
|
package/src/engine.ts
CHANGED
|
@@ -9,7 +9,8 @@ import { type CliCommand, type Arg, Strategy, registerCommand } from './registry
|
|
|
9
9
|
import type { IPage } from './types.js';
|
|
10
10
|
import { executePipeline } from './pipeline.js';
|
|
11
11
|
|
|
12
|
-
export function discoverClis(...dirs: string[]): void {
|
|
12
|
+
export async function discoverClis(...dirs: string[]): Promise<void> {
|
|
13
|
+
const promises: Promise<any>[] = [];
|
|
13
14
|
for (const dir of dirs) {
|
|
14
15
|
if (!fs.existsSync(dir)) continue;
|
|
15
16
|
for (const site of fs.readdirSync(dir)) {
|
|
@@ -19,10 +20,18 @@ export function discoverClis(...dirs: string[]): void {
|
|
|
19
20
|
const filePath = path.join(siteDir, file);
|
|
20
21
|
if (file.endsWith('.yaml') || file.endsWith('.yml')) {
|
|
21
22
|
registerYamlCli(filePath, site);
|
|
23
|
+
} else if (file.endsWith('.js')) {
|
|
24
|
+
// Dynamic import of compiled adapter modules
|
|
25
|
+
promises.push(
|
|
26
|
+
import(`file://${filePath}`).catch((err: any) => {
|
|
27
|
+
process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`);
|
|
28
|
+
})
|
|
29
|
+
);
|
|
22
30
|
}
|
|
23
31
|
}
|
|
24
32
|
}
|
|
25
33
|
}
|
|
34
|
+
await Promise.all(promises);
|
|
26
35
|
}
|
|
27
36
|
|
|
28
37
|
function registerYamlCli(filePath: string, defaultSite: string): void {
|
package/src/explore.ts
CHANGED
|
@@ -184,6 +184,8 @@ function scoreEndpoint(ep: { contentType: string; responseAnalysis: any; pattern
|
|
|
184
184
|
if (ep.hasPaginationParam) s += 2;
|
|
185
185
|
if (ep.hasLimitParam) s += 2;
|
|
186
186
|
if (ep.status === 200) s += 2;
|
|
187
|
+
// Anti-Bot Empty Value Detection: penalize JSON endpoints returning empty data
|
|
188
|
+
if (ep.responseAnalysis && ep.responseAnalysis.itemCount === 0 && ep.contentType.includes('json')) s -= 3;
|
|
187
189
|
return s;
|
|
188
190
|
}
|
|
189
191
|
|
|
@@ -277,6 +279,30 @@ export interface DiscoveredStore {
|
|
|
277
279
|
stateKeys: string[];
|
|
278
280
|
}
|
|
279
281
|
|
|
282
|
+
// ── Auto-Interaction (Fuzzing) ─────────────────────────────────────────────
|
|
283
|
+
|
|
284
|
+
const INTERACT_FUZZ_JS = `
|
|
285
|
+
async () => {
|
|
286
|
+
const sleep = ms => new Promise(r => setTimeout(r, ms));
|
|
287
|
+
const clickables = Array.from(document.querySelectorAll(
|
|
288
|
+
'button, [role="button"], [role="tab"], .tab, .btn, a[href="javascript:void(0)"], a[href="#"]'
|
|
289
|
+
)).slice(0, 15); // limit to 15 to avoid endless loops
|
|
290
|
+
|
|
291
|
+
let clicked = 0;
|
|
292
|
+
for (const el of clickables) {
|
|
293
|
+
try {
|
|
294
|
+
const rect = el.getBoundingClientRect();
|
|
295
|
+
if (rect.width > 0 && rect.height > 0) {
|
|
296
|
+
el.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
|
|
297
|
+
clicked++;
|
|
298
|
+
await sleep(300); // give it time to trigger network
|
|
299
|
+
}
|
|
300
|
+
} catch {}
|
|
301
|
+
}
|
|
302
|
+
return clicked;
|
|
303
|
+
}
|
|
304
|
+
`;
|
|
305
|
+
|
|
280
306
|
// ── Main explore function ──────────────────────────────────────────────────
|
|
281
307
|
|
|
282
308
|
export async function exploreUrl(
|
|
@@ -300,6 +326,31 @@ export async function exploreUrl(
|
|
|
300
326
|
// Step 2: Auto-scroll to trigger lazy loading (use keyboard since page.scroll may not exist)
|
|
301
327
|
for (let i = 0; i < 3; i++) { try { await page.pressKey('End'); } catch {} await page.wait(1); }
|
|
302
328
|
|
|
329
|
+
// Step 2.5: Interactive Fuzzing (if requested)
|
|
330
|
+
if (opts.auto) {
|
|
331
|
+
try {
|
|
332
|
+
// First: targeted clicks by label (e.g. "字幕", "CC", "评论")
|
|
333
|
+
if (opts.clickLabels?.length) {
|
|
334
|
+
for (const label of opts.clickLabels) {
|
|
335
|
+
const safeLabel = label.replace(/'/g, "\\'");
|
|
336
|
+
await page.evaluate(`
|
|
337
|
+
(() => {
|
|
338
|
+
const el = [...document.querySelectorAll('button, [role="button"], [role="tab"], a, span')]
|
|
339
|
+
.find(e => e.textContent && e.textContent.trim().includes('${safeLabel}'));
|
|
340
|
+
if (el) el.click();
|
|
341
|
+
})()
|
|
342
|
+
`);
|
|
343
|
+
await page.wait(1);
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
// Then: blind fuzzing on generic interactive elements
|
|
347
|
+
const clicks = await page.evaluate(INTERACT_FUZZ_JS);
|
|
348
|
+
await page.wait(2); // wait for XHRs to settle
|
|
349
|
+
} catch (e) {
|
|
350
|
+
// fuzzing is best-effort, don't fail the whole explore
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
303
354
|
// Step 3: Read page metadata
|
|
304
355
|
const metadata = await readPageMetadata(page);
|
|
305
356
|
|
package/src/main.ts
CHANGED
|
@@ -12,7 +12,6 @@ import chalk from 'chalk';
|
|
|
12
12
|
import { discoverClis, executeCommand } from './engine.js';
|
|
13
13
|
import { type CliCommand, fullName, getRegistry, strategyLabel } from './registry.js';
|
|
14
14
|
import { render as renderOutput } from './output.js';
|
|
15
|
-
import './clis/index.js';
|
|
16
15
|
import { PlaywrightMCP } from './browser.js';
|
|
17
16
|
import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
|
|
18
17
|
|
|
@@ -25,7 +24,7 @@ const USER_CLIS = path.join(os.homedir(), '.opencli', 'clis');
|
|
|
25
24
|
const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
|
|
26
25
|
const PKG_VERSION = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version ?? '0.0.0';
|
|
27
26
|
|
|
28
|
-
discoverClis(BUILTIN_CLIS, USER_CLIS);
|
|
27
|
+
await discoverClis(BUILTIN_CLIS, USER_CLIS);
|
|
29
28
|
|
|
30
29
|
const program = new Command();
|
|
31
30
|
program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
|
|
@@ -54,8 +53,8 @@ program.command('validate').description('Validate CLI definitions').argument('[t
|
|
|
54
53
|
program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false)
|
|
55
54
|
.action(async (target, opts) => { const { verifyClis, renderVerifyReport } = await import('./verify.js'); const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke }); console.log(renderVerifyReport(r)); process.exitCode = r.ok ? 0 : 1; });
|
|
56
55
|
|
|
57
|
-
program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3')
|
|
58
|
-
.action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait) }))); });
|
|
56
|
+
program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3').option('--auto', 'Enable interactive fuzzing (simulate clicks to trigger lazy APIs)').option('--click <labels>', 'Comma-separated labels to click before fuzzing (e.g. "字幕,CC,评论")')
|
|
57
|
+
.action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); const clickLabels = opts.click ? opts.click.split(',').map((s: string) => s.trim()) : undefined; console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait), auto: opts.auto, clickLabels }))); });
|
|
59
58
|
|
|
60
59
|
program.command('synthesize').description('Synthesize CLIs from explore').argument('<target>').option('--top <n>', '', '3')
|
|
61
60
|
.action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); });
|
|
@@ -104,8 +103,15 @@ for (const [, cmd] of registry) {
|
|
|
104
103
|
if (cmd.browser) {
|
|
105
104
|
result = await browserSession(PlaywrightMCP, async (page) => runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) }));
|
|
106
105
|
} else { result = await executeCommand(cmd, null, kwargs, actionOpts.verbose); }
|
|
106
|
+
if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) {
|
|
107
|
+
console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`));
|
|
108
|
+
}
|
|
107
109
|
renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) });
|
|
108
|
-
} catch (err: any) {
|
|
110
|
+
} catch (err: any) {
|
|
111
|
+
if (actionOpts.verbose && err.stack) { console.error(chalk.red(err.stack)); }
|
|
112
|
+
else { console.error(chalk.red(`Error: ${err.message ?? err}`)); }
|
|
113
|
+
process.exitCode = 1;
|
|
114
|
+
}
|
|
109
115
|
});
|
|
110
116
|
}
|
|
111
117
|
|