@jackwener/opencli 1.6.7 → 1.6.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/README.zh-CN.md +6 -2
- package/dist/clis/1688/assets.d.ts +42 -0
- package/dist/clis/1688/assets.js +204 -0
- package/dist/clis/1688/assets.test.d.ts +1 -0
- package/dist/clis/1688/assets.test.js +39 -0
- package/dist/clis/1688/download.d.ts +9 -0
- package/dist/clis/1688/download.js +76 -0
- package/dist/clis/1688/download.test.d.ts +1 -0
- package/dist/clis/1688/download.test.js +31 -0
- package/dist/clis/1688/shared.d.ts +10 -0
- package/dist/clis/1688/shared.js +43 -0
- package/dist/clis/linux-do/topic-content.d.ts +35 -0
- package/dist/clis/linux-do/topic-content.js +154 -0
- package/dist/clis/linux-do/topic-content.test.d.ts +1 -0
- package/dist/clis/linux-do/topic-content.test.js +59 -0
- package/dist/clis/linux-do/topic.yaml +1 -16
- package/dist/clis/xueqiu/groups.yaml +23 -0
- package/dist/clis/xueqiu/kline.yaml +65 -0
- package/dist/clis/xueqiu/watchlist.yaml +9 -9
- package/dist/src/analysis.d.ts +2 -0
- package/dist/src/analysis.js +6 -0
- package/dist/src/browser/cdp.js +96 -0
- package/dist/src/build-manifest.d.ts +3 -1
- package/dist/src/build-manifest.js +10 -7
- package/dist/src/build-manifest.test.js +8 -4
- package/dist/src/cli.d.ts +2 -1
- package/dist/src/cli.js +48 -46
- package/dist/src/commands/daemon.js +2 -10
- package/dist/src/diagnostic.d.ts +27 -2
- package/dist/src/diagnostic.js +201 -25
- package/dist/src/diagnostic.test.js +130 -1
- package/dist/src/discovery.js +7 -17
- package/dist/src/download/progress.js +7 -2
- package/dist/src/explore.d.ts +0 -2
- package/dist/src/explore.js +61 -38
- package/dist/src/extension-manifest-regression.test.js +0 -1
- package/dist/src/generate.d.ts +1 -1
- package/dist/src/generate.js +2 -3
- package/dist/src/package-paths.d.ts +8 -0
- package/dist/src/package-paths.js +41 -0
- package/dist/src/plugin-scaffold.js +1 -3
- package/dist/src/record.d.ts +1 -2
- package/dist/src/record.js +14 -52
- package/dist/src/synthesize.d.ts +0 -2
- package/dist/src/synthesize.js +8 -4
- package/package.json +1 -1
- package/dist/cli-manifest.json +0 -17250
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import { AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
|
|
2
|
+
import { cli, Strategy } from '@jackwener/opencli/registry';
|
|
3
|
+
import { htmlToMarkdown, isRecord } from '@jackwener/opencli/utils';
|
|
4
|
+
const LINUX_DO_DOMAIN = 'linux.do';
|
|
5
|
+
const LINUX_DO_HOME = 'https://linux.do';
|
|
6
|
+
function toLocalTime(utcStr) {
|
|
7
|
+
if (!utcStr)
|
|
8
|
+
return '';
|
|
9
|
+
const date = new Date(utcStr);
|
|
10
|
+
return Number.isNaN(date.getTime()) ? utcStr : date.toLocaleString();
|
|
11
|
+
}
|
|
12
|
+
function normalizeTopicPayload(payload) {
|
|
13
|
+
if (!isRecord(payload))
|
|
14
|
+
return null;
|
|
15
|
+
const postStream = isRecord(payload.post_stream)
|
|
16
|
+
? {
|
|
17
|
+
posts: Array.isArray(payload.post_stream.posts)
|
|
18
|
+
? payload.post_stream.posts.filter(isRecord).map((post) => ({
|
|
19
|
+
post_number: typeof post.post_number === 'number' ? post.post_number : undefined,
|
|
20
|
+
username: typeof post.username === 'string' ? post.username : undefined,
|
|
21
|
+
raw: typeof post.raw === 'string' ? post.raw : undefined,
|
|
22
|
+
cooked: typeof post.cooked === 'string' ? post.cooked : undefined,
|
|
23
|
+
like_count: typeof post.like_count === 'number' ? post.like_count : undefined,
|
|
24
|
+
created_at: typeof post.created_at === 'string' ? post.created_at : undefined,
|
|
25
|
+
}))
|
|
26
|
+
: undefined,
|
|
27
|
+
}
|
|
28
|
+
: undefined;
|
|
29
|
+
return {
|
|
30
|
+
title: typeof payload.title === 'string' ? payload.title : undefined,
|
|
31
|
+
post_stream: postStream,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
function buildTopicMarkdownDocument(params) {
|
|
35
|
+
const frontMatterLines = [];
|
|
36
|
+
const entries = [
|
|
37
|
+
['title', params.title || undefined],
|
|
38
|
+
['author', params.author || undefined],
|
|
39
|
+
['likes', typeof params.likes === 'number' && Number.isFinite(params.likes) ? params.likes : undefined],
|
|
40
|
+
['createdAt', params.createdAt || undefined],
|
|
41
|
+
['url', params.url || undefined],
|
|
42
|
+
];
|
|
43
|
+
for (const [key, value] of entries) {
|
|
44
|
+
if (value === undefined)
|
|
45
|
+
continue;
|
|
46
|
+
if (typeof value === 'number') {
|
|
47
|
+
frontMatterLines.push(`${key}: ${value}`);
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
// Quote strings that could be misinterpreted by YAML parsers
|
|
51
|
+
const needsQuote = /[#{}[\],&*?|>!%@`'"]/.test(value) || /: /.test(value) || /:$/.test(value) || value.includes('\n');
|
|
52
|
+
frontMatterLines.push(`${key}: ${needsQuote ? `'${value.replace(/'/g, "''")}'` : value}`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const frontMatter = frontMatterLines.join('\n');
|
|
56
|
+
return [
|
|
57
|
+
frontMatter ? `---\n${frontMatter}\n---` : '',
|
|
58
|
+
params.body.trim(),
|
|
59
|
+
].filter(Boolean).join('\n\n').trim();
|
|
60
|
+
}
|
|
61
|
+
function extractTopicContent(payload, id) {
|
|
62
|
+
const topic = normalizeTopicPayload(payload);
|
|
63
|
+
if (!topic) {
|
|
64
|
+
throw new CommandExecutionError('linux.do returned an unexpected topic payload');
|
|
65
|
+
}
|
|
66
|
+
const posts = topic.post_stream?.posts ?? [];
|
|
67
|
+
const mainPost = posts.find((post) => post.post_number === 1);
|
|
68
|
+
if (!mainPost) {
|
|
69
|
+
throw new EmptyResultError('linux-do/topic-content', `Could not find the main post for topic ${id}.`);
|
|
70
|
+
}
|
|
71
|
+
const body = typeof mainPost.raw === 'string' && mainPost.raw.trim()
|
|
72
|
+
? mainPost.raw.trim()
|
|
73
|
+
: htmlToMarkdown(mainPost.cooked ?? '');
|
|
74
|
+
if (!body) {
|
|
75
|
+
throw new EmptyResultError('linux-do/topic-content', `Topic ${id} does not contain a readable main post body.`);
|
|
76
|
+
}
|
|
77
|
+
return {
|
|
78
|
+
content: buildTopicMarkdownDocument({
|
|
79
|
+
title: topic.title?.trim() ?? '',
|
|
80
|
+
author: mainPost.username?.trim() ?? '',
|
|
81
|
+
likes: typeof mainPost.like_count === 'number' ? mainPost.like_count : undefined,
|
|
82
|
+
createdAt: toLocalTime(mainPost.created_at ?? ''),
|
|
83
|
+
url: `${LINUX_DO_HOME}/t/${id}`,
|
|
84
|
+
body,
|
|
85
|
+
}),
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
async function fetchTopicPayload(page, id) {
|
|
89
|
+
const result = await page.evaluate(`(async () => {
|
|
90
|
+
try {
|
|
91
|
+
const res = await fetch('/t/${id}.json?include_raw=true', { credentials: 'include' });
|
|
92
|
+
let data = null;
|
|
93
|
+
try {
|
|
94
|
+
data = await res.json();
|
|
95
|
+
} catch (_error) {
|
|
96
|
+
data = null;
|
|
97
|
+
}
|
|
98
|
+
return {
|
|
99
|
+
ok: res.ok,
|
|
100
|
+
status: res.status,
|
|
101
|
+
data,
|
|
102
|
+
error: data === null ? 'Response is not valid JSON' : '',
|
|
103
|
+
};
|
|
104
|
+
} catch (error) {
|
|
105
|
+
return {
|
|
106
|
+
ok: false,
|
|
107
|
+
error: error instanceof Error ? error.message : String(error),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
})()`);
|
|
111
|
+
if (!result) {
|
|
112
|
+
throw new CommandExecutionError('linux.do returned an empty browser response');
|
|
113
|
+
}
|
|
114
|
+
if (result.status === 401 || result.status === 403) {
|
|
115
|
+
throw new AuthRequiredError(LINUX_DO_DOMAIN, 'linux.do requires an active signed-in browser session');
|
|
116
|
+
}
|
|
117
|
+
if (result.error === 'Response is not valid JSON') {
|
|
118
|
+
throw new AuthRequiredError(LINUX_DO_DOMAIN, 'linux.do requires an active signed-in browser session');
|
|
119
|
+
}
|
|
120
|
+
if (!result.ok) {
|
|
121
|
+
throw new CommandExecutionError(result.error || `linux.do request failed: HTTP ${result.status ?? 'unknown'}`);
|
|
122
|
+
}
|
|
123
|
+
if (result.error) {
|
|
124
|
+
throw new CommandExecutionError(result.error, 'Please verify your linux.do session is still valid');
|
|
125
|
+
}
|
|
126
|
+
return result.data;
|
|
127
|
+
}
|
|
128
|
+
cli({
|
|
129
|
+
site: 'linux-do',
|
|
130
|
+
name: 'topic-content',
|
|
131
|
+
description: 'Get the main topic body as Markdown',
|
|
132
|
+
domain: LINUX_DO_DOMAIN,
|
|
133
|
+
strategy: Strategy.COOKIE,
|
|
134
|
+
browser: true,
|
|
135
|
+
defaultFormat: 'plain',
|
|
136
|
+
args: [
|
|
137
|
+
{ name: 'id', positional: true, type: 'int', required: true, help: 'Topic ID' },
|
|
138
|
+
],
|
|
139
|
+
columns: ['content'],
|
|
140
|
+
func: async (page, kwargs) => {
|
|
141
|
+
const id = Number(kwargs.id);
|
|
142
|
+
if (!Number.isInteger(id) || id <= 0) {
|
|
143
|
+
throw new CommandExecutionError(`Invalid linux.do topic id: ${String(kwargs.id ?? '')}`);
|
|
144
|
+
}
|
|
145
|
+
const payload = await fetchTopicPayload(page, id);
|
|
146
|
+
return [extractTopicContent(payload, id)];
|
|
147
|
+
},
|
|
148
|
+
});
|
|
149
|
+
export const __test__ = {
|
|
150
|
+
buildTopicMarkdownDocument,
|
|
151
|
+
extractTopicContent,
|
|
152
|
+
normalizeTopicPayload,
|
|
153
|
+
toLocalTime,
|
|
154
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { getRegistry } from '@jackwener/opencli/registry';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import { describe, expect, it } from 'vitest';
|
|
4
|
+
import { __test__ } from './topic-content.js';
|
|
5
|
+
describe('linux-do topic-content', () => {
|
|
6
|
+
it('prefers raw markdown when the topic payload includes it', () => {
|
|
7
|
+
const result = __test__.extractTopicContent({
|
|
8
|
+
title: 'Hello Linux.do',
|
|
9
|
+
post_stream: {
|
|
10
|
+
posts: [
|
|
11
|
+
{
|
|
12
|
+
post_number: 1,
|
|
13
|
+
username: 'neo',
|
|
14
|
+
raw: '## Heading\n\n- one\n- two',
|
|
15
|
+
cooked: '<h2>Heading</h2><ul><li>one</li><li>two</li></ul>',
|
|
16
|
+
like_count: 7,
|
|
17
|
+
created_at: '2025-04-05T10:00:00.000Z',
|
|
18
|
+
},
|
|
19
|
+
],
|
|
20
|
+
},
|
|
21
|
+
}, 1234);
|
|
22
|
+
expect(result.content).toContain('---');
|
|
23
|
+
expect(result.content).toContain('title: Hello Linux.do');
|
|
24
|
+
expect(result.content).toContain('author: neo');
|
|
25
|
+
expect(result.content).toContain('likes: 7');
|
|
26
|
+
expect(result.content).toContain('url: https://linux.do/t/1234');
|
|
27
|
+
expect(result.content).toContain('## Heading');
|
|
28
|
+
expect(result.content).toContain('- one');
|
|
29
|
+
});
|
|
30
|
+
it('falls back to cooked html and converts it to markdown', () => {
|
|
31
|
+
const result = __test__.extractTopicContent({
|
|
32
|
+
title: 'Converted Topic',
|
|
33
|
+
post_stream: {
|
|
34
|
+
posts: [
|
|
35
|
+
{
|
|
36
|
+
post_number: 1,
|
|
37
|
+
username: 'trinity',
|
|
38
|
+
cooked: '<p>Hello <strong>world</strong></p><blockquote><p>quoted</p></blockquote>',
|
|
39
|
+
like_count: 3,
|
|
40
|
+
created_at: '2025-04-05T10:00:00.000Z',
|
|
41
|
+
},
|
|
42
|
+
],
|
|
43
|
+
},
|
|
44
|
+
}, 42);
|
|
45
|
+
expect(result.content).toContain('Hello **world**');
|
|
46
|
+
expect(result.content).toContain('> quoted');
|
|
47
|
+
});
|
|
48
|
+
it('registers topic-content with plain default output for markdown body rendering', () => {
|
|
49
|
+
const command = getRegistry().get('linux-do/topic-content');
|
|
50
|
+
expect(command?.defaultFormat).toBe('plain');
|
|
51
|
+
expect(command?.columns).toEqual(['content']);
|
|
52
|
+
});
|
|
53
|
+
it('keeps topic yaml as a summarized first-page reader after the split', () => {
|
|
54
|
+
const topicYaml = fs.readFileSync(new URL('./topic.yaml', import.meta.url), 'utf8');
|
|
55
|
+
expect(topicYaml).not.toContain('main_only');
|
|
56
|
+
expect(topicYaml).toContain('slice(0, 200)');
|
|
57
|
+
expect(topicYaml).toContain('帖子首页摘要和回复');
|
|
58
|
+
});
|
|
59
|
+
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
site: linux-do
|
|
2
2
|
name: topic
|
|
3
|
-
description: linux.do
|
|
3
|
+
description: linux.do 帖子首页摘要和回复(首屏)
|
|
4
4
|
domain: linux.do
|
|
5
5
|
strategy: cookie
|
|
6
6
|
browser: true
|
|
@@ -15,17 +15,12 @@ args:
|
|
|
15
15
|
type: int
|
|
16
16
|
default: 20
|
|
17
17
|
description: Number of posts
|
|
18
|
-
main_only:
|
|
19
|
-
type: bool
|
|
20
|
-
default: false
|
|
21
|
-
description: Only return the main post body without truncation
|
|
22
18
|
|
|
23
19
|
pipeline:
|
|
24
20
|
- navigate: https://linux.do
|
|
25
21
|
|
|
26
22
|
- evaluate: |
|
|
27
23
|
(async () => {
|
|
28
|
-
const mainOnly = ${{ args.main_only }};
|
|
29
24
|
const toLocalTime = (utcStr) => {
|
|
30
25
|
if (!utcStr) return '';
|
|
31
26
|
const date = new Date(utcStr);
|
|
@@ -50,16 +45,6 @@ pipeline:
|
|
|
50
45
|
.replace(/\s+/g, ' ')
|
|
51
46
|
.trim();
|
|
52
47
|
const posts = data?.post_stream?.posts || [];
|
|
53
|
-
if (mainOnly) {
|
|
54
|
-
const mainPost = posts.find(p => p.post_number === 1);
|
|
55
|
-
if (!mainPost) return [];
|
|
56
|
-
return [{
|
|
57
|
-
author: mainPost.username || '',
|
|
58
|
-
content: mainPost.cooked || '',
|
|
59
|
-
likes: mainPost.like_count || 0,
|
|
60
|
-
created_at: toLocalTime(mainPost.created_at),
|
|
61
|
-
}];
|
|
62
|
-
}
|
|
63
48
|
return posts.slice(0, ${{ args.limit }}).map(p => ({
|
|
64
49
|
author: p.username,
|
|
65
50
|
content: strip(p.cooked).slice(0, 200),
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
site: xueqiu
|
|
2
|
+
name: groups
|
|
3
|
+
description: 获取雪球自选股分组列表(含模拟组合)
|
|
4
|
+
domain: xueqiu.com
|
|
5
|
+
browser: true
|
|
6
|
+
|
|
7
|
+
pipeline:
|
|
8
|
+
- navigate: https://xueqiu.com
|
|
9
|
+
- evaluate: |
|
|
10
|
+
(async () => {
|
|
11
|
+
const resp = await fetch('https://stock.xueqiu.com/v5/stock/portfolio/list.json?category=1&size=20', {credentials: 'include'});
|
|
12
|
+
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
|
13
|
+
const d = await resp.json();
|
|
14
|
+
if (!d.data || !d.data.stocks) throw new Error('获取失败,可能未登录');
|
|
15
|
+
|
|
16
|
+
return d.data.stocks.map(g => ({
|
|
17
|
+
pid: String(g.id),
|
|
18
|
+
name: g.name,
|
|
19
|
+
count: g.symbol_count || 0
|
|
20
|
+
}));
|
|
21
|
+
})()
|
|
22
|
+
|
|
23
|
+
columns: [pid, name, count]
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
site: xueqiu
|
|
2
|
+
name: kline
|
|
3
|
+
description: 获取雪球股票K线(历史行情)数据
|
|
4
|
+
domain: xueqiu.com
|
|
5
|
+
browser: true
|
|
6
|
+
|
|
7
|
+
args:
|
|
8
|
+
symbol:
|
|
9
|
+
positional: true
|
|
10
|
+
type: str
|
|
11
|
+
required: true
|
|
12
|
+
description: 股票代码,如 SH600519、SZ000858、AAPL
|
|
13
|
+
days:
|
|
14
|
+
type: int
|
|
15
|
+
default: 14
|
|
16
|
+
description: 回溯天数(默认14天)
|
|
17
|
+
|
|
18
|
+
pipeline:
|
|
19
|
+
- navigate: https://xueqiu.com
|
|
20
|
+
|
|
21
|
+
- evaluate: |
|
|
22
|
+
(async () => {
|
|
23
|
+
const symbol = (${{ args.symbol | json }} || '').toUpperCase();
|
|
24
|
+
const days = parseInt(${{ args.days | json }}) || 14;
|
|
25
|
+
if (!symbol) throw new Error('Missing argument: symbol');
|
|
26
|
+
|
|
27
|
+
// begin = now minus days (for count=-N, returns N items ending at begin)
|
|
28
|
+
const beginTs = Date.now();
|
|
29
|
+
const resp = await fetch('https://stock.xueqiu.com/v5/stock/chart/kline.json?symbol=' + encodeURIComponent(symbol) + '&begin=' + beginTs + '&period=day&type=before&count=-' + days, {credentials: 'include'});
|
|
30
|
+
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
|
31
|
+
const d = await resp.json();
|
|
32
|
+
|
|
33
|
+
if (!d.data || !d.data.item || d.data.item.length === 0) return [];
|
|
34
|
+
|
|
35
|
+
const columns = d.data.column || [];
|
|
36
|
+
const items = d.data.item || [];
|
|
37
|
+
const colIdx = {};
|
|
38
|
+
columns.forEach((name, i) => { colIdx[name] = i; });
|
|
39
|
+
|
|
40
|
+
function fmt(v) { return v == null ? null : v; }
|
|
41
|
+
|
|
42
|
+
return items.map(row => ({
|
|
43
|
+
date: colIdx.timestamp != null ? new Date(row[colIdx.timestamp]).toISOString().split('T')[0] : null,
|
|
44
|
+
open: fmt(row[colIdx.open]),
|
|
45
|
+
high: fmt(row[colIdx.high]),
|
|
46
|
+
low: fmt(row[colIdx.low]),
|
|
47
|
+
close: fmt(row[colIdx.close]),
|
|
48
|
+
volume: fmt(row[colIdx.volume]),
|
|
49
|
+
amount: fmt(row[colIdx.amount]),
|
|
50
|
+
chg: fmt(row[colIdx.chg]),
|
|
51
|
+
percent: fmt(row[colIdx.percent]),
|
|
52
|
+
symbol: symbol
|
|
53
|
+
}));
|
|
54
|
+
})()
|
|
55
|
+
|
|
56
|
+
- map:
|
|
57
|
+
date: ${{ item.date }}
|
|
58
|
+
open: ${{ item.open }}
|
|
59
|
+
high: ${{ item.high }}
|
|
60
|
+
low: ${{ item.low }}
|
|
61
|
+
close: ${{ item.close }}
|
|
62
|
+
volume: ${{ item.volume }}
|
|
63
|
+
percent: ${{ item.percent }}
|
|
64
|
+
|
|
65
|
+
columns: [date, open, high, low, close, volume]
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
site: xueqiu
|
|
2
2
|
name: watchlist
|
|
3
|
-
description:
|
|
3
|
+
description: 获取雪球自选股/模拟组合股票列表
|
|
4
4
|
domain: xueqiu.com
|
|
5
5
|
browser: true
|
|
6
6
|
|
|
7
7
|
args:
|
|
8
|
-
|
|
9
|
-
type: str
|
|
10
|
-
default: "1"
|
|
11
|
-
description: "
|
|
8
|
+
pid:
|
|
9
|
+
type: str
|
|
10
|
+
default: "-1"
|
|
11
|
+
description: "分组ID:-1=全部(默认) -4=模拟 -5=沪深 -6=美股 -7=港股 -10=实盘 0=持仓(通过 xueqiu groups 获取)"
|
|
12
12
|
limit:
|
|
13
13
|
type: int
|
|
14
14
|
default: 100
|
|
@@ -18,12 +18,12 @@ pipeline:
|
|
|
18
18
|
- navigate: https://xueqiu.com
|
|
19
19
|
- evaluate: |
|
|
20
20
|
(async () => {
|
|
21
|
-
const
|
|
22
|
-
const resp = await fetch(`https://stock.xueqiu.com/v5/stock/portfolio/stock/list.json?size=100&category=${
|
|
21
|
+
const pid = ${{ args.pid | json }} || '-1';
|
|
22
|
+
const resp = await fetch(`https://stock.xueqiu.com/v5/stock/portfolio/stock/list.json?size=100&category=1&pid=${encodeURIComponent(pid)}`, {credentials: 'include'});
|
|
23
23
|
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
|
24
24
|
const d = await resp.json();
|
|
25
25
|
if (!d.data || !d.data.stocks) throw new Error('获取失败,可能未登录');
|
|
26
|
-
|
|
26
|
+
|
|
27
27
|
return d.data.stocks.map(s => ({
|
|
28
28
|
symbol: s.symbol,
|
|
29
29
|
name: s.name,
|
|
@@ -40,7 +40,7 @@ pipeline:
|
|
|
40
40
|
name: ${{ item.name }}
|
|
41
41
|
price: ${{ item.price }}
|
|
42
42
|
changePercent: ${{ item.changePercent }}
|
|
43
|
-
|
|
43
|
+
|
|
44
44
|
- limit: ${{ args.limit }}
|
|
45
45
|
|
|
46
46
|
columns: [symbol, name, price, changePercent]
|
package/dist/src/analysis.d.ts
CHANGED
|
@@ -29,6 +29,8 @@ export declare function inferStrategy(authIndicators: string[]): string;
|
|
|
29
29
|
export declare function detectAuthFromHeaders(headers?: Record<string, string>): string[];
|
|
30
30
|
/** Detect auth indicators from URL and response body (heuristic). */
|
|
31
31
|
export declare function detectAuthFromContent(url: string, body: unknown): string[];
|
|
32
|
+
/** Check whether a URL looks like tracking/telemetry noise rather than a business API. */
|
|
33
|
+
export declare function isNoiseUrl(url: string): boolean;
|
|
32
34
|
/** Extract non-volatile query params and classify them. */
|
|
33
35
|
export declare function classifyQueryParams(url: string): {
|
|
34
36
|
params: string[];
|
package/dist/src/analysis.js
CHANGED
|
@@ -148,6 +148,12 @@ export function detectAuthFromContent(url, body) {
|
|
|
148
148
|
indicators.push('bearer');
|
|
149
149
|
return indicators;
|
|
150
150
|
}
|
|
151
|
+
// ── Noise filtering ─────────────────────────────────────────────────────────
|
|
152
|
+
const NOISE_URL_PATTERN = /\/(track|log|analytics|beacon|pixel|ping|heartbeat|keep.?alive)\b/i;
|
|
153
|
+
/** Check whether a URL looks like tracking/telemetry noise rather than a business API. */
|
|
154
|
+
export function isNoiseUrl(url) {
|
|
155
|
+
return NOISE_URL_PATTERN.test(url);
|
|
156
|
+
}
|
|
151
157
|
// ── Query param classification ──────────────────────────────────────────────
|
|
152
158
|
/** Extract non-volatile query params and classify them. */
|
|
153
159
|
export function classifyQueryParams(url) {
|
package/dist/src/browser/cdp.js
CHANGED
|
@@ -136,6 +136,14 @@ export class CDPBridge {
|
|
|
136
136
|
class CDPPage extends BasePage {
|
|
137
137
|
bridge;
|
|
138
138
|
_pageEnabled = false;
|
|
139
|
+
// Network capture state (mirrors extension/src/cdp.ts NetworkCaptureEntry shape)
|
|
140
|
+
_networkCapturing = false;
|
|
141
|
+
_networkCapturePattern = '';
|
|
142
|
+
_networkEntries = [];
|
|
143
|
+
_pendingRequests = new Map(); // requestId → index in _networkEntries
|
|
144
|
+
_pendingBodyFetches = new Set(); // track in-flight getResponseBody calls
|
|
145
|
+
_consoleMessages = [];
|
|
146
|
+
_consoleCapturing = false;
|
|
139
147
|
constructor(bridge) {
|
|
140
148
|
super();
|
|
141
149
|
this.bridge = bridge;
|
|
@@ -186,6 +194,94 @@ class CDPPage extends BasePage {
|
|
|
186
194
|
}
|
|
187
195
|
return base64;
|
|
188
196
|
}
|
|
197
|
+
async startNetworkCapture(pattern = '') {
|
|
198
|
+
this._networkCapturePattern = pattern;
|
|
199
|
+
this._networkEntries = [];
|
|
200
|
+
this._pendingRequests.clear();
|
|
201
|
+
this._pendingBodyFetches.clear();
|
|
202
|
+
if (!this._networkCapturing) {
|
|
203
|
+
await this.bridge.send('Network.enable');
|
|
204
|
+
// Step 1: Record request method/url on requestWillBeSent
|
|
205
|
+
this.bridge.on('Network.requestWillBeSent', (params) => {
|
|
206
|
+
const p = params;
|
|
207
|
+
if (!pattern || p.request.url.includes(pattern)) {
|
|
208
|
+
const idx = this._networkEntries.push({
|
|
209
|
+
url: p.request.url,
|
|
210
|
+
method: p.request.method,
|
|
211
|
+
timestamp: p.timestamp,
|
|
212
|
+
}) - 1;
|
|
213
|
+
this._pendingRequests.set(p.requestId, idx);
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
// Step 2: Fill in response metadata on responseReceived
|
|
217
|
+
this.bridge.on('Network.responseReceived', (params) => {
|
|
218
|
+
const p = params;
|
|
219
|
+
const idx = this._pendingRequests.get(p.requestId);
|
|
220
|
+
if (idx !== undefined) {
|
|
221
|
+
this._networkEntries[idx].responseStatus = p.response.status;
|
|
222
|
+
this._networkEntries[idx].responseContentType = p.response.mimeType || '';
|
|
223
|
+
}
|
|
224
|
+
});
|
|
225
|
+
// Step 3: Fetch body on loadingFinished (body is only reliably available after this)
|
|
226
|
+
this.bridge.on('Network.loadingFinished', (params) => {
|
|
227
|
+
const p = params;
|
|
228
|
+
const idx = this._pendingRequests.get(p.requestId);
|
|
229
|
+
if (idx !== undefined) {
|
|
230
|
+
const bodyFetch = this.bridge.send('Network.getResponseBody', { requestId: p.requestId }).then((result) => {
|
|
231
|
+
const r = result;
|
|
232
|
+
if (typeof r?.body === 'string') {
|
|
233
|
+
this._networkEntries[idx].responsePreview = r.base64Encoded
|
|
234
|
+
? `base64:${r.body.slice(0, 4000)}`
|
|
235
|
+
: r.body.slice(0, 4000);
|
|
236
|
+
}
|
|
237
|
+
}).catch(() => {
|
|
238
|
+
// Body unavailable for some requests (e.g. uploads) — non-fatal
|
|
239
|
+
}).finally(() => {
|
|
240
|
+
this._pendingBodyFetches.delete(bodyFetch);
|
|
241
|
+
});
|
|
242
|
+
this._pendingBodyFetches.add(bodyFetch);
|
|
243
|
+
this._pendingRequests.delete(p.requestId);
|
|
244
|
+
}
|
|
245
|
+
});
|
|
246
|
+
this._networkCapturing = true;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
async readNetworkCapture() {
|
|
250
|
+
// Await all in-flight body fetches so entries have responsePreview populated
|
|
251
|
+
if (this._pendingBodyFetches.size > 0) {
|
|
252
|
+
await Promise.all([...this._pendingBodyFetches]);
|
|
253
|
+
}
|
|
254
|
+
const entries = [...this._networkEntries];
|
|
255
|
+
this._networkEntries = [];
|
|
256
|
+
return entries;
|
|
257
|
+
}
|
|
258
|
+
async consoleMessages(level = 'all') {
|
|
259
|
+
if (!this._consoleCapturing) {
|
|
260
|
+
await this.bridge.send('Runtime.enable');
|
|
261
|
+
this.bridge.on('Runtime.consoleAPICalled', (params) => {
|
|
262
|
+
const p = params;
|
|
263
|
+
const text = (p.args || []).map(a => a.value !== undefined ? String(a.value) : (a.description || '')).join(' ');
|
|
264
|
+
this._consoleMessages.push({ type: p.type, text, timestamp: p.timestamp });
|
|
265
|
+
if (this._consoleMessages.length > 500)
|
|
266
|
+
this._consoleMessages.shift();
|
|
267
|
+
});
|
|
268
|
+
// Capture uncaught exceptions as error-level messages
|
|
269
|
+
this.bridge.on('Runtime.exceptionThrown', (params) => {
|
|
270
|
+
const p = params;
|
|
271
|
+
const desc = p.exceptionDetails?.exception?.description || p.exceptionDetails?.text || 'Unknown exception';
|
|
272
|
+
this._consoleMessages.push({ type: 'error', text: desc, timestamp: p.timestamp });
|
|
273
|
+
if (this._consoleMessages.length > 500)
|
|
274
|
+
this._consoleMessages.shift();
|
|
275
|
+
});
|
|
276
|
+
this._consoleCapturing = true;
|
|
277
|
+
}
|
|
278
|
+
if (level === 'all')
|
|
279
|
+
return [...this._consoleMessages];
|
|
280
|
+
// 'error' level includes both console.error() and uncaught exceptions
|
|
281
|
+
if (level === 'error')
|
|
282
|
+
return this._consoleMessages.filter(m => m.type === 'error' || m.type === 'warning');
|
|
283
|
+
return this._consoleMessages.filter(m => m.type === level);
|
|
284
|
+
}
|
|
189
285
|
async tabs() {
|
|
190
286
|
return [];
|
|
191
287
|
}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* manifest.json for instant cold-start registration (no runtime YAML parsing).
|
|
7
7
|
*
|
|
8
8
|
* Usage: npx tsx src/build-manifest.ts
|
|
9
|
-
* Output:
|
|
9
|
+
* Output: cli-manifest.json at the package root
|
|
10
10
|
*/
|
|
11
11
|
export interface ManifestEntry {
|
|
12
12
|
site: string;
|
|
@@ -35,6 +35,8 @@ export interface ManifestEntry {
|
|
|
35
35
|
type: 'yaml' | 'ts';
|
|
36
36
|
/** Relative path from clis/ dir, e.g. 'bilibili/hot.yaml' or 'bilibili/search.js' */
|
|
37
37
|
modulePath?: string;
|
|
38
|
+
/** Relative path to the original source file from clis/ dir (for YAML: 'site/cmd.yaml') */
|
|
39
|
+
sourceFile?: string;
|
|
38
40
|
/** Pre-navigation control — see CliCommand.navigateBefore */
|
|
39
41
|
navigateBefore?: boolean | string;
|
|
40
42
|
}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* manifest.json for instant cold-start registration (no runtime YAML parsing).
|
|
7
7
|
*
|
|
8
8
|
* Usage: npx tsx src/build-manifest.ts
|
|
9
|
-
* Output:
|
|
9
|
+
* Output: cli-manifest.json at the package root
|
|
10
10
|
*/
|
|
11
11
|
import * as fs from 'node:fs';
|
|
12
12
|
import * as path from 'node:path';
|
|
@@ -14,9 +14,10 @@ import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
|
14
14
|
import yaml from 'js-yaml';
|
|
15
15
|
import { getErrorMessage } from './errors.js';
|
|
16
16
|
import { fullName, getRegistry } from './registry.js';
|
|
17
|
-
|
|
18
|
-
const
|
|
19
|
-
const
|
|
17
|
+
import { findPackageRoot, getCliManifestPath } from './package-paths.js';
|
|
18
|
+
const PACKAGE_ROOT = findPackageRoot(fileURLToPath(import.meta.url));
|
|
19
|
+
const CLIS_DIR = path.join(PACKAGE_ROOT, 'clis');
|
|
20
|
+
const OUTPUT = getCliManifestPath(CLIS_DIR);
|
|
20
21
|
import { parseYamlArgs } from './yaml-schema.js';
|
|
21
22
|
import { isRecord } from './utils.js';
|
|
22
23
|
const CLI_MODULE_PATTERN = /\bcli\s*\(/;
|
|
@@ -43,7 +44,7 @@ function isCliCommandValue(value, site) {
|
|
|
43
44
|
&& typeof value.name === 'string'
|
|
44
45
|
&& Array.isArray(value.args);
|
|
45
46
|
}
|
|
46
|
-
function toManifestEntry(cmd, modulePath) {
|
|
47
|
+
function toManifestEntry(cmd, modulePath, sourceFile) {
|
|
47
48
|
return {
|
|
48
49
|
site: cmd.site,
|
|
49
50
|
name: cmd.name,
|
|
@@ -59,6 +60,7 @@ function toManifestEntry(cmd, modulePath) {
|
|
|
59
60
|
replacedBy: cmd.replacedBy,
|
|
60
61
|
type: 'ts',
|
|
61
62
|
modulePath,
|
|
63
|
+
sourceFile,
|
|
62
64
|
navigateBefore: cmd.navigateBefore,
|
|
63
65
|
};
|
|
64
66
|
}
|
|
@@ -90,6 +92,7 @@ function scanYaml(filePath, site) {
|
|
|
90
92
|
deprecated: cliDef.deprecated,
|
|
91
93
|
replacedBy: cliDef.replacedBy,
|
|
92
94
|
type: 'yaml',
|
|
95
|
+
sourceFile: path.relative(CLIS_DIR, filePath),
|
|
93
96
|
navigateBefore: cliDef.navigateBefore,
|
|
94
97
|
};
|
|
95
98
|
}
|
|
@@ -130,7 +133,7 @@ export async function loadTsManifestEntries(filePath, site, importer = moduleHre
|
|
|
130
133
|
return true;
|
|
131
134
|
})
|
|
132
135
|
.sort((a, b) => a.name.localeCompare(b.name))
|
|
133
|
-
.map(cmd => toManifestEntry(cmd, modulePath));
|
|
136
|
+
.map(cmd => toManifestEntry(cmd, modulePath, path.relative(CLIS_DIR, filePath)));
|
|
134
137
|
}
|
|
135
138
|
catch (err) {
|
|
136
139
|
// If parsing fails, log a warning (matching scanYaml behaviour) and skip the entry.
|
|
@@ -200,7 +203,7 @@ async function main() {
|
|
|
200
203
|
// entry-point loses its executable permission, causing "Permission denied".
|
|
201
204
|
// See: https://github.com/jackwener/opencli/issues/446
|
|
202
205
|
if (process.platform !== 'win32') {
|
|
203
|
-
const projectRoot =
|
|
206
|
+
const projectRoot = PACKAGE_ROOT;
|
|
204
207
|
const pkgPath = path.resolve(projectRoot, 'package.json');
|
|
205
208
|
try {
|
|
206
209
|
const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8'));
|
|
@@ -87,7 +87,7 @@ describe('manifest helper rules', () => {
|
|
|
87
87
|
replacedBy: 'opencli demo new',
|
|
88
88
|
}),
|
|
89
89
|
}));
|
|
90
|
-
expect(entries).
|
|
90
|
+
expect(entries).toMatchObject([
|
|
91
91
|
{
|
|
92
92
|
site,
|
|
93
93
|
name: 'dynamic',
|
|
@@ -97,7 +97,7 @@ describe('manifest helper rules', () => {
|
|
|
97
97
|
browser: false,
|
|
98
98
|
aliases: ['metadata'],
|
|
99
99
|
args: [
|
|
100
|
-
{
|
|
100
|
+
expect.objectContaining({
|
|
101
101
|
name: 'model',
|
|
102
102
|
type: 'str',
|
|
103
103
|
required: true,
|
|
@@ -105,7 +105,7 @@ describe('manifest helper rules', () => {
|
|
|
105
105
|
help: 'Choose a model',
|
|
106
106
|
choices: ['auto', 'thinking'],
|
|
107
107
|
default: '30',
|
|
108
|
-
},
|
|
108
|
+
}),
|
|
109
109
|
],
|
|
110
110
|
type: 'ts',
|
|
111
111
|
modulePath: `${site}/${site}.js`,
|
|
@@ -114,6 +114,8 @@ describe('manifest helper rules', () => {
|
|
|
114
114
|
replacedBy: 'opencli demo new',
|
|
115
115
|
},
|
|
116
116
|
]);
|
|
117
|
+
// Verify sourceFile is included
|
|
118
|
+
expect(entries[0].sourceFile).toBeDefined();
|
|
117
119
|
getRegistry().delete(key);
|
|
118
120
|
});
|
|
119
121
|
it('falls back to registry delta for side-effect-only cli modules', async () => {
|
|
@@ -133,7 +135,7 @@ describe('manifest helper rules', () => {
|
|
|
133
135
|
});
|
|
134
136
|
return {};
|
|
135
137
|
});
|
|
136
|
-
expect(entries).
|
|
138
|
+
expect(entries).toMatchObject([
|
|
137
139
|
{
|
|
138
140
|
site,
|
|
139
141
|
name: 'legacy',
|
|
@@ -147,6 +149,8 @@ describe('manifest helper rules', () => {
|
|
|
147
149
|
replacedBy: 'opencli demo new',
|
|
148
150
|
},
|
|
149
151
|
]);
|
|
152
|
+
// Verify sourceFile is included
|
|
153
|
+
expect(entries[0].sourceFile).toBeDefined();
|
|
150
154
|
getRegistry().delete(key);
|
|
151
155
|
});
|
|
152
156
|
it('keeps every command a module exports instead of guessing by site', async () => {
|