@jackwener/opencli 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -2
- package/README.zh-CN.md +9 -1
- package/SKILL.md +24 -0
- package/dist/bilibili.d.ts +6 -5
- package/dist/browser.d.ts +3 -1
- package/dist/browser.js +29 -2
- package/dist/cascade.d.ts +3 -2
- package/dist/clis/bbc/news.js +42 -0
- package/dist/clis/boss/search.d.ts +1 -0
- package/dist/clis/boss/search.js +47 -0
- package/dist/clis/ctrip/search.d.ts +1 -0
- package/dist/clis/ctrip/search.js +62 -0
- package/dist/clis/index.d.ts +8 -0
- package/dist/clis/index.js +16 -0
- package/dist/clis/reuters/search.d.ts +1 -0
- package/dist/clis/reuters/search.js +52 -0
- package/dist/clis/smzdm/search.d.ts +1 -0
- package/dist/clis/smzdm/search.js +66 -0
- package/dist/clis/weibo/hot.d.ts +1 -0
- package/dist/clis/weibo/hot.js +41 -0
- package/dist/clis/yahoo-finance/quote.d.ts +1 -0
- package/dist/clis/yahoo-finance/quote.js +74 -0
- package/dist/clis/youtube/search.d.ts +1 -0
- package/dist/clis/youtube/search.js +60 -0
- package/dist/engine.d.ts +2 -1
- package/dist/explore.js +1 -1
- package/dist/generate.js +2 -1
- package/dist/main.js +6 -4
- package/dist/output.d.ts +1 -1
- package/dist/output.js +12 -8
- package/dist/pipeline/executor.d.ts +9 -0
- package/dist/pipeline/executor.js +88 -0
- package/dist/pipeline/index.d.ts +5 -0
- package/dist/pipeline/index.js +5 -0
- package/dist/pipeline/steps/browser.d.ts +12 -0
- package/dist/pipeline/steps/browser.js +68 -0
- package/dist/pipeline/steps/fetch.d.ts +5 -0
- package/dist/pipeline/steps/fetch.js +50 -0
- package/dist/pipeline/steps/intercept.d.ts +5 -0
- package/dist/pipeline/steps/intercept.js +75 -0
- package/dist/pipeline/steps/tap.d.ts +12 -0
- package/dist/pipeline/steps/tap.js +130 -0
- package/dist/pipeline/steps/transform.d.ts +8 -0
- package/dist/pipeline/steps/transform.js +53 -0
- package/dist/pipeline/template.d.ts +16 -0
- package/dist/pipeline/template.js +172 -0
- package/dist/pipeline/template.test.d.ts +4 -0
- package/dist/pipeline/template.test.js +120 -0
- package/dist/pipeline/transform.test.d.ts +4 -0
- package/dist/pipeline/transform.test.js +90 -0
- package/dist/pipeline.d.ts +5 -7
- package/dist/pipeline.js +5 -549
- package/dist/registry.d.ts +3 -2
- package/dist/runtime.d.ts +2 -1
- package/dist/types.d.ts +27 -0
- package/dist/types.js +7 -0
- package/package.json +6 -3
- package/src/bilibili.ts +9 -7
- package/src/browser.ts +24 -3
- package/src/cascade.ts +3 -2
- package/src/clis/bbc/news.ts +42 -0
- package/src/clis/boss/search.ts +47 -0
- package/src/clis/ctrip/search.ts +62 -0
- package/src/clis/index.ts +24 -0
- package/src/clis/reuters/search.ts +52 -0
- package/src/clis/smzdm/search.ts +66 -0
- package/src/clis/weibo/hot.ts +41 -0
- package/src/clis/yahoo-finance/quote.ts +74 -0
- package/src/clis/youtube/search.ts +60 -0
- package/src/engine.ts +2 -1
- package/src/explore.ts +1 -1
- package/src/generate.ts +3 -1
- package/src/main.ts +7 -5
- package/src/output.ts +10 -6
- package/src/pipeline/executor.ts +98 -0
- package/src/pipeline/index.ts +6 -0
- package/src/pipeline/steps/browser.ts +67 -0
- package/src/pipeline/steps/fetch.ts +60 -0
- package/src/pipeline/steps/intercept.ts +78 -0
- package/src/pipeline/steps/tap.ts +137 -0
- package/src/pipeline/steps/transform.ts +50 -0
- package/src/pipeline/template.test.ts +125 -0
- package/src/pipeline/template.ts +157 -0
- package/src/pipeline/transform.test.ts +107 -0
- package/src/pipeline.ts +5 -529
- package/src/registry.ts +4 -2
- package/src/runtime.ts +3 -1
- package/src/types.ts +23 -0
- package/vitest.config.ts +7 -0
- package/dist/clis/github/search.js +0 -20
- package/dist/clis/github/trending.yaml +0 -58
- package/dist/promote.d.ts +0 -1
- package/dist/promote.js +0 -3
- package/dist/register.d.ts +0 -2
- package/dist/register.js +0 -2
- package/dist/scaffold.d.ts +0 -2
- package/dist/scaffold.js +0 -2
- package/dist/smoke.d.ts +0 -2
- package/dist/smoke.js +0 -2
- package/src/clis/github/search.ts +0 -21
- package/src/clis/github/trending.yaml +0 -58
- package/src/promote.ts +0 -3
- package/src/register.ts +0 -2
- package/src/scaffold.ts +0 -2
- package/src/smoke.ts +0 -2
- /package/dist/clis/{github/search.d.ts → bbc/news.d.ts} +0 -0
package/src/bilibili.ts
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
* Bilibili shared helpers: WBI signing, authenticated fetch, nav data, UID resolution.
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
+
import type { IPage } from './types.js';
|
|
6
|
+
|
|
5
7
|
const MIXIN_KEY_ENC_TAB = [
|
|
6
8
|
46,47,18,2,53,8,23,32,15,50,10,31,58,3,45,35,27,43,5,49,
|
|
7
9
|
33,9,42,19,29,28,14,39,12,38,41,13,37,48,7,16,24,55,40,
|
|
@@ -17,7 +19,7 @@ export function payloadData(payload: any): any {
|
|
|
17
19
|
return payload?.data ?? payload;
|
|
18
20
|
}
|
|
19
21
|
|
|
20
|
-
async function getNavData(page:
|
|
22
|
+
async function getNavData(page: IPage): Promise<any> {
|
|
21
23
|
return page.evaluate(`
|
|
22
24
|
async () => {
|
|
23
25
|
const res = await fetch('https://api.bilibili.com/x/web-interface/nav', { credentials: 'include' });
|
|
@@ -26,7 +28,7 @@ async function getNavData(page: any): Promise<any> {
|
|
|
26
28
|
`);
|
|
27
29
|
}
|
|
28
30
|
|
|
29
|
-
async function getWbiKeys(page:
|
|
31
|
+
async function getWbiKeys(page: IPage): Promise<{ imgKey: string; subKey: string }> {
|
|
30
32
|
const nav = await getNavData(page);
|
|
31
33
|
const wbiImg = nav?.data?.wbi_img ?? {};
|
|
32
34
|
const imgUrl = wbiImg.img_url ?? '';
|
|
@@ -47,7 +49,7 @@ async function md5(text: string): Promise<string> {
|
|
|
47
49
|
}
|
|
48
50
|
|
|
49
51
|
export async function wbiSign(
|
|
50
|
-
page:
|
|
52
|
+
page: IPage,
|
|
51
53
|
params: Record<string, any>,
|
|
52
54
|
): Promise<Record<string, string>> {
|
|
53
55
|
const { imgKey, subKey } = await getWbiKeys(page);
|
|
@@ -65,7 +67,7 @@ export async function wbiSign(
|
|
|
65
67
|
}
|
|
66
68
|
|
|
67
69
|
export async function apiGet(
|
|
68
|
-
page:
|
|
70
|
+
page: IPage,
|
|
69
71
|
path: string,
|
|
70
72
|
opts: { params?: Record<string, any>; signed?: boolean } = {},
|
|
71
73
|
): Promise<any> {
|
|
@@ -81,7 +83,7 @@ export async function apiGet(
|
|
|
81
83
|
return fetchJson(page, url);
|
|
82
84
|
}
|
|
83
85
|
|
|
84
|
-
export async function fetchJson(page:
|
|
86
|
+
export async function fetchJson(page: IPage, url: string): Promise<any> {
|
|
85
87
|
const escapedUrl = url.replace(/"/g, '\\"');
|
|
86
88
|
return page.evaluate(`
|
|
87
89
|
async () => {
|
|
@@ -91,14 +93,14 @@ export async function fetchJson(page: any, url: string): Promise<any> {
|
|
|
91
93
|
`);
|
|
92
94
|
}
|
|
93
95
|
|
|
94
|
-
export async function getSelfUid(page:
|
|
96
|
+
export async function getSelfUid(page: IPage): Promise<string> {
|
|
95
97
|
const nav = await getNavData(page);
|
|
96
98
|
const mid = nav?.data?.mid;
|
|
97
99
|
if (!mid) throw new Error('Not logged in to Bilibili');
|
|
98
100
|
return String(mid);
|
|
99
101
|
}
|
|
100
102
|
|
|
101
|
-
export async function resolveUid(page:
|
|
103
|
+
export async function resolveUid(page: IPage, input: string): Promise<string> {
|
|
102
104
|
if (/^\d+$/.test(input)) return input;
|
|
103
105
|
// Search for user by name
|
|
104
106
|
const payload = await apiGet(page, '/x/web-interface/wbi/search/type', {
|
package/src/browser.ts
CHANGED
|
@@ -10,6 +10,10 @@ import * as os from 'node:os';
|
|
|
10
10
|
import * as path from 'node:path';
|
|
11
11
|
import { formatSnapshot } from './snapshotFormatter.js';
|
|
12
12
|
|
|
13
|
+
// Read version from package.json (single source of truth)
|
|
14
|
+
const __browser_dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
15
|
+
const PKG_VERSION = (() => { try { return JSON.parse(fs.readFileSync(path.resolve(__browser_dirname, '..', 'package.json'), 'utf-8')).version; } catch { return '0.0.0'; } })();
|
|
16
|
+
|
|
13
17
|
const EXTENSION_LOCK_TIMEOUT = parseInt(process.env.OPENCLI_EXTENSION_LOCK_TIMEOUT ?? '120', 10);
|
|
14
18
|
const EXTENSION_LOCK_POLL = parseInt(process.env.OPENCLI_EXTENSION_LOCK_POLL_INTERVAL ?? '1', 10);
|
|
15
19
|
const CONNECT_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_CONNECT_TIMEOUT ?? '30', 10);
|
|
@@ -21,10 +25,12 @@ function jsonRpcRequest(method: string, params: Record<string, any> = {}): strin
|
|
|
21
25
|
return JSON.stringify({ jsonrpc: '2.0', id: _nextId++, method, params }) + '\n';
|
|
22
26
|
}
|
|
23
27
|
|
|
28
|
+
import type { IPage } from './types.js';
|
|
29
|
+
|
|
24
30
|
/**
|
|
25
31
|
* Page abstraction wrapping JSON-RPC calls to Playwright MCP.
|
|
26
32
|
*/
|
|
27
|
-
export class Page {
|
|
33
|
+
export class Page implements IPage {
|
|
28
34
|
constructor(private _send: (msg: string) => void, private _recv: () => Promise<any>) {}
|
|
29
35
|
|
|
30
36
|
async call(method: string, params: Record<string, any> = {}): Promise<any> {
|
|
@@ -61,7 +67,22 @@ export class Page {
|
|
|
61
67
|
}
|
|
62
68
|
|
|
63
69
|
async evaluate(js: string): Promise<any> {
|
|
64
|
-
|
|
70
|
+
// Normalize IIFE format to function format expected by MCP browser_evaluate
|
|
71
|
+
const normalized = this.normalizeEval(js);
|
|
72
|
+
return this.call('tools/call', { name: 'browser_evaluate', arguments: { function: normalized } });
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
private normalizeEval(source: string): string {
|
|
76
|
+
const s = source.trim();
|
|
77
|
+
if (!s) return '() => undefined';
|
|
78
|
+
// IIFE: (async () => {...})() → wrap as () => (...)
|
|
79
|
+
if (s.startsWith('(') && s.endsWith(')()')) return `() => (${s})`;
|
|
80
|
+
// Already a function/arrow
|
|
81
|
+
if (/^(async\s+)?\([^)]*\)\s*=>/.test(s)) return s;
|
|
82
|
+
if (/^(async\s+)?[A-Za-z_][A-Za-z0-9_]*\s*=>/.test(s)) return s;
|
|
83
|
+
if (s.startsWith('function ') || s.startsWith('async function ')) return s;
|
|
84
|
+
// Raw expression → wrap
|
|
85
|
+
return `() => (${s})`;
|
|
65
86
|
}
|
|
66
87
|
|
|
67
88
|
async snapshot(opts: { interactive?: boolean; compact?: boolean; maxDepth?: number; raw?: boolean } = {}): Promise<any> {
|
|
@@ -173,7 +194,7 @@ export class PlaywrightMCP {
|
|
|
173
194
|
const initMsg = jsonRpcRequest('initialize', {
|
|
174
195
|
protocolVersion: '2024-11-05',
|
|
175
196
|
capabilities: {},
|
|
176
|
-
clientInfo: { name: 'opencli', version:
|
|
197
|
+
clientInfo: { name: 'opencli', version: PKG_VERSION },
|
|
177
198
|
});
|
|
178
199
|
this._proc.stdin?.write(initMsg);
|
|
179
200
|
|
package/src/cascade.ts
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import { Strategy } from './registry.js';
|
|
14
|
+
import type { IPage } from './types.js';
|
|
14
15
|
|
|
15
16
|
/** Strategy cascade order (simplest → most complex) */
|
|
16
17
|
const CASCADE_ORDER: Strategy[] = [
|
|
@@ -41,7 +42,7 @@ interface CascadeResult {
|
|
|
41
42
|
* Returns whether the probe succeeded and basic response info.
|
|
42
43
|
*/
|
|
43
44
|
export async function probeEndpoint(
|
|
44
|
-
page:
|
|
45
|
+
page: IPage,
|
|
45
46
|
url: string,
|
|
46
47
|
strategy: Strategy,
|
|
47
48
|
opts: { timeout?: number } = {},
|
|
@@ -168,7 +169,7 @@ export async function probeEndpoint(
|
|
|
168
169
|
* Returns the simplest working strategy.
|
|
169
170
|
*/
|
|
170
171
|
export async function cascadeProbe(
|
|
171
|
-
page:
|
|
172
|
+
page: IPage,
|
|
172
173
|
url: string,
|
|
173
174
|
opts: { maxStrategy?: Strategy; timeout?: number } = {},
|
|
174
175
|
): Promise<CascadeResult> {
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BBC News headlines — public RSS feed, no browser needed.
|
|
3
|
+
* Source: bb-sites/bbc/news.js
|
|
4
|
+
*/
|
|
5
|
+
import { cli, Strategy } from '../../registry.js';
|
|
6
|
+
|
|
7
|
+
cli({
|
|
8
|
+
site: 'bbc',
|
|
9
|
+
name: 'news',
|
|
10
|
+
description: 'BBC News headlines (RSS)',
|
|
11
|
+
domain: 'www.bbc.com',
|
|
12
|
+
strategy: Strategy.PUBLIC,
|
|
13
|
+
args: [
|
|
14
|
+
{ name: 'limit', type: 'int', default: 20, help: 'Number of headlines (max 50)' },
|
|
15
|
+
],
|
|
16
|
+
columns: ['rank', 'title', 'description', 'url'],
|
|
17
|
+
func: async (page, kwargs) => {
|
|
18
|
+
const count = Math.min(kwargs.limit || 20, 50);
|
|
19
|
+
const resp = await fetch('https://feeds.bbci.co.uk/news/rss.xml');
|
|
20
|
+
if (!resp.ok) return [];
|
|
21
|
+
const xml = await resp.text();
|
|
22
|
+
// Simple XML parsing without DOMParser (works in Node)
|
|
23
|
+
const items: any[] = [];
|
|
24
|
+
const itemRegex = /<item>([\s\S]*?)<\/item>/g;
|
|
25
|
+
let match;
|
|
26
|
+
while ((match = itemRegex.exec(xml)) && items.length < count) {
|
|
27
|
+
const block = match[1];
|
|
28
|
+
const title = block.match(/<title><!\[CDATA\[(.*?)\]\]>|<title>(.*?)<\/title>/)?.[1] || block.match(/<title>(.*?)<\/title>/)?.[1] || '';
|
|
29
|
+
const desc = block.match(/<description><!\[CDATA\[(.*?)\]\]>|<description>(.*?)<\/description>/)?.[1] || block.match(/<description>(.*?)<\/description>/)?.[1] || '';
|
|
30
|
+
const link = block.match(/<link>(.*?)<\/link>/)?.[1] || block.match(/<guid[^>]*>(.*?)<\/guid>/)?.[1] || '';
|
|
31
|
+
if (title) {
|
|
32
|
+
items.push({
|
|
33
|
+
rank: items.length + 1,
|
|
34
|
+
title: title.trim(),
|
|
35
|
+
description: desc.trim().substring(0, 200),
|
|
36
|
+
url: link.trim(),
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return items;
|
|
41
|
+
},
|
|
42
|
+
});
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BOSS直聘 job search — browser cookie API.
|
|
3
|
+
* Source: bb-sites/boss/search.js
|
|
4
|
+
*/
|
|
5
|
+
import { cli, Strategy } from '../../registry.js';
|
|
6
|
+
|
|
7
|
+
cli({
|
|
8
|
+
site: 'boss',
|
|
9
|
+
name: 'search',
|
|
10
|
+
description: 'BOSS直聘搜索职位',
|
|
11
|
+
domain: 'www.zhipin.com',
|
|
12
|
+
strategy: Strategy.COOKIE,
|
|
13
|
+
args: [
|
|
14
|
+
{ name: 'query', required: true, help: 'Search keyword (e.g. AI agent, 前端)' },
|
|
15
|
+
{ name: 'city', default: '101010100', help: 'City code (101010100=北京, 101020100=上海, 101210100=杭州, 101280100=广州)' },
|
|
16
|
+
{ name: 'limit', type: 'int', default: 15, help: 'Number of results' },
|
|
17
|
+
],
|
|
18
|
+
columns: ['name', 'salary', 'company', 'city', 'experience', 'degree', 'boss', 'url'],
|
|
19
|
+
func: async (page, kwargs) => {
|
|
20
|
+
await page.goto('https://www.zhipin.com');
|
|
21
|
+
await page.wait(2);
|
|
22
|
+
const data = await page.evaluate(`
|
|
23
|
+
(async () => {
|
|
24
|
+
const params = new URLSearchParams({
|
|
25
|
+
scene: '1', query: '${kwargs.query.replace(/'/g, "\\'")}',
|
|
26
|
+
city: '${kwargs.city || '101010100'}', page: '1', pageSize: '15',
|
|
27
|
+
experience: '', degree: '', payType: '', partTime: '',
|
|
28
|
+
industry: '', scale: '', stage: '', position: '',
|
|
29
|
+
jobType: '', salary: '', multiBusinessDistrict: '', multiSubway: ''
|
|
30
|
+
});
|
|
31
|
+
const resp = await fetch('/wapi/zpgeek/search/joblist.json?' + params.toString(), {credentials: 'include'});
|
|
32
|
+
if (!resp.ok) return {error: 'HTTP ' + resp.status};
|
|
33
|
+
const d = await resp.json();
|
|
34
|
+
if (d.code !== 0) return {error: d.message || 'API error'};
|
|
35
|
+
const zpData = d.zpData || {};
|
|
36
|
+
return (zpData.jobList || []).map(j => ({
|
|
37
|
+
name: j.jobName, salary: j.salaryDesc, company: j.brandName,
|
|
38
|
+
city: j.cityName, experience: j.jobExperience, degree: j.jobDegree,
|
|
39
|
+
boss: j.bossName + ' · ' + j.bossTitle,
|
|
40
|
+
url: j.encryptJobId ? 'https://www.zhipin.com/job_detail/' + j.encryptJobId + '.html' : ''
|
|
41
|
+
}));
|
|
42
|
+
})()
|
|
43
|
+
`);
|
|
44
|
+
if (!Array.isArray(data)) return [];
|
|
45
|
+
return data.slice(0, kwargs.limit || 15);
|
|
46
|
+
},
|
|
47
|
+
});
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 携程旅行搜索 — browser cookie, multi-strategy.
|
|
3
|
+
* Source: bb-sites/ctrip/search.js (simplified to suggestion API)
|
|
4
|
+
*/
|
|
5
|
+
import { cli, Strategy } from '../../registry.js';
|
|
6
|
+
|
|
7
|
+
cli({
|
|
8
|
+
site: 'ctrip',
|
|
9
|
+
name: 'search',
|
|
10
|
+
description: '携程旅行搜索',
|
|
11
|
+
domain: 'www.ctrip.com',
|
|
12
|
+
strategy: Strategy.COOKIE,
|
|
13
|
+
args: [
|
|
14
|
+
{ name: 'query', required: true, help: 'Search keyword (city or attraction)' },
|
|
15
|
+
{ name: 'limit', type: 'int', default: 15, help: 'Number of results' },
|
|
16
|
+
],
|
|
17
|
+
columns: ['rank', 'name', 'type', 'score', 'price', 'url'],
|
|
18
|
+
func: async (page, kwargs) => {
|
|
19
|
+
const limit = kwargs.limit || 15;
|
|
20
|
+
await page.goto('https://www.ctrip.com');
|
|
21
|
+
await page.wait(2);
|
|
22
|
+
const data = await page.evaluate(`
|
|
23
|
+
(async () => {
|
|
24
|
+
const query = '${kwargs.query.replace(/'/g, "\\'")}';
|
|
25
|
+
const limit = ${limit};
|
|
26
|
+
|
|
27
|
+
// Strategy 1: Suggestion API
|
|
28
|
+
try {
|
|
29
|
+
const suggestUrl = 'https://m.ctrip.com/restapi/h5api/searchapp/search?action=onekeyali&keyword=' + encodeURIComponent(query);
|
|
30
|
+
const resp = await fetch(suggestUrl, {credentials: 'include'});
|
|
31
|
+
if (resp.ok) {
|
|
32
|
+
const d = await resp.json();
|
|
33
|
+
const raw = d.data || d.result || d;
|
|
34
|
+
if (raw && typeof raw === 'object') {
|
|
35
|
+
// Flatten all result categories
|
|
36
|
+
const items = [];
|
|
37
|
+
for (const key of Object.keys(raw)) {
|
|
38
|
+
const list = Array.isArray(raw[key]) ? raw[key] : [];
|
|
39
|
+
for (const item of list) {
|
|
40
|
+
if (items.length >= limit) break;
|
|
41
|
+
items.push({
|
|
42
|
+
rank: items.length + 1,
|
|
43
|
+
name: item.word || item.name || item.title || '',
|
|
44
|
+
type: item.type || item.tpName || key,
|
|
45
|
+
score: item.score || '',
|
|
46
|
+
price: item.price || item.minPrice || '',
|
|
47
|
+
url: item.url || item.surl || '',
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
if (items.length > 0) return items;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
} catch(e) {}
|
|
55
|
+
|
|
56
|
+
return {error: 'No results for: ' + query};
|
|
57
|
+
})()
|
|
58
|
+
`);
|
|
59
|
+
if (!Array.isArray(data)) return [];
|
|
60
|
+
return data;
|
|
61
|
+
},
|
|
62
|
+
});
|
package/src/clis/index.ts
CHANGED
|
@@ -20,3 +20,27 @@ import './zhihu/question.js';
|
|
|
20
20
|
|
|
21
21
|
// xiaohongshu
|
|
22
22
|
import './xiaohongshu/search.js';
|
|
23
|
+
|
|
24
|
+
// bbc
|
|
25
|
+
import './bbc/news.js';
|
|
26
|
+
|
|
27
|
+
// weibo
|
|
28
|
+
import './weibo/hot.js';
|
|
29
|
+
|
|
30
|
+
// boss
|
|
31
|
+
import './boss/search.js';
|
|
32
|
+
|
|
33
|
+
// yahoo-finance
|
|
34
|
+
import './yahoo-finance/quote.js';
|
|
35
|
+
|
|
36
|
+
// reuters
|
|
37
|
+
import './reuters/search.js';
|
|
38
|
+
|
|
39
|
+
// smzdm
|
|
40
|
+
import './smzdm/search.js';
|
|
41
|
+
|
|
42
|
+
// ctrip
|
|
43
|
+
import './ctrip/search.js';
|
|
44
|
+
|
|
45
|
+
// youtube
|
|
46
|
+
import './youtube/search.js';
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reuters news search — API with HTML fallback.
|
|
3
|
+
* Source: bb-sites/reuters/search.js
|
|
4
|
+
*/
|
|
5
|
+
import { cli, Strategy } from '../../registry.js';
|
|
6
|
+
|
|
7
|
+
cli({
|
|
8
|
+
site: 'reuters',
|
|
9
|
+
name: 'search',
|
|
10
|
+
description: 'Reuters 路透社新闻搜索',
|
|
11
|
+
domain: 'www.reuters.com',
|
|
12
|
+
strategy: Strategy.COOKIE,
|
|
13
|
+
args: [
|
|
14
|
+
{ name: 'query', required: true, help: 'Search query' },
|
|
15
|
+
{ name: 'limit', type: 'int', default: 10, help: 'Number of results (max 40)' },
|
|
16
|
+
],
|
|
17
|
+
columns: ['rank', 'title', 'date', 'section', 'url'],
|
|
18
|
+
func: async (page, kwargs) => {
|
|
19
|
+
const count = Math.min(kwargs.limit || 10, 40);
|
|
20
|
+
await page.goto('https://www.reuters.com');
|
|
21
|
+
await page.wait(2);
|
|
22
|
+
const data = await page.evaluate(`
|
|
23
|
+
(async () => {
|
|
24
|
+
const count = ${count};
|
|
25
|
+
const apiQuery = JSON.stringify({
|
|
26
|
+
keyword: '${kwargs.query.replace(/'/g, "\\'")}',
|
|
27
|
+
offset: 0, orderby: 'display_date:desc', size: count, website: 'reuters'
|
|
28
|
+
});
|
|
29
|
+
const apiUrl = 'https://www.reuters.com/pf/api/v3/content/fetch/articles-by-search-v2?query=' + encodeURIComponent(apiQuery);
|
|
30
|
+
try {
|
|
31
|
+
const resp = await fetch(apiUrl, {credentials: 'include'});
|
|
32
|
+
if (resp.ok) {
|
|
33
|
+
const data = await resp.json();
|
|
34
|
+
const articles = data.result?.articles || data.articles || [];
|
|
35
|
+
if (articles.length > 0) {
|
|
36
|
+
return articles.slice(0, count).map((a, i) => ({
|
|
37
|
+
rank: i + 1,
|
|
38
|
+
title: a.title || a.headlines?.basic || '',
|
|
39
|
+
date: (a.display_date || a.published_time || '').split('T')[0],
|
|
40
|
+
section: a.taxonomy?.section?.name || '',
|
|
41
|
+
url: a.canonical_url ? 'https://www.reuters.com' + a.canonical_url : '',
|
|
42
|
+
}));
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
} catch(e) {}
|
|
46
|
+
return {error: 'Reuters API unavailable'};
|
|
47
|
+
})()
|
|
48
|
+
`);
|
|
49
|
+
if (!Array.isArray(data)) return [];
|
|
50
|
+
return data;
|
|
51
|
+
},
|
|
52
|
+
});
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 什么值得买搜索好价 — browser cookie, HTML parse.
|
|
3
|
+
* Source: bb-sites/smzdm/search.js
|
|
4
|
+
*/
|
|
5
|
+
import { cli, Strategy } from '../../registry.js';
|
|
6
|
+
|
|
7
|
+
cli({
|
|
8
|
+
site: 'smzdm',
|
|
9
|
+
name: 'search',
|
|
10
|
+
description: '什么值得买搜索好价',
|
|
11
|
+
domain: 'www.smzdm.com',
|
|
12
|
+
strategy: Strategy.COOKIE,
|
|
13
|
+
args: [
|
|
14
|
+
{ name: 'keyword', required: true, help: 'Search keyword' },
|
|
15
|
+
{ name: 'limit', type: 'int', default: 20, help: 'Number of results' },
|
|
16
|
+
],
|
|
17
|
+
columns: ['rank', 'title', 'price', 'mall', 'comments', 'url'],
|
|
18
|
+
func: async (page, kwargs) => {
|
|
19
|
+
const q = encodeURIComponent(kwargs.keyword);
|
|
20
|
+
const limit = kwargs.limit || 20;
|
|
21
|
+
await page.goto('https://www.smzdm.com');
|
|
22
|
+
await page.wait(2);
|
|
23
|
+
const data = await page.evaluate(`
|
|
24
|
+
(async () => {
|
|
25
|
+
const q = '${q}';
|
|
26
|
+
const limit = ${limit};
|
|
27
|
+
// Try youhui channel first, then home
|
|
28
|
+
for (const channel of ['youhui', 'home']) {
|
|
29
|
+
try {
|
|
30
|
+
const resp = await fetch('https://search.smzdm.com/ajax/?c=' + channel + '&s=' + q + '&p=1&v=b', {
|
|
31
|
+
credentials: 'include',
|
|
32
|
+
headers: {'X-Requested-With': 'XMLHttpRequest'}
|
|
33
|
+
});
|
|
34
|
+
if (!resp.ok) continue;
|
|
35
|
+
const html = await resp.text();
|
|
36
|
+
if (html.indexOf('feed-row-wide') === -1) continue;
|
|
37
|
+
const parser = new DOMParser();
|
|
38
|
+
const doc = parser.parseFromString(html, 'text/html');
|
|
39
|
+
const items = doc.querySelectorAll('li.feed-row-wide');
|
|
40
|
+
const results = [];
|
|
41
|
+
items.forEach((li, i) => {
|
|
42
|
+
if (results.length >= limit) return;
|
|
43
|
+
const titleEl = li.querySelector('h5.feed-block-title > a')
|
|
44
|
+
|| li.querySelector('h5 > a');
|
|
45
|
+
if (!titleEl) return;
|
|
46
|
+
const title = (titleEl.getAttribute('title') || titleEl.textContent || '').trim();
|
|
47
|
+
const url = titleEl.getAttribute('href') || '';
|
|
48
|
+
const priceEl = li.querySelector('.z-highlight');
|
|
49
|
+
const price = priceEl ? priceEl.textContent.trim() : '';
|
|
50
|
+
let mall = '';
|
|
51
|
+
const extrasSpan = li.querySelector('.z-feed-foot-r .feed-block-extras span');
|
|
52
|
+
if (extrasSpan) mall = extrasSpan.textContent.trim();
|
|
53
|
+
const commentEl = li.querySelector('.feed-btn-comment');
|
|
54
|
+
const comments = commentEl ? parseInt(commentEl.textContent.trim()) || 0 : 0;
|
|
55
|
+
results.push({rank: results.length + 1, title, price, mall, comments, url});
|
|
56
|
+
});
|
|
57
|
+
if (results.length > 0) return results;
|
|
58
|
+
} catch(e) { continue; }
|
|
59
|
+
}
|
|
60
|
+
return {error: 'No results'};
|
|
61
|
+
})()
|
|
62
|
+
`);
|
|
63
|
+
if (!Array.isArray(data)) return [];
|
|
64
|
+
return data;
|
|
65
|
+
},
|
|
66
|
+
});
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Weibo hot search — browser cookie API.
|
|
3
|
+
* Source: bb-sites/weibo/hot.js
|
|
4
|
+
*/
|
|
5
|
+
import { cli, Strategy } from '../../registry.js';
|
|
6
|
+
|
|
7
|
+
cli({
|
|
8
|
+
site: 'weibo',
|
|
9
|
+
name: 'hot',
|
|
10
|
+
description: '微博热搜',
|
|
11
|
+
domain: 'weibo.com',
|
|
12
|
+
strategy: Strategy.COOKIE,
|
|
13
|
+
args: [
|
|
14
|
+
{ name: 'limit', type: 'int', default: 30, help: 'Number of items (max 50)' },
|
|
15
|
+
],
|
|
16
|
+
columns: ['rank', 'word', 'hot_value', 'category', 'label', 'url'],
|
|
17
|
+
func: async (page, kwargs) => {
|
|
18
|
+
const count = Math.min(kwargs.limit || 30, 50);
|
|
19
|
+
await page.goto('https://weibo.com');
|
|
20
|
+
await page.wait(2);
|
|
21
|
+
const data = await page.evaluate(`
|
|
22
|
+
(async () => {
|
|
23
|
+
const resp = await fetch('/ajax/statuses/hot_band', {credentials: 'include'});
|
|
24
|
+
if (!resp.ok) return {error: 'HTTP ' + resp.status};
|
|
25
|
+
const data = await resp.json();
|
|
26
|
+
if (!data.ok) return {error: 'API error'};
|
|
27
|
+
const bandList = data.data?.band_list || [];
|
|
28
|
+
return bandList.map((item, i) => ({
|
|
29
|
+
rank: item.realpos || (i + 1),
|
|
30
|
+
word: item.word,
|
|
31
|
+
hot_value: item.num || 0,
|
|
32
|
+
category: item.category || '',
|
|
33
|
+
label: item.label_name || '',
|
|
34
|
+
url: 'https://s.weibo.com/weibo?q=' + encodeURIComponent('#' + item.word + '#')
|
|
35
|
+
}));
|
|
36
|
+
})()
|
|
37
|
+
`);
|
|
38
|
+
if (!Array.isArray(data)) return [];
|
|
39
|
+
return data.slice(0, count);
|
|
40
|
+
},
|
|
41
|
+
});
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Yahoo Finance stock quote — multi-strategy API fallback.
|
|
3
|
+
* Source: bb-sites/yahoo-finance/quote.js
|
|
4
|
+
*/
|
|
5
|
+
import { cli, Strategy } from '../../registry.js';
|
|
6
|
+
|
|
7
|
+
cli({
|
|
8
|
+
site: 'yahoo-finance',
|
|
9
|
+
name: 'quote',
|
|
10
|
+
description: 'Yahoo Finance 股票行情',
|
|
11
|
+
domain: 'finance.yahoo.com',
|
|
12
|
+
strategy: Strategy.COOKIE,
|
|
13
|
+
args: [
|
|
14
|
+
{ name: 'symbol', required: true, help: 'Stock ticker (e.g. AAPL, MSFT, TSLA)' },
|
|
15
|
+
],
|
|
16
|
+
columns: ['symbol', 'name', 'price', 'change', 'changePercent', 'open', 'high', 'low', 'volume', 'marketCap'],
|
|
17
|
+
func: async (page, kwargs) => {
|
|
18
|
+
const symbol = kwargs.symbol.toUpperCase().trim();
|
|
19
|
+
await page.goto(`https://finance.yahoo.com/quote/${encodeURIComponent(symbol)}/`);
|
|
20
|
+
await page.wait(3);
|
|
21
|
+
const data = await page.evaluate(`
|
|
22
|
+
(async () => {
|
|
23
|
+
const sym = '${symbol}';
|
|
24
|
+
|
|
25
|
+
// Strategy 1: v8 chart API
|
|
26
|
+
try {
|
|
27
|
+
const chartUrl = 'https://query1.finance.yahoo.com/v8/finance/chart/' + encodeURIComponent(sym) + '?interval=1d&range=1d';
|
|
28
|
+
const resp = await fetch(chartUrl);
|
|
29
|
+
if (resp.ok) {
|
|
30
|
+
const d = await resp.json();
|
|
31
|
+
const chart = d?.chart?.result?.[0];
|
|
32
|
+
if (chart) {
|
|
33
|
+
const meta = chart.meta || {};
|
|
34
|
+
const prevClose = meta.previousClose || meta.chartPreviousClose;
|
|
35
|
+
const price = meta.regularMarketPrice;
|
|
36
|
+
const change = price != null && prevClose != null ? (price - prevClose) : null;
|
|
37
|
+
const changePct = change != null && prevClose ? ((change / prevClose) * 100) : null;
|
|
38
|
+
return {
|
|
39
|
+
symbol: meta.symbol || sym, name: meta.shortName || meta.longName || sym,
|
|
40
|
+
price: price != null ? Number(price.toFixed(2)) : null,
|
|
41
|
+
change: change != null ? change.toFixed(2) : null,
|
|
42
|
+
changePercent: changePct != null ? changePct.toFixed(2) + '%' : null,
|
|
43
|
+
open: chart.indicators?.quote?.[0]?.open?.[0] || null,
|
|
44
|
+
high: meta.regularMarketDayHigh || null,
|
|
45
|
+
low: meta.regularMarketDayLow || null,
|
|
46
|
+
volume: meta.regularMarketVolume || null,
|
|
47
|
+
marketCap: null, currency: meta.currency, exchange: meta.exchangeName,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
} catch(e) {}
|
|
52
|
+
|
|
53
|
+
// Strategy 2: Parse from page
|
|
54
|
+
const titleEl = document.querySelector('title');
|
|
55
|
+
const priceEl = document.querySelector('[data-testid="qsp-price"]');
|
|
56
|
+
const changeEl = document.querySelector('[data-testid="qsp-price-change"]');
|
|
57
|
+
const changePctEl = document.querySelector('[data-testid="qsp-price-change-percent"]');
|
|
58
|
+
if (priceEl) {
|
|
59
|
+
return {
|
|
60
|
+
symbol: sym,
|
|
61
|
+
name: titleEl ? titleEl.textContent.split('(')[0].trim() : sym,
|
|
62
|
+
price: priceEl.textContent.replace(/,/g, ''),
|
|
63
|
+
change: changeEl ? changeEl.textContent : null,
|
|
64
|
+
changePercent: changePctEl ? changePctEl.textContent : null,
|
|
65
|
+
open: null, high: null, low: null, volume: null, marketCap: null,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
return {error: 'Could not fetch quote for ' + sym};
|
|
69
|
+
})()
|
|
70
|
+
`);
|
|
71
|
+
if (!data || data.error) return [];
|
|
72
|
+
return [data];
|
|
73
|
+
},
|
|
74
|
+
});
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* YouTube search — innertube API via browser session.
|
|
3
|
+
* Source: bb-sites/youtube/search.js
|
|
4
|
+
*/
|
|
5
|
+
import { cli, Strategy } from '../../registry.js';
|
|
6
|
+
|
|
7
|
+
cli({
|
|
8
|
+
site: 'youtube',
|
|
9
|
+
name: 'search',
|
|
10
|
+
description: 'Search YouTube videos',
|
|
11
|
+
domain: 'www.youtube.com',
|
|
12
|
+
strategy: Strategy.COOKIE,
|
|
13
|
+
args: [
|
|
14
|
+
{ name: 'query', required: true, help: 'Search query' },
|
|
15
|
+
{ name: 'limit', type: 'int', default: 20, help: 'Max results (max 50)' },
|
|
16
|
+
],
|
|
17
|
+
columns: ['rank', 'title', 'channel', 'views', 'duration', 'url'],
|
|
18
|
+
func: async (page, kwargs) => {
|
|
19
|
+
const limit = Math.min(kwargs.limit || 20, 50);
|
|
20
|
+
await page.goto('https://www.youtube.com');
|
|
21
|
+
await page.wait(2);
|
|
22
|
+
const data = await page.evaluate(`
|
|
23
|
+
(async () => {
|
|
24
|
+
const cfg = window.ytcfg?.data_ || {};
|
|
25
|
+
const apiKey = cfg.INNERTUBE_API_KEY;
|
|
26
|
+
const context = cfg.INNERTUBE_CONTEXT;
|
|
27
|
+
if (!apiKey || !context) return {error: 'YouTube config not found'};
|
|
28
|
+
|
|
29
|
+
const resp = await fetch('/youtubei/v1/search?key=' + apiKey + '&prettyPrint=false', {
|
|
30
|
+
method: 'POST', credentials: 'include',
|
|
31
|
+
headers: {'Content-Type': 'application/json'},
|
|
32
|
+
body: JSON.stringify({context, query: '${kwargs.query.replace(/'/g, "\\'")}'})
|
|
33
|
+
});
|
|
34
|
+
if (!resp.ok) return {error: 'HTTP ' + resp.status};
|
|
35
|
+
|
|
36
|
+
const data = await resp.json();
|
|
37
|
+
const contents = data.contents?.twoColumnSearchResultsRenderer?.primaryContents?.sectionListRenderer?.contents || [];
|
|
38
|
+
const videos = [];
|
|
39
|
+
for (const section of contents) {
|
|
40
|
+
for (const item of (section.itemSectionRenderer?.contents || [])) {
|
|
41
|
+
if (item.videoRenderer && videos.length < ${limit}) {
|
|
42
|
+
const v = item.videoRenderer;
|
|
43
|
+
videos.push({
|
|
44
|
+
rank: videos.length + 1,
|
|
45
|
+
title: v.title?.runs?.[0]?.text || '',
|
|
46
|
+
channel: v.ownerText?.runs?.[0]?.text || '',
|
|
47
|
+
views: v.viewCountText?.simpleText || v.shortViewCountText?.simpleText || '',
|
|
48
|
+
duration: v.lengthText?.simpleText || 'LIVE',
|
|
49
|
+
url: 'https://www.youtube.com/watch?v=' + v.videoId
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return videos;
|
|
55
|
+
})()
|
|
56
|
+
`);
|
|
57
|
+
if (!Array.isArray(data)) return [];
|
|
58
|
+
return data;
|
|
59
|
+
},
|
|
60
|
+
});
|
package/src/engine.ts
CHANGED
|
@@ -6,6 +6,7 @@ import * as fs from 'node:fs';
|
|
|
6
6
|
import * as path from 'node:path';
|
|
7
7
|
import yaml from 'js-yaml';
|
|
8
8
|
import { type CliCommand, type Arg, Strategy, registerCommand } from './registry.js';
|
|
9
|
+
import type { IPage } from './types.js';
|
|
9
10
|
import { executePipeline } from './pipeline.js';
|
|
10
11
|
|
|
11
12
|
export function discoverClis(...dirs: string[]): void {
|
|
@@ -72,7 +73,7 @@ function registerYamlCli(filePath: string, defaultSite: string): void {
|
|
|
72
73
|
|
|
73
74
|
export async function executeCommand(
|
|
74
75
|
cmd: CliCommand,
|
|
75
|
-
page:
|
|
76
|
+
page: IPage | null,
|
|
76
77
|
kwargs: Record<string, any>,
|
|
77
78
|
debug: boolean = false,
|
|
78
79
|
): Promise<any> {
|
package/src/explore.ts
CHANGED
|
@@ -477,7 +477,7 @@ export function renderExploreSummary(result: Record<string, any>): string {
|
|
|
477
477
|
return lines.join('\n');
|
|
478
478
|
}
|
|
479
479
|
|
|
480
|
-
async function readPageMetadata(page: any): Promise<{ url: string; title: string }> {
|
|
480
|
+
async function readPageMetadata(page: any /* IPage */): Promise<{ url: string; title: string }> {
|
|
481
481
|
try {
|
|
482
482
|
const result = await page.evaluate(`() => ({ url: window.location.href, title: document.title || '' })`);
|
|
483
483
|
if (result && typeof result === 'object') return { url: String(result.url ?? ''), title: String(result.title ?? '') };
|
package/src/generate.ts
CHANGED
|
@@ -10,7 +10,9 @@
|
|
|
10
10
|
|
|
11
11
|
import { exploreUrl } from './explore.js';
|
|
12
12
|
import { synthesizeFromExplore } from './synthesize.js';
|
|
13
|
-
|
|
13
|
+
|
|
14
|
+
// TODO: implement real CLI registration (copy candidate YAML to user clis dir)
|
|
15
|
+
function registerCandidates(_opts: any): any { return { ok: true, count: 0 }; }
|
|
14
16
|
|
|
15
17
|
const CAPABILITY_ALIASES: Record<string, string[]> = {
|
|
16
18
|
search: ['search', '搜索', '查找', 'query', 'keyword'],
|