@jackwener/opencli 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI-CREATOR.md +594 -0
- package/README.md +124 -39
- package/README.zh-CN.md +151 -0
- package/SKILL.md +178 -102
- package/dist/bilibili.d.ts +6 -5
- package/dist/browser.d.ts +3 -1
- package/dist/browser.js +44 -2
- package/dist/cascade.d.ts +46 -0
- package/dist/cascade.js +180 -0
- package/dist/clis/bbc/news.js +42 -0
- package/dist/clis/bilibili/hot.yaml +38 -0
- package/dist/clis/boss/search.js +47 -0
- package/dist/clis/ctrip/search.d.ts +1 -0
- package/dist/clis/ctrip/search.js +62 -0
- package/dist/clis/hackernews/top.yaml +36 -0
- package/dist/clis/index.d.ts +10 -1
- package/dist/clis/index.js +19 -1
- package/dist/clis/reddit/hot.yaml +46 -0
- package/dist/clis/reuters/search.d.ts +1 -0
- package/dist/clis/reuters/search.js +52 -0
- package/dist/clis/smzdm/search.d.ts +1 -0
- package/dist/clis/smzdm/search.js +66 -0
- package/dist/clis/twitter/trending.yaml +40 -0
- package/dist/clis/v2ex/hot.yaml +25 -0
- package/dist/clis/v2ex/latest.yaml +25 -0
- package/dist/clis/v2ex/topic.yaml +27 -0
- package/dist/clis/weibo/hot.d.ts +1 -0
- package/dist/clis/weibo/hot.js +41 -0
- package/dist/clis/xiaohongshu/feed.yaml +32 -0
- package/dist/clis/xiaohongshu/notifications.yaml +38 -0
- package/dist/clis/xiaohongshu/search.d.ts +5 -0
- package/dist/clis/xiaohongshu/search.js +68 -0
- package/dist/clis/yahoo-finance/quote.d.ts +1 -0
- package/dist/clis/yahoo-finance/quote.js +74 -0
- package/dist/clis/youtube/search.d.ts +1 -0
- package/dist/clis/youtube/search.js +60 -0
- package/dist/clis/zhihu/hot.yaml +42 -0
- package/dist/clis/zhihu/question.d.ts +1 -0
- package/dist/clis/zhihu/question.js +39 -0
- package/dist/clis/zhihu/search.yaml +55 -0
- package/dist/engine.d.ts +2 -1
- package/dist/explore.d.ts +23 -13
- package/dist/explore.js +293 -422
- package/dist/generate.js +2 -1
- package/dist/main.js +21 -2
- package/dist/pipeline/executor.d.ts +9 -0
- package/dist/pipeline/executor.js +88 -0
- package/dist/pipeline/index.d.ts +5 -0
- package/dist/pipeline/index.js +5 -0
- package/dist/pipeline/steps/browser.d.ts +12 -0
- package/dist/pipeline/steps/browser.js +68 -0
- package/dist/pipeline/steps/fetch.d.ts +5 -0
- package/dist/pipeline/steps/fetch.js +50 -0
- package/dist/pipeline/steps/intercept.d.ts +5 -0
- package/dist/pipeline/steps/intercept.js +75 -0
- package/dist/pipeline/steps/tap.d.ts +12 -0
- package/dist/pipeline/steps/tap.js +130 -0
- package/dist/pipeline/steps/transform.d.ts +8 -0
- package/dist/pipeline/steps/transform.js +53 -0
- package/dist/pipeline/template.d.ts +16 -0
- package/dist/pipeline/template.js +115 -0
- package/dist/pipeline/template.test.d.ts +4 -0
- package/dist/pipeline/template.test.js +102 -0
- package/dist/pipeline/transform.test.d.ts +4 -0
- package/dist/pipeline/transform.test.js +90 -0
- package/dist/pipeline.d.ts +5 -7
- package/dist/pipeline.js +5 -313
- package/dist/registry.d.ts +3 -2
- package/dist/runtime.d.ts +2 -1
- package/dist/synthesize.d.ts +11 -8
- package/dist/synthesize.js +142 -118
- package/dist/types.d.ts +27 -0
- package/dist/types.js +7 -0
- package/package.json +9 -4
- package/src/bilibili.ts +9 -7
- package/src/browser.ts +41 -3
- package/src/cascade.ts +218 -0
- package/src/clis/bbc/news.ts +42 -0
- package/src/clis/boss/search.ts +47 -0
- package/src/clis/ctrip/search.ts +62 -0
- package/src/clis/index.ts +28 -1
- package/src/clis/reddit/hot.yaml +46 -0
- package/src/clis/reuters/search.ts +52 -0
- package/src/clis/smzdm/search.ts +66 -0
- package/src/clis/v2ex/hot.yaml +5 -9
- package/src/clis/v2ex/latest.yaml +5 -8
- package/src/clis/v2ex/topic.yaml +27 -0
- package/src/clis/weibo/hot.ts +41 -0
- package/src/clis/xiaohongshu/feed.yaml +32 -0
- package/src/clis/xiaohongshu/notifications.yaml +38 -0
- package/src/clis/xiaohongshu/search.ts +71 -0
- package/src/clis/yahoo-finance/quote.ts +74 -0
- package/src/clis/youtube/search.ts +60 -0
- package/src/clis/zhihu/hot.yaml +22 -8
- package/src/clis/zhihu/question.ts +45 -0
- package/src/clis/zhihu/search.yaml +55 -0
- package/src/engine.ts +2 -1
- package/src/explore.ts +303 -465
- package/src/generate.ts +3 -1
- package/src/main.ts +18 -2
- package/src/pipeline/executor.ts +98 -0
- package/src/pipeline/index.ts +6 -0
- package/src/pipeline/steps/browser.ts +67 -0
- package/src/pipeline/steps/fetch.ts +60 -0
- package/src/pipeline/steps/intercept.ts +78 -0
- package/src/pipeline/steps/tap.ts +137 -0
- package/src/pipeline/steps/transform.ts +50 -0
- package/src/pipeline/template.test.ts +107 -0
- package/src/pipeline/template.ts +101 -0
- package/src/pipeline/transform.test.ts +107 -0
- package/src/pipeline.ts +5 -292
- package/src/registry.ts +4 -2
- package/src/runtime.ts +3 -1
- package/src/synthesize.ts +142 -137
- package/src/types.ts +23 -0
- package/vitest.config.ts +7 -0
- package/dist/clis/github/search.js +0 -20
- package/dist/clis/zhihu/search.js +0 -58
- package/dist/promote.d.ts +0 -1
- package/dist/promote.js +0 -3
- package/dist/register.d.ts +0 -2
- package/dist/register.js +0 -2
- package/dist/scaffold.d.ts +0 -2
- package/dist/scaffold.js +0 -2
- package/dist/smoke.d.ts +0 -2
- package/dist/smoke.js +0 -2
- package/src/clis/github/search.ts +0 -21
- package/src/clis/github/trending.yaml +0 -58
- package/src/clis/zhihu/search.ts +0 -65
- package/src/promote.ts +0 -3
- package/src/register.ts +0 -2
- package/src/scaffold.ts +0 -2
- package/src/smoke.ts +0 -2
- /package/dist/clis/{github/search.d.ts → bbc/news.d.ts} +0 -0
- /package/dist/clis/{zhihu → boss}/search.d.ts +0 -0
package/dist/runtime.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Runtime utilities: timeouts and browser session management.
|
|
3
3
|
*/
|
|
4
|
+
import type { IPage } from './types.js';
|
|
4
5
|
export declare const DEFAULT_BROWSER_CONNECT_TIMEOUT: number;
|
|
5
6
|
export declare const DEFAULT_BROWSER_COMMAND_TIMEOUT: number;
|
|
6
7
|
export declare const DEFAULT_BROWSER_EXPLORE_TIMEOUT: number;
|
|
@@ -9,4 +10,4 @@ export declare function runWithTimeout<T>(promise: Promise<T>, opts: {
|
|
|
9
10
|
timeout: number;
|
|
10
11
|
label?: string;
|
|
11
12
|
}): Promise<T>;
|
|
12
|
-
export declare function browserSession<T>(BrowserFactory: new () => any, fn: (page:
|
|
13
|
+
export declare function browserSession<T>(BrowserFactory: new () => any, fn: (page: IPage) => Promise<T>): Promise<T>;
|
package/dist/synthesize.d.ts
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Synthesize
|
|
3
|
-
*
|
|
4
|
-
* Takes the structured capabilities from Deep Explore and generates
|
|
5
|
-
* YAML pipeline files that can be directly registered as CLI commands.
|
|
6
|
-
*
|
|
7
|
-
* This is the bridge between discovery (explore) and usability (CLI).
|
|
2
|
+
* Synthesize candidate CLIs from explore artifacts.
|
|
3
|
+
* Generates evaluate-based YAML pipelines (matching hand-written adapter patterns).
|
|
8
4
|
*/
|
|
9
|
-
export declare function synthesizeFromExplore(target: string, opts?:
|
|
10
|
-
|
|
5
|
+
export declare function synthesizeFromExplore(target: string, opts?: {
|
|
6
|
+
outDir?: string;
|
|
7
|
+
top?: number;
|
|
8
|
+
}): Record<string, any>;
|
|
9
|
+
export declare function renderSynthesizeSummary(result: Record<string, any>): string;
|
|
10
|
+
export declare function resolveExploreDir(target: string): string;
|
|
11
|
+
export declare function loadExploreBundle(exploreDir: string): Record<string, any>;
|
|
12
|
+
/** Backward-compatible export for scaffold.ts */
|
|
13
|
+
export declare function buildCandidate(site: string, targetUrl: string, cap: any, endpoint: any): any;
|
package/dist/synthesize.js
CHANGED
|
@@ -1,147 +1,181 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Synthesize
|
|
3
|
-
*
|
|
4
|
-
* Takes the structured capabilities from Deep Explore and generates
|
|
5
|
-
* YAML pipeline files that can be directly registered as CLI commands.
|
|
6
|
-
*
|
|
7
|
-
* This is the bridge between discovery (explore) and usability (CLI).
|
|
2
|
+
* Synthesize candidate CLIs from explore artifacts.
|
|
3
|
+
* Generates evaluate-based YAML pipelines (matching hand-written adapter patterns).
|
|
8
4
|
*/
|
|
9
5
|
import * as fs from 'node:fs';
|
|
10
6
|
import * as path from 'node:path';
|
|
11
7
|
import yaml from 'js-yaml';
|
|
8
|
+
/** Volatile params to strip from generated URLs */
|
|
9
|
+
const VOLATILE_PARAMS = new Set(['w_rid', 'wts', 'callback', '_', 'timestamp', 't', 'nonce', 'sign']);
|
|
10
|
+
const SEARCH_PARAM_NAMES = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'w', 'search_query']);
|
|
11
|
+
const LIMIT_PARAM_NAMES = new Set(['ps', 'page_size', 'limit', 'count', 'per_page', 'size', 'num']);
|
|
12
|
+
const PAGE_PARAM_NAMES = new Set(['pn', 'page', 'page_num', 'offset', 'cursor']);
|
|
12
13
|
export function synthesizeFromExplore(target, opts = {}) {
|
|
13
|
-
const exploreDir =
|
|
14
|
-
|
|
15
|
-
throw new Error(`Explore dir not found: ${target}`);
|
|
16
|
-
const manifest = JSON.parse(fs.readFileSync(path.join(exploreDir, 'manifest.json'), 'utf-8'));
|
|
17
|
-
const capabilities = JSON.parse(fs.readFileSync(path.join(exploreDir, 'capabilities.json'), 'utf-8'));
|
|
18
|
-
const endpoints = JSON.parse(fs.readFileSync(path.join(exploreDir, 'endpoints.json'), 'utf-8'));
|
|
19
|
-
const auth = JSON.parse(fs.readFileSync(path.join(exploreDir, 'auth.json'), 'utf-8'));
|
|
14
|
+
const exploreDir = resolveExploreDir(target);
|
|
15
|
+
const bundle = loadExploreBundle(exploreDir);
|
|
20
16
|
const targetDir = opts.outDir ?? path.join(exploreDir, 'candidates');
|
|
21
17
|
fs.mkdirSync(targetDir, { recursive: true });
|
|
22
|
-
const site = manifest.site;
|
|
23
|
-
const
|
|
24
|
-
const candidates = [];
|
|
25
|
-
// Sort capabilities by confidence
|
|
26
|
-
const sortedCaps = [...capabilities]
|
|
18
|
+
const site = bundle.manifest.site;
|
|
19
|
+
const capabilities = (bundle.capabilities ?? [])
|
|
27
20
|
.sort((a, b) => (b.confidence ?? 0) - (a.confidence ?? 0))
|
|
28
|
-
.slice(0,
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
const endpoint =
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
const
|
|
35
|
-
const filePath = path.join(targetDir,
|
|
21
|
+
.slice(0, opts.top ?? 3);
|
|
22
|
+
const candidates = [];
|
|
23
|
+
for (const cap of capabilities) {
|
|
24
|
+
const endpoint = chooseEndpoint(cap, bundle.endpoints);
|
|
25
|
+
if (!endpoint)
|
|
26
|
+
continue;
|
|
27
|
+
const candidate = buildCandidateYaml(site, bundle.manifest, cap, endpoint);
|
|
28
|
+
const filePath = path.join(targetDir, `${candidate.name}.yaml`);
|
|
36
29
|
fs.writeFileSync(filePath, yaml.dump(candidate.yaml, { sortKeys: false, lineWidth: 120 }));
|
|
37
|
-
candidates.push({
|
|
38
|
-
name: cap.name,
|
|
39
|
-
path: filePath,
|
|
40
|
-
strategy: cap.strategy,
|
|
41
|
-
endpoint: cap.endpoint,
|
|
42
|
-
confidence: cap.confidence,
|
|
43
|
-
columns: candidate.yaml.columns,
|
|
44
|
-
});
|
|
30
|
+
candidates.push({ name: candidate.name, path: filePath, strategy: cap.strategy, confidence: cap.confidence });
|
|
45
31
|
}
|
|
46
|
-
const index = {
|
|
47
|
-
site,
|
|
48
|
-
target_url: manifest.target_url,
|
|
49
|
-
generated_from: exploreDir,
|
|
50
|
-
candidate_count: candidates.length,
|
|
51
|
-
candidates,
|
|
52
|
-
};
|
|
32
|
+
const index = { site, target_url: bundle.manifest.target_url, generated_from: exploreDir, candidate_count: candidates.length, candidates };
|
|
53
33
|
fs.writeFileSync(path.join(targetDir, 'candidates.json'), JSON.stringify(index, null, 2));
|
|
34
|
+
return { site, explore_dir: exploreDir, out_dir: targetDir, candidate_count: candidates.length, candidates };
|
|
35
|
+
}
|
|
36
|
+
export function renderSynthesizeSummary(result) {
|
|
37
|
+
const lines = ['opencli synthesize: OK', `Site: ${result.site}`, `Source: ${result.explore_dir}`, `Candidates: ${result.candidate_count}`];
|
|
38
|
+
for (const c of result.candidates ?? [])
|
|
39
|
+
lines.push(` • ${c.name} (${c.strategy}, ${((c.confidence ?? 0) * 100).toFixed(0)}% confidence) → ${c.path}`);
|
|
40
|
+
return lines.join('\n');
|
|
41
|
+
}
|
|
42
|
+
export function resolveExploreDir(target) {
|
|
43
|
+
if (fs.existsSync(target))
|
|
44
|
+
return target;
|
|
45
|
+
const candidate = path.join('.opencli', 'explore', target);
|
|
46
|
+
if (fs.existsSync(candidate))
|
|
47
|
+
return candidate;
|
|
48
|
+
throw new Error(`Explore directory not found: ${target}`);
|
|
49
|
+
}
|
|
50
|
+
export function loadExploreBundle(exploreDir) {
|
|
54
51
|
return {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
candidates,
|
|
52
|
+
manifest: JSON.parse(fs.readFileSync(path.join(exploreDir, 'manifest.json'), 'utf-8')),
|
|
53
|
+
endpoints: JSON.parse(fs.readFileSync(path.join(exploreDir, 'endpoints.json'), 'utf-8')),
|
|
54
|
+
capabilities: JSON.parse(fs.readFileSync(path.join(exploreDir, 'capabilities.json'), 'utf-8')),
|
|
55
|
+
auth: JSON.parse(fs.readFileSync(path.join(exploreDir, 'auth.json'), 'utf-8')),
|
|
60
56
|
};
|
|
61
57
|
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
58
|
+
function chooseEndpoint(cap, endpoints) {
|
|
59
|
+
if (!endpoints.length)
|
|
60
|
+
return null;
|
|
61
|
+
// Match by endpoint pattern from capability
|
|
62
|
+
if (cap.endpoint) {
|
|
63
|
+
const match = endpoints.find((e) => e.pattern === cap.endpoint || e.url?.includes(cap.endpoint));
|
|
64
|
+
if (match)
|
|
65
|
+
return match;
|
|
66
|
+
}
|
|
67
|
+
return endpoints.sort((a, b) => (b.score ?? 0) - (a.score ?? 0))[0];
|
|
68
|
+
}
|
|
69
|
+
// ── URL templating ─────────────────────────────────────────────────────────
|
|
70
|
+
function buildTemplatedUrl(rawUrl, cap, _endpoint) {
|
|
74
71
|
try {
|
|
75
72
|
const u = new URL(rawUrl);
|
|
76
73
|
const base = `${u.protocol}//${u.host}${u.pathname}`;
|
|
77
74
|
const params = [];
|
|
78
75
|
const hasKeyword = cap.recommendedArgs?.some((a) => a.name === 'keyword');
|
|
79
76
|
u.searchParams.forEach((v, k) => {
|
|
80
|
-
// Skip volatile params
|
|
81
77
|
if (VOLATILE_PARAMS.has(k))
|
|
82
78
|
return;
|
|
83
|
-
|
|
84
|
-
if (hasKeyword && SEARCH_PARAM_NAMES.has(k)) {
|
|
79
|
+
if (hasKeyword && SEARCH_PARAM_NAMES.has(k))
|
|
85
80
|
params.push([k, '${{ args.keyword }}']);
|
|
86
|
-
|
|
87
|
-
else if (LIMIT_PARAM_NAMES.has(k)) {
|
|
81
|
+
else if (LIMIT_PARAM_NAMES.has(k))
|
|
88
82
|
params.push([k, '${{ args.limit | default(20) }}']);
|
|
89
|
-
|
|
90
|
-
else if (PAGE_PARAM_NAMES.has(k)) {
|
|
83
|
+
else if (PAGE_PARAM_NAMES.has(k))
|
|
91
84
|
params.push([k, '${{ args.page | default(1) }}']);
|
|
92
|
-
|
|
93
|
-
else {
|
|
85
|
+
else
|
|
94
86
|
params.push([k, v]);
|
|
95
|
-
}
|
|
96
87
|
});
|
|
97
|
-
|
|
98
|
-
return base;
|
|
99
|
-
return base + '?' + params.map(([k, v]) => `${k}=${v}`).join('&');
|
|
88
|
+
return params.length ? base + '?' + params.map(([k, v]) => `${k}=${v}`).join('&') : base;
|
|
100
89
|
}
|
|
101
90
|
catch {
|
|
102
91
|
return rawUrl;
|
|
103
92
|
}
|
|
104
93
|
}
|
|
105
94
|
/**
|
|
106
|
-
* Build
|
|
95
|
+
* Build inline evaluate script for browser-based fetch+parse.
|
|
96
|
+
* Follows patterns from bilibili/hot.yaml and twitter/trending.yaml.
|
|
107
97
|
*/
|
|
98
|
+
function buildEvaluateScript(url, itemPath, endpoint) {
|
|
99
|
+
const pathChain = itemPath.split('.').map((p) => `?.${p}`).join('');
|
|
100
|
+
const detectedFields = endpoint?.detectedFields ?? {};
|
|
101
|
+
const hasFields = Object.keys(detectedFields).length > 0;
|
|
102
|
+
let mapCode = '';
|
|
103
|
+
if (hasFields) {
|
|
104
|
+
const mappings = Object.entries(detectedFields)
|
|
105
|
+
.map(([role, field]) => ` ${role}: item${String(field).split('.').map(p => `?.${p}`).join('')}`)
|
|
106
|
+
.join(',\n');
|
|
107
|
+
mapCode = `.map((item) => ({\n${mappings}\n }))`;
|
|
108
|
+
}
|
|
109
|
+
return [
|
|
110
|
+
'(async () => {',
|
|
111
|
+
` const res = await fetch('${url}', {`,
|
|
112
|
+
` credentials: 'include'`,
|
|
113
|
+
' });',
|
|
114
|
+
' const data = await res.json();',
|
|
115
|
+
` return (data${pathChain} || [])${mapCode};`,
|
|
116
|
+
'})()\n',
|
|
117
|
+
].join('\n');
|
|
118
|
+
}
|
|
119
|
+
// ── YAML pipeline generation ───────────────────────────────────────────────
|
|
108
120
|
function buildCandidateYaml(site, manifest, cap, endpoint) {
|
|
109
121
|
const needsBrowser = cap.strategy !== 'public';
|
|
110
122
|
const pipeline = [];
|
|
111
|
-
|
|
112
|
-
|
|
123
|
+
const templatedUrl = buildTemplatedUrl(endpoint?.url ?? manifest.target_url, cap, endpoint);
|
|
124
|
+
let domain = '';
|
|
125
|
+
try {
|
|
126
|
+
domain = new URL(manifest.target_url).hostname;
|
|
127
|
+
}
|
|
128
|
+
catch { }
|
|
129
|
+
if (cap.strategy === 'store-action' && cap.storeHint) {
|
|
130
|
+
// Store Action: navigate + wait + tap (declarative, clean)
|
|
113
131
|
pipeline.push({ navigate: manifest.target_url });
|
|
132
|
+
pipeline.push({ wait: 3 });
|
|
133
|
+
const tapStep = {
|
|
134
|
+
store: cap.storeHint.store,
|
|
135
|
+
action: cap.storeHint.action,
|
|
136
|
+
timeout: 8,
|
|
137
|
+
};
|
|
138
|
+
// Infer capture pattern from endpoint URL
|
|
139
|
+
if (endpoint?.url) {
|
|
140
|
+
try {
|
|
141
|
+
const epUrl = new URL(endpoint.url);
|
|
142
|
+
const pathParts = epUrl.pathname.split('/').filter((p) => p);
|
|
143
|
+
// Use last meaningful path segment as capture pattern
|
|
144
|
+
const capturePart = pathParts.filter((p) => !p.match(/^v\d+$/)).pop();
|
|
145
|
+
if (capturePart)
|
|
146
|
+
tapStep.capture = capturePart;
|
|
147
|
+
}
|
|
148
|
+
catch { }
|
|
149
|
+
}
|
|
150
|
+
if (cap.itemPath)
|
|
151
|
+
tapStep.select = cap.itemPath;
|
|
152
|
+
pipeline.push({ tap: tapStep });
|
|
153
|
+
}
|
|
154
|
+
else if (needsBrowser) {
|
|
155
|
+
// Browser-based: navigate + evaluate (like bilibili/hot.yaml, twitter/trending.yaml)
|
|
156
|
+
pipeline.push({ navigate: manifest.target_url });
|
|
157
|
+
const itemPath = cap.itemPath ?? 'data.data.list';
|
|
158
|
+
pipeline.push({ evaluate: buildEvaluateScript(templatedUrl, itemPath, endpoint) });
|
|
114
159
|
}
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
if (cap.itemPath) {
|
|
121
|
-
pipeline.push({ select: cap.itemPath });
|
|
160
|
+
else {
|
|
161
|
+
// Public API: direct fetch (like hackernews/top.yaml)
|
|
162
|
+
pipeline.push({ fetch: { url: templatedUrl } });
|
|
163
|
+
if (cap.itemPath)
|
|
164
|
+
pipeline.push({ select: cap.itemPath });
|
|
122
165
|
}
|
|
123
|
-
//
|
|
166
|
+
// Map fields
|
|
124
167
|
const mapStep = {};
|
|
125
168
|
const columns = cap.recommendedColumns ?? ['title', 'url'];
|
|
126
|
-
|
|
127
|
-
if (!cap.recommendedArgs?.some((a) => a.name === 'keyword')) {
|
|
169
|
+
if (!cap.recommendedArgs?.some((a) => a.name === 'keyword'))
|
|
128
170
|
mapStep['rank'] = '${{ index + 1 }}';
|
|
129
|
-
}
|
|
130
|
-
// Build field mappings from the endpoint's detected fields
|
|
131
171
|
const detectedFields = endpoint?.detectedFields ?? {};
|
|
132
172
|
for (const col of columns) {
|
|
133
173
|
const fieldPath = detectedFields[col];
|
|
134
|
-
|
|
135
|
-
mapStep[col] = `\${{ item.${fieldPath} }}`;
|
|
136
|
-
}
|
|
137
|
-
else {
|
|
138
|
-
mapStep[col] = `\${{ item.${col} }}`;
|
|
139
|
-
}
|
|
174
|
+
mapStep[col] = fieldPath ? `\${{ item.${fieldPath} }}` : `\${{ item.${col} }}`;
|
|
140
175
|
}
|
|
141
176
|
pipeline.push({ map: mapStep });
|
|
142
|
-
// Step 5: Limit
|
|
143
177
|
pipeline.push({ limit: '${{ args.limit | default(20) }}' });
|
|
144
|
-
//
|
|
178
|
+
// Args
|
|
145
179
|
const argsDef = {};
|
|
146
180
|
for (const arg of cap.recommendedArgs ?? []) {
|
|
147
181
|
const def = { type: arg.type ?? 'str' };
|
|
@@ -157,35 +191,25 @@ function buildCandidateYaml(site, manifest, cap, endpoint) {
|
|
|
157
191
|
def.description = 'Page number';
|
|
158
192
|
argsDef[arg.name] = def;
|
|
159
193
|
}
|
|
160
|
-
|
|
161
|
-
if (!argsDef['limit']) {
|
|
194
|
+
if (!argsDef['limit'])
|
|
162
195
|
argsDef['limit'] = { type: 'int', default: 20, description: 'Number of items to return' };
|
|
163
|
-
}
|
|
164
|
-
const allColumns = Object.keys(mapStep);
|
|
165
196
|
return {
|
|
166
197
|
name: cap.name,
|
|
167
198
|
yaml: {
|
|
168
|
-
site,
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
domain: manifest.final_url ? new URL(manifest.final_url).hostname : undefined,
|
|
172
|
-
strategy: cap.strategy,
|
|
173
|
-
browser: needsBrowser,
|
|
174
|
-
args: argsDef,
|
|
175
|
-
pipeline,
|
|
176
|
-
columns: allColumns,
|
|
199
|
+
site, name: cap.name, description: `${cap.description || site + ' ' + cap.name} (auto-generated)`,
|
|
200
|
+
domain, strategy: cap.strategy, browser: needsBrowser,
|
|
201
|
+
args: argsDef, pipeline, columns: Object.keys(mapStep),
|
|
177
202
|
},
|
|
178
203
|
};
|
|
179
204
|
}
|
|
180
|
-
export
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
return lines.join('\n');
|
|
205
|
+
/** Backward-compatible export for scaffold.ts */
|
|
206
|
+
export function buildCandidate(site, targetUrl, cap, endpoint) {
|
|
207
|
+
// Map old-style field names to new ones
|
|
208
|
+
const normalizedCap = {
|
|
209
|
+
...cap,
|
|
210
|
+
recommendedArgs: cap.recommendedArgs ?? cap.recommended_args,
|
|
211
|
+
recommendedColumns: cap.recommendedColumns ?? cap.recommended_columns,
|
|
212
|
+
};
|
|
213
|
+
const manifest = { target_url: targetUrl, final_url: targetUrl };
|
|
214
|
+
return buildCandidateYaml(site, manifest, normalizedCap, endpoint);
|
|
191
215
|
}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Page interface: type-safe abstraction over Playwright MCP browser page.
|
|
3
|
+
*
|
|
4
|
+
* All pipeline steps and CLI adapters should use this interface
|
|
5
|
+
* instead of `any` for browser interactions.
|
|
6
|
+
*/
|
|
7
|
+
export interface IPage {
|
|
8
|
+
goto(url: string): Promise<void>;
|
|
9
|
+
evaluate(js: string): Promise<any>;
|
|
10
|
+
snapshot(opts?: {
|
|
11
|
+
interactive?: boolean;
|
|
12
|
+
compact?: boolean;
|
|
13
|
+
maxDepth?: number;
|
|
14
|
+
raw?: boolean;
|
|
15
|
+
}): Promise<any>;
|
|
16
|
+
click(ref: string): Promise<void>;
|
|
17
|
+
typeText(ref: string, text: string): Promise<void>;
|
|
18
|
+
pressKey(key: string): Promise<void>;
|
|
19
|
+
wait(seconds: number): Promise<void>;
|
|
20
|
+
tabs(): Promise<any>;
|
|
21
|
+
closeTab(index?: number): Promise<void>;
|
|
22
|
+
newTab(): Promise<void>;
|
|
23
|
+
selectTab(index: number): Promise<void>;
|
|
24
|
+
networkRequests(includeStatic?: boolean): Promise<any>;
|
|
25
|
+
consoleMessages(level?: string): Promise<any>;
|
|
26
|
+
scroll(direction?: string, amount?: number): Promise<void>;
|
|
27
|
+
}
|
package/dist/types.js
ADDED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jackwener/opencli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public"
|
|
6
6
|
},
|
|
@@ -12,11 +12,15 @@
|
|
|
12
12
|
},
|
|
13
13
|
"scripts": {
|
|
14
14
|
"dev": "tsx src/main.ts",
|
|
15
|
-
"build": "tsc",
|
|
15
|
+
"build": "tsc && npm run clean-yaml && npm run copy-yaml",
|
|
16
|
+
"clean-yaml": "find dist/clis -name '*.yaml' -o -name '*.yml' 2>/dev/null | xargs rm -f",
|
|
17
|
+
"copy-yaml": "find src/clis -name '*.yaml' -o -name '*.yml' | while read f; do d=\"dist/${f#src/}\"; mkdir -p \"$(dirname \"$d\")\"; cp \"$f\" \"$d\"; done",
|
|
16
18
|
"start": "node dist/main.js",
|
|
17
19
|
"typecheck": "tsc --noEmit",
|
|
18
20
|
"lint": "tsc --noEmit",
|
|
19
|
-
"prepublishOnly": "npm run build"
|
|
21
|
+
"prepublishOnly": "npm run build",
|
|
22
|
+
"test": "vitest run",
|
|
23
|
+
"test:watch": "vitest"
|
|
20
24
|
},
|
|
21
25
|
"keywords": [
|
|
22
26
|
"cli",
|
|
@@ -42,6 +46,7 @@
|
|
|
42
46
|
"@types/js-yaml": "^4.0.9",
|
|
43
47
|
"@types/node": "^22.13.10",
|
|
44
48
|
"tsx": "^4.19.3",
|
|
45
|
-
"typescript": "^5.8.2"
|
|
49
|
+
"typescript": "^5.8.2",
|
|
50
|
+
"vitest": "^4.1.0"
|
|
46
51
|
}
|
|
47
52
|
}
|
package/src/bilibili.ts
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
* Bilibili shared helpers: WBI signing, authenticated fetch, nav data, UID resolution.
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
+
import type { IPage } from './types.js';
|
|
6
|
+
|
|
5
7
|
const MIXIN_KEY_ENC_TAB = [
|
|
6
8
|
46,47,18,2,53,8,23,32,15,50,10,31,58,3,45,35,27,43,5,49,
|
|
7
9
|
33,9,42,19,29,28,14,39,12,38,41,13,37,48,7,16,24,55,40,
|
|
@@ -17,7 +19,7 @@ export function payloadData(payload: any): any {
|
|
|
17
19
|
return payload?.data ?? payload;
|
|
18
20
|
}
|
|
19
21
|
|
|
20
|
-
async function getNavData(page:
|
|
22
|
+
async function getNavData(page: IPage): Promise<any> {
|
|
21
23
|
return page.evaluate(`
|
|
22
24
|
async () => {
|
|
23
25
|
const res = await fetch('https://api.bilibili.com/x/web-interface/nav', { credentials: 'include' });
|
|
@@ -26,7 +28,7 @@ async function getNavData(page: any): Promise<any> {
|
|
|
26
28
|
`);
|
|
27
29
|
}
|
|
28
30
|
|
|
29
|
-
async function getWbiKeys(page:
|
|
31
|
+
async function getWbiKeys(page: IPage): Promise<{ imgKey: string; subKey: string }> {
|
|
30
32
|
const nav = await getNavData(page);
|
|
31
33
|
const wbiImg = nav?.data?.wbi_img ?? {};
|
|
32
34
|
const imgUrl = wbiImg.img_url ?? '';
|
|
@@ -47,7 +49,7 @@ async function md5(text: string): Promise<string> {
|
|
|
47
49
|
}
|
|
48
50
|
|
|
49
51
|
export async function wbiSign(
|
|
50
|
-
page:
|
|
52
|
+
page: IPage,
|
|
51
53
|
params: Record<string, any>,
|
|
52
54
|
): Promise<Record<string, string>> {
|
|
53
55
|
const { imgKey, subKey } = await getWbiKeys(page);
|
|
@@ -65,7 +67,7 @@ export async function wbiSign(
|
|
|
65
67
|
}
|
|
66
68
|
|
|
67
69
|
export async function apiGet(
|
|
68
|
-
page:
|
|
70
|
+
page: IPage,
|
|
69
71
|
path: string,
|
|
70
72
|
opts: { params?: Record<string, any>; signed?: boolean } = {},
|
|
71
73
|
): Promise<any> {
|
|
@@ -81,7 +83,7 @@ export async function apiGet(
|
|
|
81
83
|
return fetchJson(page, url);
|
|
82
84
|
}
|
|
83
85
|
|
|
84
|
-
export async function fetchJson(page:
|
|
86
|
+
export async function fetchJson(page: IPage, url: string): Promise<any> {
|
|
85
87
|
const escapedUrl = url.replace(/"/g, '\\"');
|
|
86
88
|
return page.evaluate(`
|
|
87
89
|
async () => {
|
|
@@ -91,14 +93,14 @@ export async function fetchJson(page: any, url: string): Promise<any> {
|
|
|
91
93
|
`);
|
|
92
94
|
}
|
|
93
95
|
|
|
94
|
-
export async function getSelfUid(page:
|
|
96
|
+
export async function getSelfUid(page: IPage): Promise<string> {
|
|
95
97
|
const nav = await getNavData(page);
|
|
96
98
|
const mid = nav?.data?.mid;
|
|
97
99
|
if (!mid) throw new Error('Not logged in to Bilibili');
|
|
98
100
|
return String(mid);
|
|
99
101
|
}
|
|
100
102
|
|
|
101
|
-
export async function resolveUid(page:
|
|
103
|
+
export async function resolveUid(page: IPage, input: string): Promise<string> {
|
|
102
104
|
if (/^\d+$/.test(input)) return input;
|
|
103
105
|
// Search for user by name
|
|
104
106
|
const payload = await apiGet(page, '/x/web-interface/wbi/search/type', {
|
package/src/browser.ts
CHANGED
|
@@ -10,6 +10,10 @@ import * as os from 'node:os';
|
|
|
10
10
|
import * as path from 'node:path';
|
|
11
11
|
import { formatSnapshot } from './snapshotFormatter.js';
|
|
12
12
|
|
|
13
|
+
// Read version from package.json (single source of truth)
|
|
14
|
+
const __browser_dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
15
|
+
const PKG_VERSION = (() => { try { return JSON.parse(fs.readFileSync(path.resolve(__browser_dirname, '..', 'package.json'), 'utf-8')).version; } catch { return '0.0.0'; } })();
|
|
16
|
+
|
|
13
17
|
const EXTENSION_LOCK_TIMEOUT = parseInt(process.env.OPENCLI_EXTENSION_LOCK_TIMEOUT ?? '120', 10);
|
|
14
18
|
const EXTENSION_LOCK_POLL = parseInt(process.env.OPENCLI_EXTENSION_LOCK_POLL_INTERVAL ?? '1', 10);
|
|
15
19
|
const CONNECT_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_CONNECT_TIMEOUT ?? '30', 10);
|
|
@@ -21,10 +25,12 @@ function jsonRpcRequest(method: string, params: Record<string, any> = {}): strin
|
|
|
21
25
|
return JSON.stringify({ jsonrpc: '2.0', id: _nextId++, method, params }) + '\n';
|
|
22
26
|
}
|
|
23
27
|
|
|
28
|
+
import type { IPage } from './types.js';
|
|
29
|
+
|
|
24
30
|
/**
|
|
25
31
|
* Page abstraction wrapping JSON-RPC calls to Playwright MCP.
|
|
26
32
|
*/
|
|
27
|
-
export class Page {
|
|
33
|
+
export class Page implements IPage {
|
|
28
34
|
constructor(private _send: (msg: string) => void, private _recv: () => Promise<any>) {}
|
|
29
35
|
|
|
30
36
|
async call(method: string, params: Record<string, any> = {}): Promise<any> {
|
|
@@ -36,7 +42,18 @@ export class Page {
|
|
|
36
42
|
if (result?.content) {
|
|
37
43
|
const textParts = result.content.filter((c: any) => c.type === 'text');
|
|
38
44
|
if (textParts.length === 1) {
|
|
39
|
-
|
|
45
|
+
let text = textParts[0].text;
|
|
46
|
+
// MCP browser_evaluate returns: "[JSON]\n### Ran Playwright code\n```js\n...\n```"
|
|
47
|
+
// Strip the "### Ran Playwright code" suffix to get clean JSON
|
|
48
|
+
const codeMarker = text.indexOf('### Ran Playwright code');
|
|
49
|
+
if (codeMarker !== -1) {
|
|
50
|
+
text = text.slice(0, codeMarker).trim();
|
|
51
|
+
}
|
|
52
|
+
// Also handle "### Result\n[JSON]" format (some MCP versions)
|
|
53
|
+
const resultMarker = text.indexOf('### Result\n');
|
|
54
|
+
if (resultMarker !== -1) {
|
|
55
|
+
text = text.slice(resultMarker + '### Result\n'.length).trim();
|
|
56
|
+
}
|
|
40
57
|
try { return JSON.parse(text); } catch { return text; }
|
|
41
58
|
}
|
|
42
59
|
}
|
|
@@ -115,6 +132,8 @@ export class PlaywrightMCP {
|
|
|
115
132
|
private _lockAcquired = false;
|
|
116
133
|
private _initialTabCount = 0;
|
|
117
134
|
|
|
135
|
+
private _page: Page | null = null;
|
|
136
|
+
|
|
118
137
|
async connect(opts: { timeout?: number } = {}): Promise<Page> {
|
|
119
138
|
await this._acquireLock();
|
|
120
139
|
const timeout = opts.timeout ?? CONNECT_TIMEOUT;
|
|
@@ -137,6 +156,7 @@ export class PlaywrightMCP {
|
|
|
137
156
|
(msg) => { if (this._proc?.stdin?.writable) this._proc.stdin.write(msg); },
|
|
138
157
|
() => new Promise<any>((res) => { this._waiters.push(res); }),
|
|
139
158
|
);
|
|
159
|
+
this._page = page;
|
|
140
160
|
|
|
141
161
|
this._proc.stdout?.on('data', (chunk: Buffer) => {
|
|
142
162
|
this._buffer += chunk.toString();
|
|
@@ -159,7 +179,7 @@ export class PlaywrightMCP {
|
|
|
159
179
|
const initMsg = jsonRpcRequest('initialize', {
|
|
160
180
|
protocolVersion: '2024-11-05',
|
|
161
181
|
capabilities: {},
|
|
162
|
-
clientInfo: { name: 'opencli', version:
|
|
182
|
+
clientInfo: { name: 'opencli', version: PKG_VERSION },
|
|
163
183
|
});
|
|
164
184
|
this._proc.stdin?.write(initMsg);
|
|
165
185
|
|
|
@@ -185,11 +205,29 @@ export class PlaywrightMCP {
|
|
|
185
205
|
|
|
186
206
|
async close(): Promise<void> {
|
|
187
207
|
try {
|
|
208
|
+
// Close tabs opened during this session (site tabs + extension tabs)
|
|
209
|
+
if (this._page && this._proc && !this._proc.killed) {
|
|
210
|
+
try {
|
|
211
|
+
const tabs = await this._page.tabs();
|
|
212
|
+
const tabStr = typeof tabs === 'string' ? tabs : JSON.stringify(tabs);
|
|
213
|
+
const allTabs = tabStr.match(/Tab (\d+)/g) || [];
|
|
214
|
+
const currentTabCount = allTabs.length;
|
|
215
|
+
|
|
216
|
+
// Close tabs in reverse order to avoid index shifting issues
|
|
217
|
+
// Keep the original tabs that existed before the command started
|
|
218
|
+
if (currentTabCount > this._initialTabCount && this._initialTabCount > 0) {
|
|
219
|
+
for (let i = currentTabCount - 1; i >= this._initialTabCount; i--) {
|
|
220
|
+
try { await this._page.closeTab(i); } catch {}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
} catch {}
|
|
224
|
+
}
|
|
188
225
|
if (this._proc && !this._proc.killed) {
|
|
189
226
|
this._proc.kill('SIGTERM');
|
|
190
227
|
await new Promise<void>((res) => { this._proc?.on('exit', () => res()); setTimeout(res, 3000); });
|
|
191
228
|
}
|
|
192
229
|
} finally {
|
|
230
|
+
this._page = null;
|
|
193
231
|
this._releaseLock();
|
|
194
232
|
}
|
|
195
233
|
}
|