@jackwener/opencli 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -2
- package/README.zh-CN.md +9 -1
- package/SKILL.md +24 -0
- package/dist/bilibili.d.ts +6 -5
- package/dist/browser.d.ts +2 -1
- package/dist/browser.js +9 -1
- package/dist/cascade.d.ts +3 -2
- package/dist/clis/bbc/news.js +42 -0
- package/dist/clis/boss/search.d.ts +1 -0
- package/dist/clis/boss/search.js +47 -0
- package/dist/clis/ctrip/search.d.ts +1 -0
- package/dist/clis/ctrip/search.js +62 -0
- package/dist/clis/index.d.ts +8 -0
- package/dist/clis/index.js +16 -0
- package/dist/clis/reuters/search.d.ts +1 -0
- package/dist/clis/reuters/search.js +52 -0
- package/dist/clis/smzdm/search.d.ts +1 -0
- package/dist/clis/smzdm/search.js +66 -0
- package/dist/clis/weibo/hot.d.ts +1 -0
- package/dist/clis/weibo/hot.js +41 -0
- package/dist/clis/yahoo-finance/quote.d.ts +1 -0
- package/dist/clis/yahoo-finance/quote.js +74 -0
- package/dist/clis/youtube/search.d.ts +1 -0
- package/dist/clis/youtube/search.js +60 -0
- package/dist/engine.d.ts +2 -1
- package/dist/explore.js +1 -1
- package/dist/generate.js +2 -1
- package/dist/main.js +6 -4
- package/dist/pipeline/executor.d.ts +9 -0
- package/dist/pipeline/executor.js +88 -0
- package/dist/pipeline/index.d.ts +5 -0
- package/dist/pipeline/index.js +5 -0
- package/dist/pipeline/steps/browser.d.ts +12 -0
- package/dist/pipeline/steps/browser.js +68 -0
- package/dist/pipeline/steps/fetch.d.ts +5 -0
- package/dist/pipeline/steps/fetch.js +50 -0
- package/dist/pipeline/steps/intercept.d.ts +5 -0
- package/dist/pipeline/steps/intercept.js +75 -0
- package/dist/pipeline/steps/tap.d.ts +12 -0
- package/dist/pipeline/steps/tap.js +130 -0
- package/dist/pipeline/steps/transform.d.ts +8 -0
- package/dist/pipeline/steps/transform.js +53 -0
- package/dist/pipeline/template.d.ts +16 -0
- package/dist/pipeline/template.js +115 -0
- package/dist/pipeline/template.test.d.ts +4 -0
- package/dist/pipeline/template.test.js +102 -0
- package/dist/pipeline/transform.test.d.ts +4 -0
- package/dist/pipeline/transform.test.js +90 -0
- package/dist/pipeline.d.ts +5 -7
- package/dist/pipeline.js +5 -549
- package/dist/registry.d.ts +3 -2
- package/dist/runtime.d.ts +2 -1
- package/dist/types.d.ts +27 -0
- package/dist/types.js +7 -0
- package/package.json +6 -3
- package/src/bilibili.ts +9 -7
- package/src/browser.ts +8 -2
- package/src/cascade.ts +3 -2
- package/src/clis/bbc/news.ts +42 -0
- package/src/clis/boss/search.ts +47 -0
- package/src/clis/ctrip/search.ts +62 -0
- package/src/clis/index.ts +24 -0
- package/src/clis/reuters/search.ts +52 -0
- package/src/clis/smzdm/search.ts +66 -0
- package/src/clis/weibo/hot.ts +41 -0
- package/src/clis/yahoo-finance/quote.ts +74 -0
- package/src/clis/youtube/search.ts +60 -0
- package/src/engine.ts +2 -1
- package/src/explore.ts +1 -1
- package/src/generate.ts +3 -1
- package/src/main.ts +7 -5
- package/src/pipeline/executor.ts +98 -0
- package/src/pipeline/index.ts +6 -0
- package/src/pipeline/steps/browser.ts +67 -0
- package/src/pipeline/steps/fetch.ts +60 -0
- package/src/pipeline/steps/intercept.ts +78 -0
- package/src/pipeline/steps/tap.ts +137 -0
- package/src/pipeline/steps/transform.ts +50 -0
- package/src/pipeline/template.test.ts +107 -0
- package/src/pipeline/template.ts +101 -0
- package/src/pipeline/transform.test.ts +107 -0
- package/src/pipeline.ts +5 -529
- package/src/registry.ts +4 -2
- package/src/runtime.ts +3 -1
- package/src/types.ts +23 -0
- package/vitest.config.ts +7 -0
- package/dist/clis/github/search.js +0 -20
- package/dist/clis/github/trending.yaml +0 -58
- package/dist/promote.d.ts +0 -1
- package/dist/promote.js +0 -3
- package/dist/register.d.ts +0 -2
- package/dist/register.js +0 -2
- package/dist/scaffold.d.ts +0 -2
- package/dist/scaffold.js +0 -2
- package/dist/smoke.d.ts +0 -2
- package/dist/smoke.js +0 -2
- package/src/clis/github/search.ts +0 -21
- package/src/clis/github/trending.yaml +0 -58
- package/src/promote.ts +0 -3
- package/src/register.ts +0 -2
- package/src/scaffold.ts +0 -2
- package/src/smoke.ts +0 -2
- /package/dist/clis/{github/search.d.ts → bbc/news.d.ts} +0 -0
package/src/pipeline.ts
CHANGED
|
@@ -1,532 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* YAML pipeline executor.
|
|
3
|
-
*
|
|
2
|
+
* YAML pipeline executor — re-exports from modular pipeline system.
|
|
3
|
+
*
|
|
4
|
+
* This file exists for backward compatibility. All logic has been
|
|
5
|
+
* refactored into src/pipeline/ with modular step handlers.
|
|
4
6
|
*/
|
|
5
7
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
export interface PipelineContext {
|
|
9
|
-
args?: Record<string, any>;
|
|
10
|
-
debug?: boolean;
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
export async function executePipeline(
|
|
14
|
-
page: any,
|
|
15
|
-
pipeline: any[],
|
|
16
|
-
ctx: PipelineContext = {},
|
|
17
|
-
): Promise<any> {
|
|
18
|
-
const args = ctx.args ?? {};
|
|
19
|
-
const debug = ctx.debug ?? false;
|
|
20
|
-
let data: any = null;
|
|
21
|
-
const total = pipeline.length;
|
|
22
|
-
|
|
23
|
-
for (let i = 0; i < pipeline.length; i++) {
|
|
24
|
-
const step = pipeline[i];
|
|
25
|
-
if (!step || typeof step !== 'object') continue;
|
|
26
|
-
for (const [op, params] of Object.entries(step)) {
|
|
27
|
-
if (debug) debugStepStart(i + 1, total, op, params);
|
|
28
|
-
data = await executeStep(page, op, params, data, args);
|
|
29
|
-
// Detect error objects returned by steps (e.g. tap store not found)
|
|
30
|
-
if (data && typeof data === 'object' && !Array.isArray(data) && data.error) {
|
|
31
|
-
process.stderr.write(` ${chalk.yellow('⚠')} ${chalk.yellow(op)}: ${data.error}\n`);
|
|
32
|
-
if (data.hint) process.stderr.write(` ${chalk.dim('💡')} ${chalk.dim(data.hint)}\n`);
|
|
33
|
-
}
|
|
34
|
-
if (debug) debugStepResult(op, data);
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
return data;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
function normalizeEvaluateSource(source: string): string {
|
|
41
|
-
const stripped = source.trim();
|
|
42
|
-
if (!stripped) return '() => undefined';
|
|
43
|
-
if (stripped.startsWith('(') && stripped.endsWith(')()')) return `() => (${stripped})`;
|
|
44
|
-
if (/^(async\s+)?\([^)]*\)\s*=>/.test(stripped)) return stripped;
|
|
45
|
-
if (/^(async\s+)?[A-Za-z_][A-Za-z0-9_]*\s*=>/.test(stripped)) return stripped;
|
|
46
|
-
if (stripped.startsWith('function ') || stripped.startsWith('async function ')) return stripped;
|
|
47
|
-
return `() => (${stripped})`;
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
function debugStepStart(stepNum: number, total: number, op: string, params: any): void {
|
|
51
|
-
let preview = '';
|
|
52
|
-
if (typeof params === 'string') {
|
|
53
|
-
preview = params.length <= 80 ? ` → ${params}` : ` → ${params.slice(0, 77)}...`;
|
|
54
|
-
} else if (params && typeof params === 'object' && !Array.isArray(params)) {
|
|
55
|
-
preview = ` (${Object.keys(params).join(', ')})`;
|
|
56
|
-
}
|
|
57
|
-
process.stderr.write(` ${chalk.dim(`[${stepNum}/${total}]`)} ${chalk.bold.cyan(op)}${preview}\n`);
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
function debugStepResult(op: string, data: any): void {
|
|
61
|
-
if (data === null || data === undefined) {
|
|
62
|
-
process.stderr.write(` ${chalk.dim('→ (no data)')}\n`);
|
|
63
|
-
} else if (Array.isArray(data)) {
|
|
64
|
-
process.stderr.write(` ${chalk.dim(`→ ${data.length} items`)}\n`);
|
|
65
|
-
} else if (typeof data === 'object') {
|
|
66
|
-
const keys = Object.keys(data).slice(0, 5);
|
|
67
|
-
process.stderr.write(` ${chalk.dim(`→ dict (${keys.join(', ')}${Object.keys(data).length > 5 ? '...' : ''})`)}\n`);
|
|
68
|
-
} else if (typeof data === 'string') {
|
|
69
|
-
const p = data.slice(0, 60).replace(/\n/g, '\\n');
|
|
70
|
-
process.stderr.write(` ${chalk.dim(`→ "${p}${data.length > 60 ? '...' : ''}"`)}\n`);
|
|
71
|
-
} else {
|
|
72
|
-
process.stderr.write(` ${chalk.dim(`→ ${typeof data}`)}\n`);
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
// Single URL fetch helper
|
|
77
|
-
async function fetchSingle(
|
|
78
|
-
page: any, url: string, method: string,
|
|
79
|
-
queryParams: Record<string, any>, headers: Record<string, any>,
|
|
80
|
-
args: Record<string, any>, data: any,
|
|
81
|
-
): Promise<any> {
|
|
82
|
-
const renderedParams: Record<string, string> = {};
|
|
83
|
-
for (const [k, v] of Object.entries(queryParams)) renderedParams[k] = String(render(v, { args, data }));
|
|
84
|
-
const renderedHeaders: Record<string, string> = {};
|
|
85
|
-
for (const [k, v] of Object.entries(headers)) renderedHeaders[k] = String(render(v, { args, data }));
|
|
86
|
-
|
|
87
|
-
let finalUrl = url;
|
|
88
|
-
if (Object.keys(renderedParams).length > 0) {
|
|
89
|
-
const qs = new URLSearchParams(renderedParams).toString();
|
|
90
|
-
finalUrl = `${finalUrl}${finalUrl.includes('?') ? '&' : '?'}${qs}`;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
if (page === null) {
|
|
94
|
-
const resp = await fetch(finalUrl, { method: method.toUpperCase(), headers: renderedHeaders });
|
|
95
|
-
return resp.json();
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
const headersJs = JSON.stringify(renderedHeaders);
|
|
99
|
-
const escapedUrl = finalUrl.replace(/"/g, '\\"');
|
|
100
|
-
return page.evaluate(`
|
|
101
|
-
async () => {
|
|
102
|
-
const resp = await fetch("${escapedUrl}", {
|
|
103
|
-
method: "${method}", headers: ${headersJs}, credentials: "include"
|
|
104
|
-
});
|
|
105
|
-
return await resp.json();
|
|
106
|
-
}
|
|
107
|
-
`);
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
async function executeStep(page: any, op: string, params: any, data: any, args: Record<string, any>): Promise<any> {
|
|
111
|
-
switch (op) {
|
|
112
|
-
case 'navigate': {
|
|
113
|
-
const url = render(params, { args, data });
|
|
114
|
-
await page.goto(String(url));
|
|
115
|
-
return data;
|
|
116
|
-
}
|
|
117
|
-
case 'fetch': {
|
|
118
|
-
const urlOrObj = typeof params === 'string' ? params : (params?.url ?? '');
|
|
119
|
-
const method = params?.method ?? 'GET';
|
|
120
|
-
const queryParams: Record<string, any> = params?.params ?? {};
|
|
121
|
-
const headers: Record<string, any> = params?.headers ?? {};
|
|
122
|
-
const urlTemplate = String(urlOrObj);
|
|
123
|
-
|
|
124
|
-
// Per-item fetch when data is array and URL references item
|
|
125
|
-
if (Array.isArray(data) && urlTemplate.includes('item')) {
|
|
126
|
-
const results: any[] = [];
|
|
127
|
-
for (let i = 0; i < data.length; i++) {
|
|
128
|
-
const itemUrl = String(render(urlTemplate, { args, data, item: data[i], index: i }));
|
|
129
|
-
results.push(await fetchSingle(page, itemUrl, method, queryParams, headers, args, data));
|
|
130
|
-
}
|
|
131
|
-
return results;
|
|
132
|
-
}
|
|
133
|
-
const url = render(urlOrObj, { args, data });
|
|
134
|
-
return fetchSingle(page, String(url), method, queryParams, headers, args, data);
|
|
135
|
-
}
|
|
136
|
-
case 'select': {
|
|
137
|
-
const pathStr = String(render(params, { args, data }));
|
|
138
|
-
if (data && typeof data === 'object') {
|
|
139
|
-
let current = data;
|
|
140
|
-
for (const part of pathStr.split('.')) {
|
|
141
|
-
if (current && typeof current === 'object' && !Array.isArray(current)) current = (current as any)[part];
|
|
142
|
-
else if (Array.isArray(current) && /^\d+$/.test(part)) current = current[parseInt(part, 10)];
|
|
143
|
-
else return null;
|
|
144
|
-
}
|
|
145
|
-
return current;
|
|
146
|
-
}
|
|
147
|
-
return data;
|
|
148
|
-
}
|
|
149
|
-
case 'evaluate': {
|
|
150
|
-
const js = String(render(params, { args, data }));
|
|
151
|
-
let result = await page.evaluate(normalizeEvaluateSource(js));
|
|
152
|
-
// MCP may return JSON as a string — auto-parse it
|
|
153
|
-
if (typeof result === 'string') {
|
|
154
|
-
const trimmed = result.trim();
|
|
155
|
-
if ((trimmed.startsWith('[') && trimmed.endsWith(']')) || (trimmed.startsWith('{') && trimmed.endsWith('}'))) {
|
|
156
|
-
try { result = JSON.parse(trimmed); } catch {}
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
return result;
|
|
160
|
-
}
|
|
161
|
-
case 'snapshot': {
|
|
162
|
-
const opts = (typeof params === 'object' && params) ? params : {};
|
|
163
|
-
return page.snapshot({ interactive: opts.interactive ?? false, compact: opts.compact ?? false, maxDepth: opts.max_depth, raw: opts.raw ?? false });
|
|
164
|
-
}
|
|
165
|
-
case 'click': {
|
|
166
|
-
await page.click(String(render(params, { args, data })).replace(/^@/, ''));
|
|
167
|
-
return data;
|
|
168
|
-
}
|
|
169
|
-
case 'type': {
|
|
170
|
-
if (typeof params === 'object' && params) {
|
|
171
|
-
const ref = String(render(params.ref ?? '', { args, data })).replace(/^@/, '');
|
|
172
|
-
const text = String(render(params.text ?? '', { args, data }));
|
|
173
|
-
await page.typeText(ref, text);
|
|
174
|
-
if (params.submit) await page.pressKey('Enter');
|
|
175
|
-
}
|
|
176
|
-
return data;
|
|
177
|
-
}
|
|
178
|
-
case 'wait': {
|
|
179
|
-
if (typeof params === 'number') await page.wait(params);
|
|
180
|
-
else if (typeof params === 'object' && params) {
|
|
181
|
-
if ('text' in params) {
|
|
182
|
-
const timeout = params.timeout ?? 10;
|
|
183
|
-
const start = Date.now();
|
|
184
|
-
while ((Date.now() - start) / 1000 < timeout) {
|
|
185
|
-
const snap = await page.snapshot({ raw: true });
|
|
186
|
-
if (typeof snap === 'string' && snap.includes(params.text)) break;
|
|
187
|
-
await page.wait(0.5);
|
|
188
|
-
}
|
|
189
|
-
} else if ('time' in params) await page.wait(Number(params.time));
|
|
190
|
-
} else if (typeof params === 'string') await page.wait(Number(render(params, { args, data })));
|
|
191
|
-
return data;
|
|
192
|
-
}
|
|
193
|
-
case 'press': {
|
|
194
|
-
await page.pressKey(String(render(params, { args, data })));
|
|
195
|
-
return data;
|
|
196
|
-
}
|
|
197
|
-
case 'map': {
|
|
198
|
-
if (!data || typeof data !== 'object') return data;
|
|
199
|
-
let items: any[] = Array.isArray(data) ? data : [data];
|
|
200
|
-
if (!Array.isArray(data) && typeof data === 'object' && 'data' in data) items = data.data;
|
|
201
|
-
const result: any[] = [];
|
|
202
|
-
for (let i = 0; i < items.length; i++) {
|
|
203
|
-
const item = items[i];
|
|
204
|
-
const row: Record<string, any> = {};
|
|
205
|
-
for (const [key, template] of Object.entries(params)) row[key] = render(template, { args, data, item, index: i });
|
|
206
|
-
result.push(row);
|
|
207
|
-
}
|
|
208
|
-
return result;
|
|
209
|
-
}
|
|
210
|
-
case 'filter': {
|
|
211
|
-
if (!Array.isArray(data)) return data;
|
|
212
|
-
return data.filter((item, i) => evalExpr(String(params), { args, item, index: i }));
|
|
213
|
-
}
|
|
214
|
-
case 'sort': {
|
|
215
|
-
if (!Array.isArray(data)) return data;
|
|
216
|
-
const key = typeof params === 'object' ? (params.by ?? '') : String(params);
|
|
217
|
-
const reverse = typeof params === 'object' ? params.order === 'desc' : false;
|
|
218
|
-
return [...data].sort((a, b) => { const va = a[key] ?? ''; const vb = b[key] ?? ''; const cmp = va < vb ? -1 : va > vb ? 1 : 0; return reverse ? -cmp : cmp; });
|
|
219
|
-
}
|
|
220
|
-
case 'limit': {
|
|
221
|
-
if (!Array.isArray(data)) return data;
|
|
222
|
-
return data.slice(0, Number(render(params, { args, data })));
|
|
223
|
-
}
|
|
224
|
-
case 'intercept': {
|
|
225
|
-
// Declarative XHR interception step
|
|
226
|
-
// Usage:
|
|
227
|
-
// intercept:
|
|
228
|
-
// trigger: "navigate:https://..." | "evaluate:store.note.fetch()" | "click:ref"
|
|
229
|
-
// capture: "api/pattern" # URL substring to match
|
|
230
|
-
// timeout: 5 # seconds to wait for matching request
|
|
231
|
-
// select: "data.items" # optional: extract sub-path from response
|
|
232
|
-
const cfg = typeof params === 'object' ? params : {};
|
|
233
|
-
const trigger = cfg.trigger ?? '';
|
|
234
|
-
const capturePattern = cfg.capture ?? '';
|
|
235
|
-
const timeout = cfg.timeout ?? 8;
|
|
236
|
-
const selectPath = cfg.select ?? null;
|
|
237
|
-
|
|
238
|
-
if (!capturePattern) return data;
|
|
239
|
-
|
|
240
|
-
// Step 1: Execute the trigger action
|
|
241
|
-
if (trigger.startsWith('navigate:')) {
|
|
242
|
-
const url = render(trigger.slice('navigate:'.length), { args, data });
|
|
243
|
-
await page.goto(String(url));
|
|
244
|
-
} else if (trigger.startsWith('evaluate:')) {
|
|
245
|
-
const js = trigger.slice('evaluate:'.length);
|
|
246
|
-
await page.evaluate(normalizeEvaluateSource(render(js, { args, data }) as string));
|
|
247
|
-
} else if (trigger.startsWith('click:')) {
|
|
248
|
-
const ref = render(trigger.slice('click:'.length), { args, data });
|
|
249
|
-
await page.click(String(ref).replace(/^@/, ''));
|
|
250
|
-
} else if (trigger === 'scroll') {
|
|
251
|
-
await page.scroll('down');
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
// Step 2: Wait a bit for network requests to fire
|
|
255
|
-
await page.wait(Math.min(timeout, 3));
|
|
256
|
-
|
|
257
|
-
// Step 3: Get network requests and find matching ones
|
|
258
|
-
const rawNetwork = await page.networkRequests(false);
|
|
259
|
-
const matchingResponses: any[] = [];
|
|
260
|
-
|
|
261
|
-
if (typeof rawNetwork === 'string') {
|
|
262
|
-
// Parse the network output to find matching URLs
|
|
263
|
-
const lines = rawNetwork.split('\n');
|
|
264
|
-
for (const line of lines) {
|
|
265
|
-
const match = line.match(/\[?(GET|POST)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i);
|
|
266
|
-
if (match) {
|
|
267
|
-
const [, method, url, status] = match;
|
|
268
|
-
if (url.includes(capturePattern) && status === '200') {
|
|
269
|
-
// Re-fetch the matching URL to get the response body
|
|
270
|
-
try {
|
|
271
|
-
const body = await page.evaluate(`
|
|
272
|
-
async () => {
|
|
273
|
-
try {
|
|
274
|
-
const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
|
|
275
|
-
if (!resp.ok) return null;
|
|
276
|
-
return await resp.json();
|
|
277
|
-
} catch { return null; }
|
|
278
|
-
}
|
|
279
|
-
`);
|
|
280
|
-
if (body) matchingResponses.push(body);
|
|
281
|
-
} catch {}
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
// Step 4: Select from response if specified
|
|
288
|
-
let result = matchingResponses.length === 1 ? matchingResponses[0] :
|
|
289
|
-
matchingResponses.length > 1 ? matchingResponses : data;
|
|
290
|
-
|
|
291
|
-
if (selectPath && result) {
|
|
292
|
-
let current = result;
|
|
293
|
-
for (const part of String(selectPath).split('.')) {
|
|
294
|
-
if (current && typeof current === 'object' && !Array.isArray(current)) {
|
|
295
|
-
current = current[part];
|
|
296
|
-
} else break;
|
|
297
|
-
}
|
|
298
|
-
result = current ?? result;
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
return result;
|
|
302
|
-
}
|
|
303
|
-
case 'tap': {
|
|
304
|
-
// ── Declarative Store Action Bridge ──────────────────────────────────
|
|
305
|
-
// Usage:
|
|
306
|
-
// tap:
|
|
307
|
-
// store: feed # Pinia/Vuex store name
|
|
308
|
-
// action: fetchFeeds # Store action to call
|
|
309
|
-
// args: [] # Optional args to pass to action
|
|
310
|
-
// capture: homefeed # URL pattern to capture response
|
|
311
|
-
// timeout: 5 # Seconds to wait for network (default: 5)
|
|
312
|
-
// select: data.items # Optional: extract sub-path from response
|
|
313
|
-
// framework: pinia # Optional: pinia | vuex (auto-detected if omitted)
|
|
314
|
-
//
|
|
315
|
-
// Generates a self-contained IIFE that:
|
|
316
|
-
// 1. Injects fetch + XHR dual interception proxy
|
|
317
|
-
// 2. Finds the Pinia/Vuex store and calls the action
|
|
318
|
-
// 3. Captures the response matching the URL pattern
|
|
319
|
-
// 4. Auto-cleans up interception in finally block
|
|
320
|
-
// 5. Returns the captured data (optionally sub-selected)
|
|
321
|
-
|
|
322
|
-
const cfg = typeof params === 'object' ? params : {};
|
|
323
|
-
const storeName = String(render(cfg.store ?? '', { args, data }));
|
|
324
|
-
const actionName = String(render(cfg.action ?? '', { args, data }));
|
|
325
|
-
const capturePattern = String(render(cfg.capture ?? '', { args, data }));
|
|
326
|
-
const timeout = cfg.timeout ?? 5;
|
|
327
|
-
const selectPath = cfg.select ? String(render(cfg.select, { args, data })) : null;
|
|
328
|
-
const framework = cfg.framework ?? null; // auto-detect if null
|
|
329
|
-
const actionArgs = cfg.args ?? [];
|
|
330
|
-
|
|
331
|
-
if (!storeName || !actionName) throw new Error('tap: store and action are required');
|
|
332
|
-
|
|
333
|
-
// Build select chain for the captured response
|
|
334
|
-
const selectChain = selectPath
|
|
335
|
-
? selectPath.split('.').map((p: string) => `?.[${JSON.stringify(p)}]`).join('')
|
|
336
|
-
: '';
|
|
337
|
-
|
|
338
|
-
// Serialize action arguments
|
|
339
|
-
const actionArgsRendered = actionArgs.map((a: any) => {
|
|
340
|
-
const rendered = render(a, { args, data });
|
|
341
|
-
return JSON.stringify(rendered);
|
|
342
|
-
});
|
|
343
|
-
const actionCall = actionArgsRendered.length
|
|
344
|
-
? `store[${JSON.stringify(actionName)}](${actionArgsRendered.join(', ')})`
|
|
345
|
-
: `store[${JSON.stringify(actionName)}]()`;
|
|
346
|
-
|
|
347
|
-
const js = `
|
|
348
|
-
async () => {
|
|
349
|
-
// ── 1. Setup capture proxy (fetch + XHR dual interception) ──
|
|
350
|
-
let captured = null;
|
|
351
|
-
const capturePattern = ${JSON.stringify(capturePattern)};
|
|
352
|
-
|
|
353
|
-
// Intercept fetch API
|
|
354
|
-
const origFetch = window.fetch;
|
|
355
|
-
window.fetch = async function(...fetchArgs) {
|
|
356
|
-
const resp = await origFetch.apply(this, fetchArgs);
|
|
357
|
-
try {
|
|
358
|
-
const url = typeof fetchArgs[0] === 'string' ? fetchArgs[0]
|
|
359
|
-
: fetchArgs[0] instanceof Request ? fetchArgs[0].url : String(fetchArgs[0]);
|
|
360
|
-
if (capturePattern && url.includes(capturePattern) && !captured) {
|
|
361
|
-
try { captured = await resp.clone().json(); } catch {}
|
|
362
|
-
}
|
|
363
|
-
} catch {}
|
|
364
|
-
return resp;
|
|
365
|
-
};
|
|
366
|
-
|
|
367
|
-
// Intercept XMLHttpRequest
|
|
368
|
-
const origXhrOpen = XMLHttpRequest.prototype.open;
|
|
369
|
-
const origXhrSend = XMLHttpRequest.prototype.send;
|
|
370
|
-
XMLHttpRequest.prototype.open = function(method, url) {
|
|
371
|
-
this.__tapUrl = String(url);
|
|
372
|
-
return origXhrOpen.apply(this, arguments);
|
|
373
|
-
};
|
|
374
|
-
XMLHttpRequest.prototype.send = function(body) {
|
|
375
|
-
if (capturePattern && this.__tapUrl?.includes(capturePattern)) {
|
|
376
|
-
const xhr = this;
|
|
377
|
-
const origHandler = xhr.onreadystatechange;
|
|
378
|
-
xhr.onreadystatechange = function() {
|
|
379
|
-
if (xhr.readyState === 4 && !captured) {
|
|
380
|
-
try { captured = JSON.parse(xhr.responseText); } catch {}
|
|
381
|
-
}
|
|
382
|
-
if (origHandler) origHandler.apply(this, arguments);
|
|
383
|
-
};
|
|
384
|
-
// Also handle onload
|
|
385
|
-
const origOnload = xhr.onload;
|
|
386
|
-
xhr.onload = function() {
|
|
387
|
-
if (!captured) { try { captured = JSON.parse(xhr.responseText); } catch {} }
|
|
388
|
-
if (origOnload) origOnload.apply(this, arguments);
|
|
389
|
-
};
|
|
390
|
-
}
|
|
391
|
-
return origXhrSend.apply(this, arguments);
|
|
392
|
-
};
|
|
393
|
-
|
|
394
|
-
try {
|
|
395
|
-
// ── 2. Find store ──
|
|
396
|
-
let store = null;
|
|
397
|
-
const storeName = ${JSON.stringify(storeName)};
|
|
398
|
-
const fw = ${JSON.stringify(framework)};
|
|
399
|
-
|
|
400
|
-
// Auto-detect framework if not specified
|
|
401
|
-
const app = document.querySelector('#app');
|
|
402
|
-
if (!fw || fw === 'pinia') {
|
|
403
|
-
// Try Pinia (Vue 3)
|
|
404
|
-
try {
|
|
405
|
-
const pinia = app?.__vue_app__?.config?.globalProperties?.$pinia;
|
|
406
|
-
if (pinia?._s) store = pinia._s.get(storeName);
|
|
407
|
-
} catch {}
|
|
408
|
-
}
|
|
409
|
-
if (!store && (!fw || fw === 'vuex')) {
|
|
410
|
-
// Try Vuex (Vue 2/3)
|
|
411
|
-
try {
|
|
412
|
-
const vuexStore = app?.__vue_app__?.config?.globalProperties?.$store
|
|
413
|
-
?? app?.__vue__?.$store;
|
|
414
|
-
if (vuexStore) {
|
|
415
|
-
// Vuex doesn't have named stores like Pinia, dispatch action
|
|
416
|
-
store = { [${JSON.stringify(actionName)}]: (...a) => vuexStore.dispatch(storeName + '/' + ${JSON.stringify(actionName)}, ...a) };
|
|
417
|
-
}
|
|
418
|
-
} catch {}
|
|
419
|
-
}
|
|
420
|
-
|
|
421
|
-
if (!store) return { error: 'Store not found: ' + storeName, hint: 'Page may not be fully loaded or store name may be incorrect' };
|
|
422
|
-
if (typeof store[${JSON.stringify(actionName)}] !== 'function') {
|
|
423
|
-
return { error: 'Action not found: ' + ${JSON.stringify(actionName)} + ' on store ' + storeName,
|
|
424
|
-
hint: 'Available: ' + Object.keys(store).filter(k => typeof store[k] === 'function' && !k.startsWith('$') && !k.startsWith('_')).join(', ') };
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
// ── 3. Call store action ──
|
|
428
|
-
await ${actionCall};
|
|
429
|
-
|
|
430
|
-
// ── 4. Wait for network response ──
|
|
431
|
-
const deadline = Date.now() + ${timeout} * 1000;
|
|
432
|
-
while (!captured && Date.now() < deadline) {
|
|
433
|
-
await new Promise(r => setTimeout(r, 200));
|
|
434
|
-
}
|
|
435
|
-
} finally {
|
|
436
|
-
// ── 5. Always restore originals ──
|
|
437
|
-
window.fetch = origFetch;
|
|
438
|
-
XMLHttpRequest.prototype.open = origXhrOpen;
|
|
439
|
-
XMLHttpRequest.prototype.send = origXhrSend;
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
if (!captured) return { error: 'No matching response captured for pattern: ' + capturePattern };
|
|
443
|
-
return captured${selectChain} ?? captured;
|
|
444
|
-
}
|
|
445
|
-
`;
|
|
446
|
-
|
|
447
|
-
return page.evaluate(js);
|
|
448
|
-
}
|
|
449
|
-
default: return data;
|
|
450
|
-
}
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
// Template engine: ${{ ... }}
|
|
454
|
-
interface RenderContext { args?: Record<string, any>; data?: any; item?: any; index?: number; }
|
|
455
|
-
|
|
456
|
-
function render(template: any, ctx: RenderContext): any {
|
|
457
|
-
if (typeof template !== 'string') return template;
|
|
458
|
-
const fullMatch = template.match(/^\$\{\{\s*(.*?)\s*\}\}$/);
|
|
459
|
-
if (fullMatch) return evalExpr(fullMatch[1].trim(), ctx);
|
|
460
|
-
return template.replace(/\$\{\{\s*(.*?)\s*\}\}/g, (_m, expr) => String(evalExpr(expr.trim(), ctx)));
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
function evalExpr(expr: string, ctx: RenderContext): any {
|
|
464
|
-
const args = ctx.args ?? {};
|
|
465
|
-
const item = ctx.item ?? {};
|
|
466
|
-
const data = ctx.data;
|
|
467
|
-
const index = ctx.index ?? 0;
|
|
468
|
-
|
|
469
|
-
// Default filter: args.limit | default(20)
|
|
470
|
-
if (expr.includes('|') && expr.includes('default(')) {
|
|
471
|
-
const [mainExpr, rest] = expr.split('|', 2);
|
|
472
|
-
const defaultMatch = rest.match(/default\((.+?)\)/);
|
|
473
|
-
const defaultVal = defaultMatch ? defaultMatch[1] : null;
|
|
474
|
-
const result = resolvePath(mainExpr.trim(), { args, item, data, index });
|
|
475
|
-
if (result === null || result === undefined) {
|
|
476
|
-
if (defaultVal !== null) {
|
|
477
|
-
const intVal = parseInt(defaultVal!, 10);
|
|
478
|
-
if (!isNaN(intVal) && String(intVal) === defaultVal!.trim()) return intVal;
|
|
479
|
-
return defaultVal!.replace(/^['"]|['"]$/g, '');
|
|
480
|
-
}
|
|
481
|
-
}
|
|
482
|
-
return result;
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
// Arithmetic: index + 1
|
|
486
|
-
const arithMatch = expr.match(/^([\w][\w.]*)\s*([+\-*/])\s*(\d+)$/);
|
|
487
|
-
if (arithMatch) {
|
|
488
|
-
const [, varName, op, numStr] = arithMatch;
|
|
489
|
-
const val = resolvePath(varName, { args, item, data, index });
|
|
490
|
-
if (val !== null && val !== undefined) {
|
|
491
|
-
const numVal = Number(val); const num = Number(numStr);
|
|
492
|
-
if (!isNaN(numVal)) {
|
|
493
|
-
switch (op) {
|
|
494
|
-
case '+': return numVal + num; case '-': return numVal - num;
|
|
495
|
-
case '*': return numVal * num; case '/': return num !== 0 ? numVal / num : 0;
|
|
496
|
-
}
|
|
497
|
-
}
|
|
498
|
-
}
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
// JS-like fallback expression: item.tweetCount || 'N/A'
|
|
502
|
-
const orMatch = expr.match(/^(.+?)\s*\|\|\s*(.+)$/);
|
|
503
|
-
if (orMatch) {
|
|
504
|
-
const left = evalExpr(orMatch[1].trim(), ctx);
|
|
505
|
-
if (left) return left;
|
|
506
|
-
const right = orMatch[2].trim();
|
|
507
|
-
return right.replace(/^['"]|['"]$/g, '');
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
return resolvePath(expr, { args, item, data, index });
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
function resolvePath(pathStr: string, ctx: RenderContext): any {
|
|
514
|
-
const args = ctx.args ?? {};
|
|
515
|
-
const item = ctx.item ?? {};
|
|
516
|
-
const data = ctx.data;
|
|
517
|
-
const index = ctx.index ?? 0;
|
|
518
|
-
const parts = pathStr.split('.');
|
|
519
|
-
const rootName = parts[0];
|
|
520
|
-
let obj: any; let rest: string[];
|
|
521
|
-
if (rootName === 'args') { obj = args; rest = parts.slice(1); }
|
|
522
|
-
else if (rootName === 'item') { obj = item; rest = parts.slice(1); }
|
|
523
|
-
else if (rootName === 'data') { obj = data; rest = parts.slice(1); }
|
|
524
|
-
else if (rootName === 'index') return index;
|
|
525
|
-
else { obj = item; rest = parts; }
|
|
526
|
-
for (const part of rest) {
|
|
527
|
-
if (obj && typeof obj === 'object' && !Array.isArray(obj)) obj = obj[part];
|
|
528
|
-
else if (Array.isArray(obj) && /^\d+$/.test(part)) obj = obj[parseInt(part, 10)];
|
|
529
|
-
else return null;
|
|
530
|
-
}
|
|
531
|
-
return obj;
|
|
532
|
-
}
|
|
8
|
+
export { executePipeline, type PipelineContext } from './pipeline/index.js';
|
package/src/registry.ts
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
* Core registry: Strategy enum, Arg/CliCommand interfaces, cli() registration.
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
+
import type { IPage } from './types.js';
|
|
6
|
+
|
|
5
7
|
export enum Strategy {
|
|
6
8
|
PUBLIC = 'public',
|
|
7
9
|
COOKIE = 'cookie',
|
|
@@ -28,7 +30,7 @@ export interface CliCommand {
|
|
|
28
30
|
browser?: boolean;
|
|
29
31
|
args: Arg[];
|
|
30
32
|
columns?: string[];
|
|
31
|
-
func?: (page:
|
|
33
|
+
func?: (page: IPage | null, kwargs: Record<string, any>, debug?: boolean) => Promise<any>;
|
|
32
34
|
pipeline?: any[];
|
|
33
35
|
timeoutSeconds?: number;
|
|
34
36
|
source?: string;
|
|
@@ -43,7 +45,7 @@ export interface CliOptions {
|
|
|
43
45
|
browser?: boolean;
|
|
44
46
|
args?: Arg[];
|
|
45
47
|
columns?: string[];
|
|
46
|
-
func?: (page:
|
|
48
|
+
func?: (page: IPage | null, kwargs: Record<string, any>, debug?: boolean) => Promise<any>;
|
|
47
49
|
pipeline?: any[];
|
|
48
50
|
timeoutSeconds?: number;
|
|
49
51
|
}
|
package/src/runtime.ts
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
* Runtime utilities: timeouts and browser session management.
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
+
import type { IPage } from './types.js';
|
|
6
|
+
|
|
5
7
|
export const DEFAULT_BROWSER_CONNECT_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_CONNECT_TIMEOUT ?? '30', 10);
|
|
6
8
|
export const DEFAULT_BROWSER_COMMAND_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_COMMAND_TIMEOUT ?? '45', 10);
|
|
7
9
|
export const DEFAULT_BROWSER_EXPLORE_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_EXPLORE_TIMEOUT ?? '120', 10);
|
|
@@ -24,7 +26,7 @@ export async function runWithTimeout<T>(
|
|
|
24
26
|
|
|
25
27
|
export async function browserSession<T>(
|
|
26
28
|
BrowserFactory: new () => any,
|
|
27
|
-
fn: (page:
|
|
29
|
+
fn: (page: IPage) => Promise<T>,
|
|
28
30
|
): Promise<T> {
|
|
29
31
|
const mcp = new BrowserFactory();
|
|
30
32
|
try {
|
package/src/types.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Page interface: type-safe abstraction over Playwright MCP browser page.
|
|
3
|
+
*
|
|
4
|
+
* All pipeline steps and CLI adapters should use this interface
|
|
5
|
+
* instead of `any` for browser interactions.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export interface IPage {
|
|
9
|
+
goto(url: string): Promise<void>;
|
|
10
|
+
evaluate(js: string): Promise<any>;
|
|
11
|
+
snapshot(opts?: { interactive?: boolean; compact?: boolean; maxDepth?: number; raw?: boolean }): Promise<any>;
|
|
12
|
+
click(ref: string): Promise<void>;
|
|
13
|
+
typeText(ref: string, text: string): Promise<void>;
|
|
14
|
+
pressKey(key: string): Promise<void>;
|
|
15
|
+
wait(seconds: number): Promise<void>;
|
|
16
|
+
tabs(): Promise<any>;
|
|
17
|
+
closeTab(index?: number): Promise<void>;
|
|
18
|
+
newTab(): Promise<void>;
|
|
19
|
+
selectTab(index: number): Promise<void>;
|
|
20
|
+
networkRequests(includeStatic?: boolean): Promise<any>;
|
|
21
|
+
consoleMessages(level?: string): Promise<any>;
|
|
22
|
+
scroll(direction?: string, amount?: number): Promise<void>;
|
|
23
|
+
}
|
package/vitest.config.ts
ADDED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import { cli, Strategy } from '../../registry.js';
|
|
2
|
-
cli({
|
|
3
|
-
site: 'github', name: 'search', description: 'Search GitHub repositories', domain: 'github.com', strategy: Strategy.PUBLIC, browser: false,
|
|
4
|
-
args: [
|
|
5
|
-
{ name: 'keyword', required: true, help: 'Search keyword' },
|
|
6
|
-
{ name: 'sort', default: 'stars', help: 'Sort by: stars, forks, updated' },
|
|
7
|
-
{ name: 'limit', type: 'int', default: 20, help: 'Number of results' },
|
|
8
|
-
],
|
|
9
|
-
columns: ['rank', 'name', 'stars', 'language', 'description'],
|
|
10
|
-
func: async (_page, kwargs) => {
|
|
11
|
-
const { keyword, sort = 'stars', limit = 20 } = kwargs;
|
|
12
|
-
const resp = await fetch(`https://api.github.com/search/repositories?${new URLSearchParams({ q: keyword, sort, order: 'desc', per_page: String(Math.min(Number(limit), 100)) })}`, {
|
|
13
|
-
headers: { Accept: 'application/vnd.github.v3+json', 'User-Agent': 'opencli/0.1' },
|
|
14
|
-
});
|
|
15
|
-
const data = await resp.json();
|
|
16
|
-
return (data.items ?? []).slice(0, Number(limit)).map((item, i) => ({
|
|
17
|
-
rank: i + 1, name: item.full_name, stars: item.stargazers_count, language: item.language ?? '', description: (item.description ?? '').slice(0, 80),
|
|
18
|
-
}));
|
|
19
|
-
},
|
|
20
|
-
});
|