@jackwener/opencli 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI-CREATOR.md +594 -0
- package/README.md +124 -39
- package/README.zh-CN.md +151 -0
- package/SKILL.md +178 -102
- package/dist/bilibili.d.ts +6 -5
- package/dist/browser.d.ts +3 -1
- package/dist/browser.js +44 -2
- package/dist/cascade.d.ts +46 -0
- package/dist/cascade.js +180 -0
- package/dist/clis/bbc/news.js +42 -0
- package/dist/clis/bilibili/hot.yaml +38 -0
- package/dist/clis/boss/search.js +47 -0
- package/dist/clis/ctrip/search.d.ts +1 -0
- package/dist/clis/ctrip/search.js +62 -0
- package/dist/clis/hackernews/top.yaml +36 -0
- package/dist/clis/index.d.ts +10 -1
- package/dist/clis/index.js +19 -1
- package/dist/clis/reddit/hot.yaml +46 -0
- package/dist/clis/reuters/search.d.ts +1 -0
- package/dist/clis/reuters/search.js +52 -0
- package/dist/clis/smzdm/search.d.ts +1 -0
- package/dist/clis/smzdm/search.js +66 -0
- package/dist/clis/twitter/trending.yaml +40 -0
- package/dist/clis/v2ex/hot.yaml +25 -0
- package/dist/clis/v2ex/latest.yaml +25 -0
- package/dist/clis/v2ex/topic.yaml +27 -0
- package/dist/clis/weibo/hot.d.ts +1 -0
- package/dist/clis/weibo/hot.js +41 -0
- package/dist/clis/xiaohongshu/feed.yaml +32 -0
- package/dist/clis/xiaohongshu/notifications.yaml +38 -0
- package/dist/clis/xiaohongshu/search.d.ts +5 -0
- package/dist/clis/xiaohongshu/search.js +68 -0
- package/dist/clis/yahoo-finance/quote.d.ts +1 -0
- package/dist/clis/yahoo-finance/quote.js +74 -0
- package/dist/clis/youtube/search.d.ts +1 -0
- package/dist/clis/youtube/search.js +60 -0
- package/dist/clis/zhihu/hot.yaml +42 -0
- package/dist/clis/zhihu/question.d.ts +1 -0
- package/dist/clis/zhihu/question.js +39 -0
- package/dist/clis/zhihu/search.yaml +55 -0
- package/dist/engine.d.ts +2 -1
- package/dist/explore.d.ts +23 -13
- package/dist/explore.js +293 -422
- package/dist/generate.js +2 -1
- package/dist/main.js +21 -2
- package/dist/pipeline/executor.d.ts +9 -0
- package/dist/pipeline/executor.js +88 -0
- package/dist/pipeline/index.d.ts +5 -0
- package/dist/pipeline/index.js +5 -0
- package/dist/pipeline/steps/browser.d.ts +12 -0
- package/dist/pipeline/steps/browser.js +68 -0
- package/dist/pipeline/steps/fetch.d.ts +5 -0
- package/dist/pipeline/steps/fetch.js +50 -0
- package/dist/pipeline/steps/intercept.d.ts +5 -0
- package/dist/pipeline/steps/intercept.js +75 -0
- package/dist/pipeline/steps/tap.d.ts +12 -0
- package/dist/pipeline/steps/tap.js +130 -0
- package/dist/pipeline/steps/transform.d.ts +8 -0
- package/dist/pipeline/steps/transform.js +53 -0
- package/dist/pipeline/template.d.ts +16 -0
- package/dist/pipeline/template.js +115 -0
- package/dist/pipeline/template.test.d.ts +4 -0
- package/dist/pipeline/template.test.js +102 -0
- package/dist/pipeline/transform.test.d.ts +4 -0
- package/dist/pipeline/transform.test.js +90 -0
- package/dist/pipeline.d.ts +5 -7
- package/dist/pipeline.js +5 -313
- package/dist/registry.d.ts +3 -2
- package/dist/runtime.d.ts +2 -1
- package/dist/synthesize.d.ts +11 -8
- package/dist/synthesize.js +142 -118
- package/dist/types.d.ts +27 -0
- package/dist/types.js +7 -0
- package/package.json +9 -4
- package/src/bilibili.ts +9 -7
- package/src/browser.ts +41 -3
- package/src/cascade.ts +218 -0
- package/src/clis/bbc/news.ts +42 -0
- package/src/clis/boss/search.ts +47 -0
- package/src/clis/ctrip/search.ts +62 -0
- package/src/clis/index.ts +28 -1
- package/src/clis/reddit/hot.yaml +46 -0
- package/src/clis/reuters/search.ts +52 -0
- package/src/clis/smzdm/search.ts +66 -0
- package/src/clis/v2ex/hot.yaml +5 -9
- package/src/clis/v2ex/latest.yaml +5 -8
- package/src/clis/v2ex/topic.yaml +27 -0
- package/src/clis/weibo/hot.ts +41 -0
- package/src/clis/xiaohongshu/feed.yaml +32 -0
- package/src/clis/xiaohongshu/notifications.yaml +38 -0
- package/src/clis/xiaohongshu/search.ts +71 -0
- package/src/clis/yahoo-finance/quote.ts +74 -0
- package/src/clis/youtube/search.ts +60 -0
- package/src/clis/zhihu/hot.yaml +22 -8
- package/src/clis/zhihu/question.ts +45 -0
- package/src/clis/zhihu/search.yaml +55 -0
- package/src/engine.ts +2 -1
- package/src/explore.ts +303 -465
- package/src/generate.ts +3 -1
- package/src/main.ts +18 -2
- package/src/pipeline/executor.ts +98 -0
- package/src/pipeline/index.ts +6 -0
- package/src/pipeline/steps/browser.ts +67 -0
- package/src/pipeline/steps/fetch.ts +60 -0
- package/src/pipeline/steps/intercept.ts +78 -0
- package/src/pipeline/steps/tap.ts +137 -0
- package/src/pipeline/steps/transform.ts +50 -0
- package/src/pipeline/template.test.ts +107 -0
- package/src/pipeline/template.ts +101 -0
- package/src/pipeline/transform.test.ts +107 -0
- package/src/pipeline.ts +5 -292
- package/src/registry.ts +4 -2
- package/src/runtime.ts +3 -1
- package/src/synthesize.ts +142 -137
- package/src/types.ts +23 -0
- package/vitest.config.ts +7 -0
- package/dist/clis/github/search.js +0 -20
- package/dist/clis/zhihu/search.js +0 -58
- package/dist/promote.d.ts +0 -1
- package/dist/promote.js +0 -3
- package/dist/register.d.ts +0 -2
- package/dist/register.js +0 -2
- package/dist/scaffold.d.ts +0 -2
- package/dist/scaffold.js +0 -2
- package/dist/smoke.d.ts +0 -2
- package/dist/smoke.js +0 -2
- package/src/clis/github/search.ts +0 -21
- package/src/clis/github/trending.yaml +0 -58
- package/src/clis/zhihu/search.ts +0 -65
- package/src/promote.ts +0 -3
- package/src/register.ts +0 -2
- package/src/scaffold.ts +0 -2
- package/src/smoke.ts +0 -2
- /package/dist/clis/{github/search.d.ts → bbc/news.d.ts} +0 -0
- /package/dist/clis/{zhihu → boss}/search.d.ts +0 -0
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pipeline template engine: ${{ ... }} expression rendering.
|
|
3
|
+
*/
|
|
4
|
+
export function render(template, ctx) {
|
|
5
|
+
if (typeof template !== 'string')
|
|
6
|
+
return template;
|
|
7
|
+
const fullMatch = template.match(/^\$\{\{\s*(.*?)\s*\}\}$/);
|
|
8
|
+
if (fullMatch)
|
|
9
|
+
return evalExpr(fullMatch[1].trim(), ctx);
|
|
10
|
+
return template.replace(/\$\{\{\s*(.*?)\s*\}\}/g, (_m, expr) => String(evalExpr(expr.trim(), ctx)));
|
|
11
|
+
}
|
|
12
|
+
export function evalExpr(expr, ctx) {
|
|
13
|
+
const args = ctx.args ?? {};
|
|
14
|
+
const item = ctx.item ?? {};
|
|
15
|
+
const data = ctx.data;
|
|
16
|
+
const index = ctx.index ?? 0;
|
|
17
|
+
// Default filter: args.limit | default(20)
|
|
18
|
+
if (expr.includes('|') && expr.includes('default(')) {
|
|
19
|
+
const [mainExpr, rest] = expr.split('|', 2);
|
|
20
|
+
const defaultMatch = rest.match(/default\((.+?)\)/);
|
|
21
|
+
const defaultVal = defaultMatch ? defaultMatch[1] : null;
|
|
22
|
+
const result = resolvePath(mainExpr.trim(), { args, item, data, index });
|
|
23
|
+
if (result === null || result === undefined) {
|
|
24
|
+
if (defaultVal !== null) {
|
|
25
|
+
const intVal = parseInt(defaultVal, 10);
|
|
26
|
+
if (!isNaN(intVal) && String(intVal) === defaultVal.trim())
|
|
27
|
+
return intVal;
|
|
28
|
+
return defaultVal.replace(/^['"]|['"]$/g, '');
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return result;
|
|
32
|
+
}
|
|
33
|
+
// Arithmetic: index + 1
|
|
34
|
+
const arithMatch = expr.match(/^([\w][\w.]*)\s*([+\-*/])\s*(\d+)$/);
|
|
35
|
+
if (arithMatch) {
|
|
36
|
+
const [, varName, op, numStr] = arithMatch;
|
|
37
|
+
const val = resolvePath(varName, { args, item, data, index });
|
|
38
|
+
if (val !== null && val !== undefined) {
|
|
39
|
+
const numVal = Number(val);
|
|
40
|
+
const num = Number(numStr);
|
|
41
|
+
if (!isNaN(numVal)) {
|
|
42
|
+
switch (op) {
|
|
43
|
+
case '+': return numVal + num;
|
|
44
|
+
case '-': return numVal - num;
|
|
45
|
+
case '*': return numVal * num;
|
|
46
|
+
case '/': return num !== 0 ? numVal / num : 0;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
// JS-like fallback expression: item.tweetCount || 'N/A'
|
|
52
|
+
const orMatch = expr.match(/^(.+?)\s*\|\|\s*(.+)$/);
|
|
53
|
+
if (orMatch) {
|
|
54
|
+
const left = evalExpr(orMatch[1].trim(), ctx);
|
|
55
|
+
if (left)
|
|
56
|
+
return left;
|
|
57
|
+
const right = orMatch[2].trim();
|
|
58
|
+
return right.replace(/^['"]|['"]$/g, '');
|
|
59
|
+
}
|
|
60
|
+
return resolvePath(expr, { args, item, data, index });
|
|
61
|
+
}
|
|
62
|
+
export function resolvePath(pathStr, ctx) {
|
|
63
|
+
const args = ctx.args ?? {};
|
|
64
|
+
const item = ctx.item ?? {};
|
|
65
|
+
const data = ctx.data;
|
|
66
|
+
const index = ctx.index ?? 0;
|
|
67
|
+
const parts = pathStr.split('.');
|
|
68
|
+
const rootName = parts[0];
|
|
69
|
+
let obj;
|
|
70
|
+
let rest;
|
|
71
|
+
if (rootName === 'args') {
|
|
72
|
+
obj = args;
|
|
73
|
+
rest = parts.slice(1);
|
|
74
|
+
}
|
|
75
|
+
else if (rootName === 'item') {
|
|
76
|
+
obj = item;
|
|
77
|
+
rest = parts.slice(1);
|
|
78
|
+
}
|
|
79
|
+
else if (rootName === 'data') {
|
|
80
|
+
obj = data;
|
|
81
|
+
rest = parts.slice(1);
|
|
82
|
+
}
|
|
83
|
+
else if (rootName === 'index')
|
|
84
|
+
return index;
|
|
85
|
+
else {
|
|
86
|
+
obj = item;
|
|
87
|
+
rest = parts;
|
|
88
|
+
}
|
|
89
|
+
for (const part of rest) {
|
|
90
|
+
if (obj && typeof obj === 'object' && !Array.isArray(obj))
|
|
91
|
+
obj = obj[part];
|
|
92
|
+
else if (Array.isArray(obj) && /^\d+$/.test(part))
|
|
93
|
+
obj = obj[parseInt(part, 10)];
|
|
94
|
+
else
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
return obj;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Normalize JavaScript source for browser evaluate() calls.
|
|
101
|
+
*/
|
|
102
|
+
export function normalizeEvaluateSource(source) {
|
|
103
|
+
const stripped = source.trim();
|
|
104
|
+
if (!stripped)
|
|
105
|
+
return '() => undefined';
|
|
106
|
+
if (stripped.startsWith('(') && stripped.endsWith(')()'))
|
|
107
|
+
return `() => (${stripped})`;
|
|
108
|
+
if (/^(async\s+)?\([^)]*\)\s*=>/.test(stripped))
|
|
109
|
+
return stripped;
|
|
110
|
+
if (/^(async\s+)?[A-Za-z_][A-Za-z0-9_]*\s*=>/.test(stripped))
|
|
111
|
+
return stripped;
|
|
112
|
+
if (stripped.startsWith('function ') || stripped.startsWith('async function '))
|
|
113
|
+
return stripped;
|
|
114
|
+
return `() => (${stripped})`;
|
|
115
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for the pipeline template engine: render, evalExpr, resolvePath.
|
|
3
|
+
*/
|
|
4
|
+
import { describe, it, expect } from 'vitest';
|
|
5
|
+
import { render, evalExpr, resolvePath, normalizeEvaluateSource } from './template.js';
|
|
6
|
+
describe('resolvePath', () => {
|
|
7
|
+
it('resolves args path', () => {
|
|
8
|
+
expect(resolvePath('args.limit', { args: { limit: 20 } })).toBe(20);
|
|
9
|
+
});
|
|
10
|
+
it('resolves nested args path', () => {
|
|
11
|
+
expect(resolvePath('args.query.keyword', { args: { query: { keyword: 'test' } } })).toBe('test');
|
|
12
|
+
});
|
|
13
|
+
it('resolves item path', () => {
|
|
14
|
+
expect(resolvePath('item.title', { item: { title: 'Hello' } })).toBe('Hello');
|
|
15
|
+
});
|
|
16
|
+
it('resolves implicit item path (no prefix)', () => {
|
|
17
|
+
expect(resolvePath('title', { item: { title: 'World' } })).toBe('World');
|
|
18
|
+
});
|
|
19
|
+
it('resolves index', () => {
|
|
20
|
+
expect(resolvePath('index', { index: 5 })).toBe(5);
|
|
21
|
+
});
|
|
22
|
+
it('resolves data path', () => {
|
|
23
|
+
expect(resolvePath('data.items', { data: { items: [1, 2, 3] } })).toEqual([1, 2, 3]);
|
|
24
|
+
});
|
|
25
|
+
it('returns null for missing path', () => {
|
|
26
|
+
expect(resolvePath('args.missing', { args: {} })).toBeUndefined();
|
|
27
|
+
});
|
|
28
|
+
it('resolves array index', () => {
|
|
29
|
+
expect(resolvePath('data.0', { data: ['a', 'b'] })).toBe('a');
|
|
30
|
+
});
|
|
31
|
+
});
|
|
32
|
+
describe('evalExpr', () => {
|
|
33
|
+
it('evaluates default filter', () => {
|
|
34
|
+
expect(evalExpr('args.limit | default(20)', { args: {} })).toBe(20);
|
|
35
|
+
});
|
|
36
|
+
it('uses actual value over default', () => {
|
|
37
|
+
expect(evalExpr('args.limit | default(20)', { args: { limit: 10 } })).toBe(10);
|
|
38
|
+
});
|
|
39
|
+
it('evaluates string default', () => {
|
|
40
|
+
expect(evalExpr("args.name | default('unknown')", { args: {} })).toBe('unknown');
|
|
41
|
+
});
|
|
42
|
+
it('evaluates arithmetic: index + 1', () => {
|
|
43
|
+
expect(evalExpr('index + 1', { index: 0 })).toBe(1);
|
|
44
|
+
});
|
|
45
|
+
it('evaluates arithmetic: index * 2', () => {
|
|
46
|
+
expect(evalExpr('index * 2', { index: 5 })).toBe(10);
|
|
47
|
+
});
|
|
48
|
+
it('evaluates || fallback', () => {
|
|
49
|
+
expect(evalExpr("item.name || 'N/A'", { item: {} })).toBe('N/A');
|
|
50
|
+
});
|
|
51
|
+
it('evaluates || with truthy left', () => {
|
|
52
|
+
expect(evalExpr("item.name || 'N/A'", { item: { name: 'Alice' } })).toBe('Alice');
|
|
53
|
+
});
|
|
54
|
+
it('resolves simple path', () => {
|
|
55
|
+
expect(evalExpr('item.title', { item: { title: 'Test' } })).toBe('Test');
|
|
56
|
+
});
|
|
57
|
+
});
|
|
58
|
+
describe('render', () => {
|
|
59
|
+
it('renders full expression', () => {
|
|
60
|
+
expect(render('${{ args.limit }}', { args: { limit: 30 } })).toBe(30);
|
|
61
|
+
});
|
|
62
|
+
it('renders inline expression in string', () => {
|
|
63
|
+
expect(render('Hello ${{ item.name }}!', { item: { name: 'World' } })).toBe('Hello World!');
|
|
64
|
+
});
|
|
65
|
+
it('renders multiple inline expressions', () => {
|
|
66
|
+
expect(render('${{ item.first }}-${{ item.second }}', { item: { first: 'X', second: 'Y' } })).toBe('X-Y');
|
|
67
|
+
});
|
|
68
|
+
it('returns non-string values as-is', () => {
|
|
69
|
+
expect(render(42, {})).toBe(42);
|
|
70
|
+
expect(render(null, {})).toBeNull();
|
|
71
|
+
expect(render(undefined, {})).toBeUndefined();
|
|
72
|
+
});
|
|
73
|
+
it('returns full expression result as native type', () => {
|
|
74
|
+
expect(render('${{ args.list }}', { args: { list: [1, 2, 3] } })).toEqual([1, 2, 3]);
|
|
75
|
+
});
|
|
76
|
+
it('renders URL template', () => {
|
|
77
|
+
expect(render('https://api.example.com/search?q=${{ args.keyword }}', { args: { keyword: 'test' } })).toBe('https://api.example.com/search?q=test');
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
describe('normalizeEvaluateSource', () => {
|
|
81
|
+
it('wraps bare expression', () => {
|
|
82
|
+
expect(normalizeEvaluateSource('document.title')).toBe('() => (document.title)');
|
|
83
|
+
});
|
|
84
|
+
it('passes through arrow function', () => {
|
|
85
|
+
expect(normalizeEvaluateSource('() => 42')).toBe('() => 42');
|
|
86
|
+
});
|
|
87
|
+
it('passes through async arrow function', () => {
|
|
88
|
+
const src = 'async () => { return 1; }';
|
|
89
|
+
expect(normalizeEvaluateSource(src)).toBe(src);
|
|
90
|
+
});
|
|
91
|
+
it('passes through named function', () => {
|
|
92
|
+
const src = 'function foo() { return 1; }';
|
|
93
|
+
expect(normalizeEvaluateSource(src)).toBe(src);
|
|
94
|
+
});
|
|
95
|
+
it('wraps IIFE pattern', () => {
|
|
96
|
+
const src = '(async () => { return 1; })()';
|
|
97
|
+
expect(normalizeEvaluateSource(src)).toBe(`() => (${src})`);
|
|
98
|
+
});
|
|
99
|
+
it('handles empty string', () => {
|
|
100
|
+
expect(normalizeEvaluateSource('')).toBe('() => undefined');
|
|
101
|
+
});
|
|
102
|
+
});
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for pipeline transform steps: select, map, filter, sort, limit.
|
|
3
|
+
*/
|
|
4
|
+
import { describe, it, expect } from 'vitest';
|
|
5
|
+
import { stepSelect, stepMap, stepFilter, stepSort, stepLimit } from './steps/transform.js';
|
|
6
|
+
const SAMPLE_DATA = [
|
|
7
|
+
{ title: 'Alpha', score: 10, author: 'Alice' },
|
|
8
|
+
{ title: 'Beta', score: 30, author: 'Bob' },
|
|
9
|
+
{ title: 'Gamma', score: 20, author: 'Charlie' },
|
|
10
|
+
];
|
|
11
|
+
describe('stepSelect', () => {
|
|
12
|
+
it('selects nested path', async () => {
|
|
13
|
+
const data = { result: { items: [1, 2, 3] } };
|
|
14
|
+
const result = await stepSelect(null, 'result.items', data, {});
|
|
15
|
+
expect(result).toEqual([1, 2, 3]);
|
|
16
|
+
});
|
|
17
|
+
it('selects array by index', async () => {
|
|
18
|
+
const data = { list: ['a', 'b', 'c'] };
|
|
19
|
+
const result = await stepSelect(null, 'list.1', data, {});
|
|
20
|
+
expect(result).toBe('b');
|
|
21
|
+
});
|
|
22
|
+
it('returns null for missing path', async () => {
|
|
23
|
+
const result = await stepSelect(null, 'missing.path', { foo: 1 }, {});
|
|
24
|
+
expect(result).toBeNull();
|
|
25
|
+
});
|
|
26
|
+
it('returns data as-is for non-object', async () => {
|
|
27
|
+
const result = await stepSelect(null, 'foo', 'string-data', {});
|
|
28
|
+
expect(result).toBe('string-data');
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
describe('stepMap', () => {
|
|
32
|
+
it('maps array items', async () => {
|
|
33
|
+
const result = await stepMap(null, {
|
|
34
|
+
name: '${{ item.title }}',
|
|
35
|
+
rank: '${{ index + 1 }}',
|
|
36
|
+
}, SAMPLE_DATA, {});
|
|
37
|
+
expect(result).toEqual([
|
|
38
|
+
{ name: 'Alpha', rank: 1 },
|
|
39
|
+
{ name: 'Beta', rank: 2 },
|
|
40
|
+
{ name: 'Gamma', rank: 3 },
|
|
41
|
+
]);
|
|
42
|
+
});
|
|
43
|
+
it('handles single object', async () => {
|
|
44
|
+
const result = await stepMap(null, {
|
|
45
|
+
name: '${{ item.title }}',
|
|
46
|
+
}, { title: 'Solo' }, {});
|
|
47
|
+
expect(result).toEqual([{ name: 'Solo' }]);
|
|
48
|
+
});
|
|
49
|
+
it('returns null/undefined as-is', async () => {
|
|
50
|
+
expect(await stepMap(null, { x: '${{ item.x }}' }, null, {})).toBeNull();
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
describe('stepFilter', () => {
|
|
54
|
+
it('filters by expression', async () => {
|
|
55
|
+
const result = await stepFilter(null, 'item.score', SAMPLE_DATA, {});
|
|
56
|
+
expect(result).toHaveLength(3); // all truthy
|
|
57
|
+
});
|
|
58
|
+
it('returns non-array as-is', async () => {
|
|
59
|
+
const result = await stepFilter(null, 'item.x', 'not-array', {});
|
|
60
|
+
expect(result).toBe('not-array');
|
|
61
|
+
});
|
|
62
|
+
});
|
|
63
|
+
describe('stepSort', () => {
|
|
64
|
+
it('sorts ascending by key', async () => {
|
|
65
|
+
const result = await stepSort(null, 'score', SAMPLE_DATA, {});
|
|
66
|
+
expect(result.map((r) => r.title)).toEqual(['Alpha', 'Gamma', 'Beta']);
|
|
67
|
+
});
|
|
68
|
+
it('sorts descending', async () => {
|
|
69
|
+
const result = await stepSort(null, { by: 'score', order: 'desc' }, SAMPLE_DATA, {});
|
|
70
|
+
expect(result.map((r) => r.title)).toEqual(['Beta', 'Gamma', 'Alpha']);
|
|
71
|
+
});
|
|
72
|
+
it('does not mutate original', async () => {
|
|
73
|
+
const original = [...SAMPLE_DATA];
|
|
74
|
+
await stepSort(null, 'score', SAMPLE_DATA, {});
|
|
75
|
+
expect(SAMPLE_DATA).toEqual(original);
|
|
76
|
+
});
|
|
77
|
+
});
|
|
78
|
+
describe('stepLimit', () => {
|
|
79
|
+
it('limits array to N items', async () => {
|
|
80
|
+
const result = await stepLimit(null, '2', SAMPLE_DATA, {});
|
|
81
|
+
expect(result).toHaveLength(2);
|
|
82
|
+
});
|
|
83
|
+
it('limits using template expression', async () => {
|
|
84
|
+
const result = await stepLimit(null, '${{ args.limit }}', SAMPLE_DATA, { limit: 1 });
|
|
85
|
+
expect(result).toHaveLength(1);
|
|
86
|
+
});
|
|
87
|
+
it('returns non-array as-is', async () => {
|
|
88
|
+
expect(await stepLimit(null, '5', 'string', {})).toBe('string');
|
|
89
|
+
});
|
|
90
|
+
});
|
package/dist/pipeline.d.ts
CHANGED
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* YAML pipeline executor.
|
|
3
|
-
*
|
|
2
|
+
* YAML pipeline executor — re-exports from modular pipeline system.
|
|
3
|
+
*
|
|
4
|
+
* This file exists for backward compatibility. All logic has been
|
|
5
|
+
* refactored into src/pipeline/ with modular step handlers.
|
|
4
6
|
*/
|
|
5
|
-
export
|
|
6
|
-
args?: Record<string, any>;
|
|
7
|
-
debug?: boolean;
|
|
8
|
-
}
|
|
9
|
-
export declare function executePipeline(page: any, pipeline: any[], ctx?: PipelineContext): Promise<any>;
|
|
7
|
+
export { executePipeline, type PipelineContext } from './pipeline/index.js';
|
package/dist/pipeline.js
CHANGED
|
@@ -1,315 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* YAML pipeline executor.
|
|
3
|
-
*
|
|
2
|
+
* YAML pipeline executor — re-exports from modular pipeline system.
|
|
3
|
+
*
|
|
4
|
+
* This file exists for backward compatibility. All logic has been
|
|
5
|
+
* refactored into src/pipeline/ with modular step handlers.
|
|
4
6
|
*/
|
|
5
|
-
|
|
6
|
-
export async function executePipeline(page, pipeline, ctx = {}) {
|
|
7
|
-
const args = ctx.args ?? {};
|
|
8
|
-
const debug = ctx.debug ?? false;
|
|
9
|
-
let data = null;
|
|
10
|
-
const total = pipeline.length;
|
|
11
|
-
for (let i = 0; i < pipeline.length; i++) {
|
|
12
|
-
const step = pipeline[i];
|
|
13
|
-
if (!step || typeof step !== 'object')
|
|
14
|
-
continue;
|
|
15
|
-
for (const [op, params] of Object.entries(step)) {
|
|
16
|
-
if (debug)
|
|
17
|
-
debugStepStart(i + 1, total, op, params);
|
|
18
|
-
data = await executeStep(page, op, params, data, args);
|
|
19
|
-
if (debug)
|
|
20
|
-
debugStepResult(op, data);
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
|
-
return data;
|
|
24
|
-
}
|
|
25
|
-
function normalizeEvaluateSource(source) {
|
|
26
|
-
const stripped = source.trim();
|
|
27
|
-
if (!stripped)
|
|
28
|
-
return '() => undefined';
|
|
29
|
-
if (stripped.startsWith('(') && stripped.endsWith(')()'))
|
|
30
|
-
return `() => (${stripped})`;
|
|
31
|
-
if (/^(async\s+)?\([^)]*\)\s*=>/.test(stripped))
|
|
32
|
-
return stripped;
|
|
33
|
-
if (/^(async\s+)?[A-Za-z_][A-Za-z0-9_]*\s*=>/.test(stripped))
|
|
34
|
-
return stripped;
|
|
35
|
-
if (stripped.startsWith('function ') || stripped.startsWith('async function '))
|
|
36
|
-
return stripped;
|
|
37
|
-
return `() => (${stripped})`;
|
|
38
|
-
}
|
|
39
|
-
function debugStepStart(stepNum, total, op, params) {
|
|
40
|
-
let preview = '';
|
|
41
|
-
if (typeof params === 'string') {
|
|
42
|
-
preview = params.length <= 80 ? ` → ${params}` : ` → ${params.slice(0, 77)}...`;
|
|
43
|
-
}
|
|
44
|
-
else if (params && typeof params === 'object' && !Array.isArray(params)) {
|
|
45
|
-
preview = ` (${Object.keys(params).join(', ')})`;
|
|
46
|
-
}
|
|
47
|
-
process.stderr.write(` ${chalk.dim(`[${stepNum}/${total}]`)} ${chalk.bold.cyan(op)}${preview}\n`);
|
|
48
|
-
}
|
|
49
|
-
function debugStepResult(op, data) {
|
|
50
|
-
if (data === null || data === undefined) {
|
|
51
|
-
process.stderr.write(` ${chalk.dim('→ (no data)')}\n`);
|
|
52
|
-
}
|
|
53
|
-
else if (Array.isArray(data)) {
|
|
54
|
-
process.stderr.write(` ${chalk.dim(`→ ${data.length} items`)}\n`);
|
|
55
|
-
}
|
|
56
|
-
else if (typeof data === 'object') {
|
|
57
|
-
const keys = Object.keys(data).slice(0, 5);
|
|
58
|
-
process.stderr.write(` ${chalk.dim(`→ dict (${keys.join(', ')}${Object.keys(data).length > 5 ? '...' : ''})`)}\n`);
|
|
59
|
-
}
|
|
60
|
-
else if (typeof data === 'string') {
|
|
61
|
-
const p = data.slice(0, 60).replace(/\n/g, '\\n');
|
|
62
|
-
process.stderr.write(` ${chalk.dim(`→ "${p}${data.length > 60 ? '...' : ''}"`)}\n`);
|
|
63
|
-
}
|
|
64
|
-
else {
|
|
65
|
-
process.stderr.write(` ${chalk.dim(`→ ${typeof data}`)}\n`);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
// Single URL fetch helper
|
|
69
|
-
async function fetchSingle(page, url, method, queryParams, headers, args, data) {
|
|
70
|
-
const renderedParams = {};
|
|
71
|
-
for (const [k, v] of Object.entries(queryParams))
|
|
72
|
-
renderedParams[k] = String(render(v, { args, data }));
|
|
73
|
-
const renderedHeaders = {};
|
|
74
|
-
for (const [k, v] of Object.entries(headers))
|
|
75
|
-
renderedHeaders[k] = String(render(v, { args, data }));
|
|
76
|
-
let finalUrl = url;
|
|
77
|
-
if (Object.keys(renderedParams).length > 0) {
|
|
78
|
-
const qs = new URLSearchParams(renderedParams).toString();
|
|
79
|
-
finalUrl = `${finalUrl}${finalUrl.includes('?') ? '&' : '?'}${qs}`;
|
|
80
|
-
}
|
|
81
|
-
if (page === null) {
|
|
82
|
-
const resp = await fetch(finalUrl, { method: method.toUpperCase(), headers: renderedHeaders });
|
|
83
|
-
return resp.json();
|
|
84
|
-
}
|
|
85
|
-
const headersJs = JSON.stringify(renderedHeaders);
|
|
86
|
-
const escapedUrl = finalUrl.replace(/"/g, '\\"');
|
|
87
|
-
return page.evaluate(`
|
|
88
|
-
async () => {
|
|
89
|
-
const resp = await fetch("${escapedUrl}", {
|
|
90
|
-
method: "${method}", headers: ${headersJs}, credentials: "include"
|
|
91
|
-
});
|
|
92
|
-
return await resp.json();
|
|
93
|
-
}
|
|
94
|
-
`);
|
|
95
|
-
}
|
|
96
|
-
async function executeStep(page, op, params, data, args) {
|
|
97
|
-
switch (op) {
|
|
98
|
-
case 'navigate': {
|
|
99
|
-
const url = render(params, { args, data });
|
|
100
|
-
await page.goto(String(url));
|
|
101
|
-
return data;
|
|
102
|
-
}
|
|
103
|
-
case 'fetch': {
|
|
104
|
-
const urlOrObj = typeof params === 'string' ? params : (params?.url ?? '');
|
|
105
|
-
const method = params?.method ?? 'GET';
|
|
106
|
-
const queryParams = params?.params ?? {};
|
|
107
|
-
const headers = params?.headers ?? {};
|
|
108
|
-
const urlTemplate = String(urlOrObj);
|
|
109
|
-
// Per-item fetch when data is array and URL references item
|
|
110
|
-
if (Array.isArray(data) && urlTemplate.includes('item')) {
|
|
111
|
-
const results = [];
|
|
112
|
-
for (let i = 0; i < data.length; i++) {
|
|
113
|
-
const itemUrl = String(render(urlTemplate, { args, data, item: data[i], index: i }));
|
|
114
|
-
results.push(await fetchSingle(page, itemUrl, method, queryParams, headers, args, data));
|
|
115
|
-
}
|
|
116
|
-
return results;
|
|
117
|
-
}
|
|
118
|
-
const url = render(urlOrObj, { args, data });
|
|
119
|
-
return fetchSingle(page, String(url), method, queryParams, headers, args, data);
|
|
120
|
-
}
|
|
121
|
-
case 'select': {
|
|
122
|
-
const pathStr = String(render(params, { args, data }));
|
|
123
|
-
if (data && typeof data === 'object') {
|
|
124
|
-
let current = data;
|
|
125
|
-
for (const part of pathStr.split('.')) {
|
|
126
|
-
if (current && typeof current === 'object' && !Array.isArray(current))
|
|
127
|
-
current = current[part];
|
|
128
|
-
else if (Array.isArray(current) && /^\d+$/.test(part))
|
|
129
|
-
current = current[parseInt(part, 10)];
|
|
130
|
-
else
|
|
131
|
-
return null;
|
|
132
|
-
}
|
|
133
|
-
return current;
|
|
134
|
-
}
|
|
135
|
-
return data;
|
|
136
|
-
}
|
|
137
|
-
case 'evaluate': {
|
|
138
|
-
const js = String(render(params, { args, data }));
|
|
139
|
-
return page.evaluate(normalizeEvaluateSource(js));
|
|
140
|
-
}
|
|
141
|
-
case 'snapshot': {
|
|
142
|
-
const opts = (typeof params === 'object' && params) ? params : {};
|
|
143
|
-
return page.snapshot({ interactive: opts.interactive ?? false, compact: opts.compact ?? false, maxDepth: opts.max_depth, raw: opts.raw ?? false });
|
|
144
|
-
}
|
|
145
|
-
case 'click': {
|
|
146
|
-
await page.click(String(render(params, { args, data })).replace(/^@/, ''));
|
|
147
|
-
return data;
|
|
148
|
-
}
|
|
149
|
-
case 'type': {
|
|
150
|
-
if (typeof params === 'object' && params) {
|
|
151
|
-
const ref = String(render(params.ref ?? '', { args, data })).replace(/^@/, '');
|
|
152
|
-
const text = String(render(params.text ?? '', { args, data }));
|
|
153
|
-
await page.typeText(ref, text);
|
|
154
|
-
if (params.submit)
|
|
155
|
-
await page.pressKey('Enter');
|
|
156
|
-
}
|
|
157
|
-
return data;
|
|
158
|
-
}
|
|
159
|
-
case 'wait': {
|
|
160
|
-
if (typeof params === 'number')
|
|
161
|
-
await page.wait(params);
|
|
162
|
-
else if (typeof params === 'object' && params) {
|
|
163
|
-
if ('text' in params) {
|
|
164
|
-
const timeout = params.timeout ?? 10;
|
|
165
|
-
const start = Date.now();
|
|
166
|
-
while ((Date.now() - start) / 1000 < timeout) {
|
|
167
|
-
const snap = await page.snapshot({ raw: true });
|
|
168
|
-
if (typeof snap === 'string' && snap.includes(params.text))
|
|
169
|
-
break;
|
|
170
|
-
await page.wait(0.5);
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
else if ('time' in params)
|
|
174
|
-
await page.wait(Number(params.time));
|
|
175
|
-
}
|
|
176
|
-
else if (typeof params === 'string')
|
|
177
|
-
await page.wait(Number(render(params, { args, data })));
|
|
178
|
-
return data;
|
|
179
|
-
}
|
|
180
|
-
case 'press': {
|
|
181
|
-
await page.pressKey(String(render(params, { args, data })));
|
|
182
|
-
return data;
|
|
183
|
-
}
|
|
184
|
-
case 'map': {
|
|
185
|
-
if (!data || typeof data !== 'object')
|
|
186
|
-
return data;
|
|
187
|
-
let items = Array.isArray(data) ? data : [data];
|
|
188
|
-
if (!Array.isArray(data) && typeof data === 'object' && 'data' in data)
|
|
189
|
-
items = data.data;
|
|
190
|
-
const result = [];
|
|
191
|
-
for (let i = 0; i < items.length; i++) {
|
|
192
|
-
const item = items[i];
|
|
193
|
-
const row = {};
|
|
194
|
-
for (const [key, template] of Object.entries(params))
|
|
195
|
-
row[key] = render(template, { args, data, item, index: i });
|
|
196
|
-
result.push(row);
|
|
197
|
-
}
|
|
198
|
-
return result;
|
|
199
|
-
}
|
|
200
|
-
case 'filter': {
|
|
201
|
-
if (!Array.isArray(data))
|
|
202
|
-
return data;
|
|
203
|
-
return data.filter((item, i) => evalExpr(String(params), { args, item, index: i }));
|
|
204
|
-
}
|
|
205
|
-
case 'sort': {
|
|
206
|
-
if (!Array.isArray(data))
|
|
207
|
-
return data;
|
|
208
|
-
const key = typeof params === 'object' ? (params.by ?? '') : String(params);
|
|
209
|
-
const reverse = typeof params === 'object' ? params.order === 'desc' : false;
|
|
210
|
-
return [...data].sort((a, b) => { const va = a[key] ?? ''; const vb = b[key] ?? ''; const cmp = va < vb ? -1 : va > vb ? 1 : 0; return reverse ? -cmp : cmp; });
|
|
211
|
-
}
|
|
212
|
-
case 'limit': {
|
|
213
|
-
if (!Array.isArray(data))
|
|
214
|
-
return data;
|
|
215
|
-
return data.slice(0, Number(render(params, { args, data })));
|
|
216
|
-
}
|
|
217
|
-
case 'intercept': return data;
|
|
218
|
-
default: return data;
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
function render(template, ctx) {
|
|
222
|
-
if (typeof template !== 'string')
|
|
223
|
-
return template;
|
|
224
|
-
const fullMatch = template.match(/^\$\{\{\s*(.*?)\s*\}\}$/);
|
|
225
|
-
if (fullMatch)
|
|
226
|
-
return evalExpr(fullMatch[1].trim(), ctx);
|
|
227
|
-
return template.replace(/\$\{\{\s*(.*?)\s*\}\}/g, (_m, expr) => String(evalExpr(expr.trim(), ctx)));
|
|
228
|
-
}
|
|
229
|
-
function evalExpr(expr, ctx) {
|
|
230
|
-
const args = ctx.args ?? {};
|
|
231
|
-
const item = ctx.item ?? {};
|
|
232
|
-
const data = ctx.data;
|
|
233
|
-
const index = ctx.index ?? 0;
|
|
234
|
-
// Default filter: args.limit | default(20)
|
|
235
|
-
if (expr.includes('|') && expr.includes('default(')) {
|
|
236
|
-
const [mainExpr, rest] = expr.split('|', 2);
|
|
237
|
-
const defaultMatch = rest.match(/default\((.+?)\)/);
|
|
238
|
-
const defaultVal = defaultMatch ? defaultMatch[1] : null;
|
|
239
|
-
const result = resolvePath(mainExpr.trim(), { args, item, data, index });
|
|
240
|
-
if (result === null || result === undefined) {
|
|
241
|
-
if (defaultVal !== null) {
|
|
242
|
-
const intVal = parseInt(defaultVal, 10);
|
|
243
|
-
if (!isNaN(intVal) && String(intVal) === defaultVal.trim())
|
|
244
|
-
return intVal;
|
|
245
|
-
return defaultVal.replace(/^['"]|['"]$/g, '');
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
return result;
|
|
249
|
-
}
|
|
250
|
-
// Arithmetic: index + 1
|
|
251
|
-
const arithMatch = expr.match(/^([\w][\w.]*)\s*([+\-*/])\s*(\d+)$/);
|
|
252
|
-
if (arithMatch) {
|
|
253
|
-
const [, varName, op, numStr] = arithMatch;
|
|
254
|
-
const val = resolvePath(varName, { args, item, data, index });
|
|
255
|
-
if (val !== null && val !== undefined) {
|
|
256
|
-
const numVal = Number(val);
|
|
257
|
-
const num = Number(numStr);
|
|
258
|
-
if (!isNaN(numVal)) {
|
|
259
|
-
switch (op) {
|
|
260
|
-
case '+': return numVal + num;
|
|
261
|
-
case '-': return numVal - num;
|
|
262
|
-
case '*': return numVal * num;
|
|
263
|
-
case '/': return num !== 0 ? numVal / num : 0;
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
// JS-like fallback expression: item.tweetCount || 'N/A'
|
|
269
|
-
const orMatch = expr.match(/^(.+?)\s*\|\|\s*(.+)$/);
|
|
270
|
-
if (orMatch) {
|
|
271
|
-
const left = evalExpr(orMatch[1].trim(), ctx);
|
|
272
|
-
if (left)
|
|
273
|
-
return left;
|
|
274
|
-
const right = orMatch[2].trim();
|
|
275
|
-
return right.replace(/^['"]|['"]$/g, '');
|
|
276
|
-
}
|
|
277
|
-
return resolvePath(expr, { args, item, data, index });
|
|
278
|
-
}
|
|
279
|
-
function resolvePath(pathStr, ctx) {
|
|
280
|
-
const args = ctx.args ?? {};
|
|
281
|
-
const item = ctx.item ?? {};
|
|
282
|
-
const data = ctx.data;
|
|
283
|
-
const index = ctx.index ?? 0;
|
|
284
|
-
const parts = pathStr.split('.');
|
|
285
|
-
const rootName = parts[0];
|
|
286
|
-
let obj;
|
|
287
|
-
let rest;
|
|
288
|
-
if (rootName === 'args') {
|
|
289
|
-
obj = args;
|
|
290
|
-
rest = parts.slice(1);
|
|
291
|
-
}
|
|
292
|
-
else if (rootName === 'item') {
|
|
293
|
-
obj = item;
|
|
294
|
-
rest = parts.slice(1);
|
|
295
|
-
}
|
|
296
|
-
else if (rootName === 'data') {
|
|
297
|
-
obj = data;
|
|
298
|
-
rest = parts.slice(1);
|
|
299
|
-
}
|
|
300
|
-
else if (rootName === 'index')
|
|
301
|
-
return index;
|
|
302
|
-
else {
|
|
303
|
-
obj = item;
|
|
304
|
-
rest = parts;
|
|
305
|
-
}
|
|
306
|
-
for (const part of rest) {
|
|
307
|
-
if (obj && typeof obj === 'object' && !Array.isArray(obj))
|
|
308
|
-
obj = obj[part];
|
|
309
|
-
else if (Array.isArray(obj) && /^\d+$/.test(part))
|
|
310
|
-
obj = obj[parseInt(part, 10)];
|
|
311
|
-
else
|
|
312
|
-
return null;
|
|
313
|
-
}
|
|
314
|
-
return obj;
|
|
315
|
-
}
|
|
7
|
+
export { executePipeline } from './pipeline/index.js';
|
package/dist/registry.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Core registry: Strategy enum, Arg/CliCommand interfaces, cli() registration.
|
|
3
3
|
*/
|
|
4
|
+
import type { IPage } from './types.js';
|
|
4
5
|
export declare enum Strategy {
|
|
5
6
|
PUBLIC = "public",
|
|
6
7
|
COOKIE = "cookie",
|
|
@@ -25,7 +26,7 @@ export interface CliCommand {
|
|
|
25
26
|
browser?: boolean;
|
|
26
27
|
args: Arg[];
|
|
27
28
|
columns?: string[];
|
|
28
|
-
func?: (page:
|
|
29
|
+
func?: (page: IPage | null, kwargs: Record<string, any>, debug?: boolean) => Promise<any>;
|
|
29
30
|
pipeline?: any[];
|
|
30
31
|
timeoutSeconds?: number;
|
|
31
32
|
source?: string;
|
|
@@ -39,7 +40,7 @@ export interface CliOptions {
|
|
|
39
40
|
browser?: boolean;
|
|
40
41
|
args?: Arg[];
|
|
41
42
|
columns?: string[];
|
|
42
|
-
func?: (page:
|
|
43
|
+
func?: (page: IPage | null, kwargs: Record<string, any>, debug?: boolean) => Promise<any>;
|
|
43
44
|
pipeline?: any[];
|
|
44
45
|
timeoutSeconds?: number;
|
|
45
46
|
}
|