@jackwener/opencli 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/README.md +9 -2
  2. package/README.zh-CN.md +9 -1
  3. package/SKILL.md +24 -0
  4. package/dist/bilibili.d.ts +6 -5
  5. package/dist/browser.d.ts +2 -1
  6. package/dist/browser.js +9 -1
  7. package/dist/cascade.d.ts +3 -2
  8. package/dist/clis/bbc/news.js +42 -0
  9. package/dist/clis/boss/search.d.ts +1 -0
  10. package/dist/clis/boss/search.js +47 -0
  11. package/dist/clis/ctrip/search.d.ts +1 -0
  12. package/dist/clis/ctrip/search.js +62 -0
  13. package/dist/clis/index.d.ts +8 -0
  14. package/dist/clis/index.js +16 -0
  15. package/dist/clis/reuters/search.d.ts +1 -0
  16. package/dist/clis/reuters/search.js +52 -0
  17. package/dist/clis/smzdm/search.d.ts +1 -0
  18. package/dist/clis/smzdm/search.js +66 -0
  19. package/dist/clis/weibo/hot.d.ts +1 -0
  20. package/dist/clis/weibo/hot.js +41 -0
  21. package/dist/clis/yahoo-finance/quote.d.ts +1 -0
  22. package/dist/clis/yahoo-finance/quote.js +74 -0
  23. package/dist/clis/youtube/search.d.ts +1 -0
  24. package/dist/clis/youtube/search.js +60 -0
  25. package/dist/engine.d.ts +2 -1
  26. package/dist/explore.js +1 -1
  27. package/dist/generate.js +2 -1
  28. package/dist/main.js +6 -4
  29. package/dist/pipeline/executor.d.ts +9 -0
  30. package/dist/pipeline/executor.js +88 -0
  31. package/dist/pipeline/index.d.ts +5 -0
  32. package/dist/pipeline/index.js +5 -0
  33. package/dist/pipeline/steps/browser.d.ts +12 -0
  34. package/dist/pipeline/steps/browser.js +68 -0
  35. package/dist/pipeline/steps/fetch.d.ts +5 -0
  36. package/dist/pipeline/steps/fetch.js +50 -0
  37. package/dist/pipeline/steps/intercept.d.ts +5 -0
  38. package/dist/pipeline/steps/intercept.js +75 -0
  39. package/dist/pipeline/steps/tap.d.ts +12 -0
  40. package/dist/pipeline/steps/tap.js +130 -0
  41. package/dist/pipeline/steps/transform.d.ts +8 -0
  42. package/dist/pipeline/steps/transform.js +53 -0
  43. package/dist/pipeline/template.d.ts +16 -0
  44. package/dist/pipeline/template.js +115 -0
  45. package/dist/pipeline/template.test.d.ts +4 -0
  46. package/dist/pipeline/template.test.js +102 -0
  47. package/dist/pipeline/transform.test.d.ts +4 -0
  48. package/dist/pipeline/transform.test.js +90 -0
  49. package/dist/pipeline.d.ts +5 -7
  50. package/dist/pipeline.js +5 -549
  51. package/dist/registry.d.ts +3 -2
  52. package/dist/runtime.d.ts +2 -1
  53. package/dist/types.d.ts +27 -0
  54. package/dist/types.js +7 -0
  55. package/package.json +6 -3
  56. package/src/bilibili.ts +9 -7
  57. package/src/browser.ts +8 -2
  58. package/src/cascade.ts +3 -2
  59. package/src/clis/bbc/news.ts +42 -0
  60. package/src/clis/boss/search.ts +47 -0
  61. package/src/clis/ctrip/search.ts +62 -0
  62. package/src/clis/index.ts +24 -0
  63. package/src/clis/reuters/search.ts +52 -0
  64. package/src/clis/smzdm/search.ts +66 -0
  65. package/src/clis/weibo/hot.ts +41 -0
  66. package/src/clis/yahoo-finance/quote.ts +74 -0
  67. package/src/clis/youtube/search.ts +60 -0
  68. package/src/engine.ts +2 -1
  69. package/src/explore.ts +1 -1
  70. package/src/generate.ts +3 -1
  71. package/src/main.ts +7 -5
  72. package/src/pipeline/executor.ts +98 -0
  73. package/src/pipeline/index.ts +6 -0
  74. package/src/pipeline/steps/browser.ts +67 -0
  75. package/src/pipeline/steps/fetch.ts +60 -0
  76. package/src/pipeline/steps/intercept.ts +78 -0
  77. package/src/pipeline/steps/tap.ts +137 -0
  78. package/src/pipeline/steps/transform.ts +50 -0
  79. package/src/pipeline/template.test.ts +107 -0
  80. package/src/pipeline/template.ts +101 -0
  81. package/src/pipeline/transform.test.ts +107 -0
  82. package/src/pipeline.ts +5 -529
  83. package/src/registry.ts +4 -2
  84. package/src/runtime.ts +3 -1
  85. package/src/types.ts +23 -0
  86. package/vitest.config.ts +7 -0
  87. package/dist/clis/github/search.js +0 -20
  88. package/dist/clis/github/trending.yaml +0 -58
  89. package/dist/promote.d.ts +0 -1
  90. package/dist/promote.js +0 -3
  91. package/dist/register.d.ts +0 -2
  92. package/dist/register.js +0 -2
  93. package/dist/scaffold.d.ts +0 -2
  94. package/dist/scaffold.js +0 -2
  95. package/dist/smoke.d.ts +0 -2
  96. package/dist/smoke.js +0 -2
  97. package/src/clis/github/search.ts +0 -21
  98. package/src/clis/github/trending.yaml +0 -58
  99. package/src/promote.ts +0 -3
  100. package/src/register.ts +0 -2
  101. package/src/scaffold.ts +0 -2
  102. package/src/smoke.ts +0 -2
  103. /package/dist/clis/{github/search.d.ts → bbc/news.d.ts} +0 -0
@@ -0,0 +1,60 @@
1
+ /**
2
+ * YouTube search — innertube API via browser session.
3
+ * Source: bb-sites/youtube/search.js
4
+ */
5
+ import { cli, Strategy } from '../../registry.js';
6
+ cli({
7
+ site: 'youtube',
8
+ name: 'search',
9
+ description: 'Search YouTube videos',
10
+ domain: 'www.youtube.com',
11
+ strategy: Strategy.COOKIE,
12
+ args: [
13
+ { name: 'query', required: true, help: 'Search query' },
14
+ { name: 'limit', type: 'int', default: 20, help: 'Max results (max 50)' },
15
+ ],
16
+ columns: ['rank', 'title', 'channel', 'views', 'duration', 'url'],
17
+ func: async (page, kwargs) => {
18
+ const limit = Math.min(kwargs.limit || 20, 50);
19
+ await page.goto('https://www.youtube.com');
20
+ await page.wait(2);
21
+ const data = await page.evaluate(`
22
+ (async () => {
23
+ const cfg = window.ytcfg?.data_ || {};
24
+ const apiKey = cfg.INNERTUBE_API_KEY;
25
+ const context = cfg.INNERTUBE_CONTEXT;
26
+ if (!apiKey || !context) return {error: 'YouTube config not found'};
27
+
28
+ const resp = await fetch('/youtubei/v1/search?key=' + apiKey + '&prettyPrint=false', {
29
+ method: 'POST', credentials: 'include',
30
+ headers: {'Content-Type': 'application/json'},
31
+ body: JSON.stringify({context, query: '${kwargs.query.replace(/'/g, "\\'")}'})
32
+ });
33
+ if (!resp.ok) return {error: 'HTTP ' + resp.status};
34
+
35
+ const data = await resp.json();
36
+ const contents = data.contents?.twoColumnSearchResultsRenderer?.primaryContents?.sectionListRenderer?.contents || [];
37
+ const videos = [];
38
+ for (const section of contents) {
39
+ for (const item of (section.itemSectionRenderer?.contents || [])) {
40
+ if (item.videoRenderer && videos.length < ${limit}) {
41
+ const v = item.videoRenderer;
42
+ videos.push({
43
+ rank: videos.length + 1,
44
+ title: v.title?.runs?.[0]?.text || '',
45
+ channel: v.ownerText?.runs?.[0]?.text || '',
46
+ views: v.viewCountText?.simpleText || v.shortViewCountText?.simpleText || '',
47
+ duration: v.lengthText?.simpleText || 'LIVE',
48
+ url: 'https://www.youtube.com/watch?v=' + v.videoId
49
+ });
50
+ }
51
+ }
52
+ }
53
+ return videos;
54
+ })()
55
+ `);
56
+ if (!Array.isArray(data))
57
+ return [];
58
+ return data;
59
+ },
60
+ });
package/dist/engine.d.ts CHANGED
@@ -2,5 +2,6 @@
2
2
  * CLI discovery: finds YAML/TS CLI definitions and registers them.
3
3
  */
4
4
  import { type CliCommand } from './registry.js';
5
+ import type { IPage } from './types.js';
5
6
  export declare function discoverClis(...dirs: string[]): void;
6
- export declare function executeCommand(cmd: CliCommand, page: any, kwargs: Record<string, any>, debug?: boolean): Promise<any>;
7
+ export declare function executeCommand(cmd: CliCommand, page: IPage | null, kwargs: Record<string, any>, debug?: boolean): Promise<any>;
package/dist/explore.js CHANGED
@@ -463,7 +463,7 @@ export function renderExploreSummary(result) {
463
463
  lines.push(`Output: ${result.out_dir}`);
464
464
  return lines.join('\n');
465
465
  }
466
- async function readPageMetadata(page) {
466
+ async function readPageMetadata(page /* IPage */) {
467
467
  try {
468
468
  const result = await page.evaluate(`() => ({ url: window.location.href, title: document.title || '' })`);
469
469
  if (result && typeof result === 'object')
package/dist/generate.js CHANGED
@@ -9,7 +9,8 @@
9
9
  */
10
10
  import { exploreUrl } from './explore.js';
11
11
  import { synthesizeFromExplore } from './synthesize.js';
12
- import { registerCandidates } from './register.js';
12
+ // TODO: implement real CLI registration (copy candidate YAML to user clis dir)
13
+ function registerCandidates(_opts) { return { ok: true, count: 0 }; }
13
14
  const CAPABILITY_ALIASES = {
14
15
  search: ['search', '搜索', '查找', 'query', 'keyword'],
15
16
  hot: ['hot', '热门', '热榜', '热搜', 'popular', 'top', 'ranking'],
package/dist/main.js CHANGED
@@ -2,6 +2,7 @@
2
2
  /**
3
3
  * opencli — Make any website your CLI. AI-powered.
4
4
  */
5
+ import * as fs from 'node:fs';
5
6
  import * as os from 'node:os';
6
7
  import * as path from 'node:path';
7
8
  import { fileURLToPath } from 'node:url';
@@ -17,9 +18,12 @@ const __filename = fileURLToPath(import.meta.url);
17
18
  const __dirname = path.dirname(__filename);
18
19
  const BUILTIN_CLIS = path.resolve(__dirname, 'clis');
19
20
  const USER_CLIS = path.join(os.homedir(), '.opencli', 'clis');
21
+ // Read version from package.json (single source of truth)
22
+ const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
23
+ const PKG_VERSION = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version ?? '0.0.0';
20
24
  discoverClis(BUILTIN_CLIS, USER_CLIS);
21
25
  const program = new Command();
22
- program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version('0.1.0');
26
+ program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
23
27
  // ── Built-in commands ──────────────────────────────────────────────────────
24
28
  program.command('list').description('List all available CLI commands').option('--json', 'JSON output')
25
29
  .action((opts) => {
@@ -53,9 +57,7 @@ program.command('validate').description('Validate CLI definitions').argument('[t
53
57
  .action(async (target) => { const { validateClisWithTarget, renderValidationReport } = await import('./validate.js'); console.log(renderValidationReport(validateClisWithTarget([BUILTIN_CLIS, USER_CLIS], target))); });
54
58
  program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false)
55
59
  .action(async (target, opts) => { const { verifyClis, renderVerifyReport } = await import('./verify.js'); const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke }); console.log(renderVerifyReport(r)); process.exitCode = r.ok ? 0 : 1; });
56
- program.command('explore').description('Explore a website').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3')
57
- .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait) }))); });
58
- program.command('probe').description('Probe a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3')
60
+ program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3')
59
61
  .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait) }))); });
60
62
  program.command('synthesize').description('Synthesize CLIs from explore').argument('<target>').option('--top <n>', '', '3')
61
63
  .action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); });
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Pipeline executor: runs YAML pipeline steps sequentially.
3
+ */
4
+ import type { IPage } from '../types.js';
5
+ export interface PipelineContext {
6
+ args?: Record<string, any>;
7
+ debug?: boolean;
8
+ }
9
+ export declare function executePipeline(page: IPage | null, pipeline: any[], ctx?: PipelineContext): Promise<any>;
@@ -0,0 +1,88 @@
1
+ /**
2
+ * Pipeline executor: runs YAML pipeline steps sequentially.
3
+ */
4
+ import chalk from 'chalk';
5
+ import { stepNavigate, stepClick, stepType, stepWait, stepPress, stepSnapshot, stepEvaluate } from './steps/browser.js';
6
+ import { stepFetch } from './steps/fetch.js';
7
+ import { stepSelect, stepMap, stepFilter, stepSort, stepLimit } from './steps/transform.js';
8
+ import { stepIntercept } from './steps/intercept.js';
9
+ import { stepTap } from './steps/tap.js';
10
+ /** Registry of all available step handlers */
11
+ const STEP_HANDLERS = {
12
+ navigate: stepNavigate,
13
+ fetch: stepFetch,
14
+ select: stepSelect,
15
+ evaluate: stepEvaluate,
16
+ snapshot: stepSnapshot,
17
+ click: stepClick,
18
+ type: stepType,
19
+ wait: stepWait,
20
+ press: stepPress,
21
+ map: stepMap,
22
+ filter: stepFilter,
23
+ sort: stepSort,
24
+ limit: stepLimit,
25
+ intercept: stepIntercept,
26
+ tap: stepTap,
27
+ };
28
+ export async function executePipeline(page, pipeline, ctx = {}) {
29
+ const args = ctx.args ?? {};
30
+ const debug = ctx.debug ?? false;
31
+ let data = null;
32
+ const total = pipeline.length;
33
+ for (let i = 0; i < pipeline.length; i++) {
34
+ const step = pipeline[i];
35
+ if (!step || typeof step !== 'object')
36
+ continue;
37
+ for (const [op, params] of Object.entries(step)) {
38
+ if (debug)
39
+ debugStepStart(i + 1, total, op, params);
40
+ const handler = STEP_HANDLERS[op];
41
+ if (handler) {
42
+ data = await handler(page, params, data, args);
43
+ }
44
+ else {
45
+ if (debug)
46
+ process.stderr.write(` ${chalk.yellow('⚠')} Unknown step: ${op}\n`);
47
+ }
48
+ // Detect error objects returned by steps (e.g. tap store not found)
49
+ if (data && typeof data === 'object' && !Array.isArray(data) && data.error) {
50
+ process.stderr.write(` ${chalk.yellow('⚠')} ${chalk.yellow(op)}: ${data.error}\n`);
51
+ if (data.hint)
52
+ process.stderr.write(` ${chalk.dim('💡')} ${chalk.dim(data.hint)}\n`);
53
+ }
54
+ if (debug)
55
+ debugStepResult(op, data);
56
+ }
57
+ }
58
+ return data;
59
+ }
60
+ function debugStepStart(stepNum, total, op, params) {
61
+ let preview = '';
62
+ if (typeof params === 'string') {
63
+ preview = params.length <= 80 ? ` → ${params}` : ` → ${params.slice(0, 77)}...`;
64
+ }
65
+ else if (params && typeof params === 'object' && !Array.isArray(params)) {
66
+ preview = ` (${Object.keys(params).join(', ')})`;
67
+ }
68
+ process.stderr.write(` ${chalk.dim(`[${stepNum}/${total}]`)} ${chalk.bold.cyan(op)}${preview}\n`);
69
+ }
70
+ function debugStepResult(op, data) {
71
+ if (data === null || data === undefined) {
72
+ process.stderr.write(` ${chalk.dim('→ (no data)')}\n`);
73
+ }
74
+ else if (Array.isArray(data)) {
75
+ process.stderr.write(` ${chalk.dim(`→ ${data.length} items`)}\n`);
76
+ }
77
+ else if (typeof data === 'object') {
78
+ const keys = Object.keys(data).slice(0, 5);
79
+ process.stderr.write(` ${chalk.dim(`→ dict (${keys.join(', ')}${Object.keys(data).length > 5 ? '...' : ''})`)}\n`);
80
+ }
81
+ else if (typeof data === 'string') {
82
+ const p = data.slice(0, 60).replace(/\n/g, '\\n');
83
+ process.stderr.write(` ${chalk.dim(`→ "${p}${data.length > 60 ? '...' : ''}"`)}\n`);
84
+ }
85
+ else {
86
+ process.stderr.write(` ${chalk.dim(`→ ${typeof data}`)}\n`);
87
+ }
88
+ }
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Pipeline module: re-exports for backward compatibility.
3
+ */
4
+ export { executePipeline, type PipelineContext } from './executor.js';
5
+ export { render, evalExpr, resolvePath, normalizeEvaluateSource, type RenderContext } from './template.js';
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Pipeline module: re-exports for backward compatibility.
3
+ */
4
+ export { executePipeline } from './executor.js';
5
+ export { render, evalExpr, resolvePath, normalizeEvaluateSource } from './template.js';
@@ -0,0 +1,12 @@
1
+ /**
2
+ * Pipeline step: navigate, click, type, wait, press, snapshot.
3
+ * Browser interaction primitives.
4
+ */
5
+ import type { IPage } from '../../types.js';
6
+ export declare function stepNavigate(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;
7
+ export declare function stepClick(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;
8
+ export declare function stepType(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;
9
+ export declare function stepWait(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;
10
+ export declare function stepPress(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;
11
+ export declare function stepSnapshot(page: IPage, params: any, _data: any, _args: Record<string, any>): Promise<any>;
12
+ export declare function stepEvaluate(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;
@@ -0,0 +1,68 @@
1
+ /**
2
+ * Pipeline step: navigate, click, type, wait, press, snapshot.
3
+ * Browser interaction primitives.
4
+ */
5
+ import { render, normalizeEvaluateSource } from '../template.js';
6
+ export async function stepNavigate(page, params, data, args) {
7
+ const url = render(params, { args, data });
8
+ await page.goto(String(url));
9
+ return data;
10
+ }
11
+ export async function stepClick(page, params, data, args) {
12
+ await page.click(String(render(params, { args, data })).replace(/^@/, ''));
13
+ return data;
14
+ }
15
+ export async function stepType(page, params, data, args) {
16
+ if (typeof params === 'object' && params) {
17
+ const ref = String(render(params.ref ?? '', { args, data })).replace(/^@/, '');
18
+ const text = String(render(params.text ?? '', { args, data }));
19
+ await page.typeText(ref, text);
20
+ if (params.submit)
21
+ await page.pressKey('Enter');
22
+ }
23
+ return data;
24
+ }
25
+ export async function stepWait(page, params, data, args) {
26
+ if (typeof params === 'number')
27
+ await page.wait(params);
28
+ else if (typeof params === 'object' && params) {
29
+ if ('text' in params) {
30
+ const timeout = params.timeout ?? 10;
31
+ const start = Date.now();
32
+ while ((Date.now() - start) / 1000 < timeout) {
33
+ const snap = await page.snapshot({ raw: true });
34
+ if (typeof snap === 'string' && snap.includes(params.text))
35
+ break;
36
+ await page.wait(0.5);
37
+ }
38
+ }
39
+ else if ('time' in params)
40
+ await page.wait(Number(params.time));
41
+ }
42
+ else if (typeof params === 'string')
43
+ await page.wait(Number(render(params, { args, data })));
44
+ return data;
45
+ }
46
+ export async function stepPress(page, params, data, args) {
47
+ await page.pressKey(String(render(params, { args, data })));
48
+ return data;
49
+ }
50
+ export async function stepSnapshot(page, params, _data, _args) {
51
+ const opts = (typeof params === 'object' && params) ? params : {};
52
+ return page.snapshot({ interactive: opts.interactive ?? false, compact: opts.compact ?? false, maxDepth: opts.max_depth, raw: opts.raw ?? false });
53
+ }
54
+ export async function stepEvaluate(page, params, data, args) {
55
+ const js = String(render(params, { args, data }));
56
+ let result = await page.evaluate(normalizeEvaluateSource(js));
57
+ // MCP may return JSON as a string — auto-parse it
58
+ if (typeof result === 'string') {
59
+ const trimmed = result.trim();
60
+ if ((trimmed.startsWith('[') && trimmed.endsWith(']')) || (trimmed.startsWith('{') && trimmed.endsWith('}'))) {
61
+ try {
62
+ result = JSON.parse(trimmed);
63
+ }
64
+ catch { }
65
+ }
66
+ }
67
+ return result;
68
+ }
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Pipeline step: fetch — HTTP API requests.
3
+ */
4
+ import type { IPage } from '../../types.js';
5
+ export declare function stepFetch(page: IPage | null, params: any, data: any, args: Record<string, any>): Promise<any>;
@@ -0,0 +1,50 @@
1
+ /**
2
+ * Pipeline step: fetch — HTTP API requests.
3
+ */
4
+ import { render } from '../template.js';
5
+ /** Single URL fetch helper */
6
+ async function fetchSingle(page, url, method, queryParams, headers, args, data) {
7
+ const renderedParams = {};
8
+ for (const [k, v] of Object.entries(queryParams))
9
+ renderedParams[k] = String(render(v, { args, data }));
10
+ const renderedHeaders = {};
11
+ for (const [k, v] of Object.entries(headers))
12
+ renderedHeaders[k] = String(render(v, { args, data }));
13
+ let finalUrl = url;
14
+ if (Object.keys(renderedParams).length > 0) {
15
+ const qs = new URLSearchParams(renderedParams).toString();
16
+ finalUrl = `${finalUrl}${finalUrl.includes('?') ? '&' : '?'}${qs}`;
17
+ }
18
+ if (page === null) {
19
+ const resp = await fetch(finalUrl, { method: method.toUpperCase(), headers: renderedHeaders });
20
+ return resp.json();
21
+ }
22
+ const headersJs = JSON.stringify(renderedHeaders);
23
+ const escapedUrl = finalUrl.replace(/"/g, '\\"');
24
+ return page.evaluate(`
25
+ async () => {
26
+ const resp = await fetch("${escapedUrl}", {
27
+ method: "${method}", headers: ${headersJs}, credentials: "include"
28
+ });
29
+ return await resp.json();
30
+ }
31
+ `);
32
+ }
33
+ export async function stepFetch(page, params, data, args) {
34
+ const urlOrObj = typeof params === 'string' ? params : (params?.url ?? '');
35
+ const method = params?.method ?? 'GET';
36
+ const queryParams = params?.params ?? {};
37
+ const headers = params?.headers ?? {};
38
+ const urlTemplate = String(urlOrObj);
39
+ // Per-item fetch when data is array and URL references item
40
+ if (Array.isArray(data) && urlTemplate.includes('item')) {
41
+ const results = [];
42
+ for (let i = 0; i < data.length; i++) {
43
+ const itemUrl = String(render(urlTemplate, { args, data, item: data[i], index: i }));
44
+ results.push(await fetchSingle(page, itemUrl, method, queryParams, headers, args, data));
45
+ }
46
+ return results;
47
+ }
48
+ const url = render(urlOrObj, { args, data });
49
+ return fetchSingle(page, String(url), method, queryParams, headers, args, data);
50
+ }
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Pipeline step: intercept — declarative XHR interception.
3
+ */
4
+ import type { IPage } from '../../types.js';
5
+ export declare function stepIntercept(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;
@@ -0,0 +1,75 @@
1
+ /**
2
+ * Pipeline step: intercept — declarative XHR interception.
3
+ */
4
+ import { render } from '../template.js';
5
+ export async function stepIntercept(page, params, data, args) {
6
+ const cfg = typeof params === 'object' ? params : {};
7
+ const trigger = cfg.trigger ?? '';
8
+ const capturePattern = cfg.capture ?? '';
9
+ const timeout = cfg.timeout ?? 8;
10
+ const selectPath = cfg.select ?? null;
11
+ if (!capturePattern)
12
+ return data;
13
+ // Step 1: Execute the trigger action
14
+ if (trigger.startsWith('navigate:')) {
15
+ const url = render(trigger.slice('navigate:'.length), { args, data });
16
+ await page.goto(String(url));
17
+ }
18
+ else if (trigger.startsWith('evaluate:')) {
19
+ const js = trigger.slice('evaluate:'.length);
20
+ const { normalizeEvaluateSource } = await import('../template.js');
21
+ await page.evaluate(normalizeEvaluateSource(render(js, { args, data })));
22
+ }
23
+ else if (trigger.startsWith('click:')) {
24
+ const ref = render(trigger.slice('click:'.length), { args, data });
25
+ await page.click(String(ref).replace(/^@/, ''));
26
+ }
27
+ else if (trigger === 'scroll') {
28
+ await page.scroll('down');
29
+ }
30
+ // Step 2: Wait a bit for network requests to fire
31
+ await page.wait(Math.min(timeout, 3));
32
+ // Step 3: Get network requests and find matching ones
33
+ const rawNetwork = await page.networkRequests(false);
34
+ const matchingResponses = [];
35
+ if (typeof rawNetwork === 'string') {
36
+ const lines = rawNetwork.split('\n');
37
+ for (const line of lines) {
38
+ const match = line.match(/\[?(GET|POST)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i);
39
+ if (match) {
40
+ const [, , url, status] = match;
41
+ if (url.includes(capturePattern) && status === '200') {
42
+ try {
43
+ const body = await page.evaluate(`
44
+ async () => {
45
+ try {
46
+ const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
47
+ if (!resp.ok) return null;
48
+ return await resp.json();
49
+ } catch { return null; }
50
+ }
51
+ `);
52
+ if (body)
53
+ matchingResponses.push(body);
54
+ }
55
+ catch { }
56
+ }
57
+ }
58
+ }
59
+ }
60
+ // Step 4: Select from response if specified
61
+ let result = matchingResponses.length === 1 ? matchingResponses[0] :
62
+ matchingResponses.length > 1 ? matchingResponses : data;
63
+ if (selectPath && result) {
64
+ let current = result;
65
+ for (const part of String(selectPath).split('.')) {
66
+ if (current && typeof current === 'object' && !Array.isArray(current)) {
67
+ current = current[part];
68
+ }
69
+ else
70
+ break;
71
+ }
72
+ result = current ?? result;
73
+ }
74
+ return result;
75
+ }
@@ -0,0 +1,12 @@
1
+ /**
2
+ * Pipeline step: tap — declarative Store Action Bridge.
3
+ *
4
+ * Generates a self-contained IIFE that:
5
+ * 1. Injects fetch + XHR dual interception proxy
6
+ * 2. Finds the Pinia/Vuex store and calls the action
7
+ * 3. Captures the response matching the URL pattern
8
+ * 4. Auto-cleans up interception in finally block
9
+ * 5. Returns the captured data (optionally sub-selected)
10
+ */
11
+ import type { IPage } from '../../types.js';
12
+ export declare function stepTap(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;
@@ -0,0 +1,130 @@
1
+ /**
2
+ * Pipeline step: tap — declarative Store Action Bridge.
3
+ *
4
+ * Generates a self-contained IIFE that:
5
+ * 1. Injects fetch + XHR dual interception proxy
6
+ * 2. Finds the Pinia/Vuex store and calls the action
7
+ * 3. Captures the response matching the URL pattern
8
+ * 4. Auto-cleans up interception in finally block
9
+ * 5. Returns the captured data (optionally sub-selected)
10
+ */
11
+ import { render } from '../template.js';
12
+ export async function stepTap(page, params, data, args) {
13
+ const cfg = typeof params === 'object' ? params : {};
14
+ const storeName = String(render(cfg.store ?? '', { args, data }));
15
+ const actionName = String(render(cfg.action ?? '', { args, data }));
16
+ const capturePattern = String(render(cfg.capture ?? '', { args, data }));
17
+ const timeout = cfg.timeout ?? 5;
18
+ const selectPath = cfg.select ? String(render(cfg.select, { args, data })) : null;
19
+ const framework = cfg.framework ?? null;
20
+ const actionArgs = cfg.args ?? [];
21
+ if (!storeName || !actionName)
22
+ throw new Error('tap: store and action are required');
23
+ // Build select chain for the captured response
24
+ const selectChain = selectPath
25
+ ? selectPath.split('.').map((p) => `?.[${JSON.stringify(p)}]`).join('')
26
+ : '';
27
+ // Serialize action arguments
28
+ const actionArgsRendered = actionArgs.map((a) => {
29
+ const rendered = render(a, { args, data });
30
+ return JSON.stringify(rendered);
31
+ });
32
+ const actionCall = actionArgsRendered.length
33
+ ? `store[${JSON.stringify(actionName)}](${actionArgsRendered.join(', ')})`
34
+ : `store[${JSON.stringify(actionName)}]()`;
35
+ const js = `
36
+ async () => {
37
+ // ── 1. Setup capture proxy (fetch + XHR dual interception) ──
38
+ let captured = null;
39
+ const capturePattern = ${JSON.stringify(capturePattern)};
40
+
41
+ // Intercept fetch API
42
+ const origFetch = window.fetch;
43
+ window.fetch = async function(...fetchArgs) {
44
+ const resp = await origFetch.apply(this, fetchArgs);
45
+ try {
46
+ const url = typeof fetchArgs[0] === 'string' ? fetchArgs[0]
47
+ : fetchArgs[0] instanceof Request ? fetchArgs[0].url : String(fetchArgs[0]);
48
+ if (capturePattern && url.includes(capturePattern) && !captured) {
49
+ try { captured = await resp.clone().json(); } catch {}
50
+ }
51
+ } catch {}
52
+ return resp;
53
+ };
54
+
55
+ // Intercept XMLHttpRequest
56
+ const origXhrOpen = XMLHttpRequest.prototype.open;
57
+ const origXhrSend = XMLHttpRequest.prototype.send;
58
+ XMLHttpRequest.prototype.open = function(method, url) {
59
+ this.__tapUrl = String(url);
60
+ return origXhrOpen.apply(this, arguments);
61
+ };
62
+ XMLHttpRequest.prototype.send = function(body) {
63
+ if (capturePattern && this.__tapUrl?.includes(capturePattern)) {
64
+ const xhr = this;
65
+ const origHandler = xhr.onreadystatechange;
66
+ xhr.onreadystatechange = function() {
67
+ if (xhr.readyState === 4 && !captured) {
68
+ try { captured = JSON.parse(xhr.responseText); } catch {}
69
+ }
70
+ if (origHandler) origHandler.apply(this, arguments);
71
+ };
72
+ const origOnload = xhr.onload;
73
+ xhr.onload = function() {
74
+ if (!captured) { try { captured = JSON.parse(xhr.responseText); } catch {} }
75
+ if (origOnload) origOnload.apply(this, arguments);
76
+ };
77
+ }
78
+ return origXhrSend.apply(this, arguments);
79
+ };
80
+
81
+ try {
82
+ // ── 2. Find store ──
83
+ let store = null;
84
+ const storeName = ${JSON.stringify(storeName)};
85
+ const fw = ${JSON.stringify(framework)};
86
+
87
+ const app = document.querySelector('#app');
88
+ if (!fw || fw === 'pinia') {
89
+ try {
90
+ const pinia = app?.__vue_app__?.config?.globalProperties?.$pinia;
91
+ if (pinia?._s) store = pinia._s.get(storeName);
92
+ } catch {}
93
+ }
94
+ if (!store && (!fw || fw === 'vuex')) {
95
+ try {
96
+ const vuexStore = app?.__vue_app__?.config?.globalProperties?.$store
97
+ ?? app?.__vue__?.$store;
98
+ if (vuexStore) {
99
+ store = { [${JSON.stringify(actionName)}]: (...a) => vuexStore.dispatch(storeName + '/' + ${JSON.stringify(actionName)}, ...a) };
100
+ }
101
+ } catch {}
102
+ }
103
+
104
+ if (!store) return { error: 'Store not found: ' + storeName, hint: 'Page may not be fully loaded or store name may be incorrect' };
105
+ if (typeof store[${JSON.stringify(actionName)}] !== 'function') {
106
+ return { error: 'Action not found: ' + ${JSON.stringify(actionName)} + ' on store ' + storeName,
107
+ hint: 'Available: ' + Object.keys(store).filter(k => typeof store[k] === 'function' && !k.startsWith('$') && !k.startsWith('_')).join(', ') };
108
+ }
109
+
110
+ // ── 3. Call store action ──
111
+ await ${actionCall};
112
+
113
+ // ── 4. Wait for network response ──
114
+ const deadline = Date.now() + ${timeout} * 1000;
115
+ while (!captured && Date.now() < deadline) {
116
+ await new Promise(r => setTimeout(r, 200));
117
+ }
118
+ } finally {
119
+ // ── 5. Always restore originals ──
120
+ window.fetch = origFetch;
121
+ XMLHttpRequest.prototype.open = origXhrOpen;
122
+ XMLHttpRequest.prototype.send = origXhrSend;
123
+ }
124
+
125
+ if (!captured) return { error: 'No matching response captured for pattern: ' + capturePattern };
126
+ return captured${selectChain} ?? captured;
127
+ }
128
+ `;
129
+ return page.evaluate(js);
130
+ }
@@ -0,0 +1,8 @@
1
+ /**
2
+ * Pipeline steps: data transforms — select, map, filter, sort, limit.
3
+ */
4
+ export declare function stepSelect(_page: any, params: any, data: any, args: Record<string, any>): Promise<any>;
5
+ export declare function stepMap(_page: any, params: any, data: any, args: Record<string, any>): Promise<any>;
6
+ export declare function stepFilter(_page: any, params: any, data: any, args: Record<string, any>): Promise<any>;
7
+ export declare function stepSort(_page: any, params: any, data: any, _args: Record<string, any>): Promise<any>;
8
+ export declare function stepLimit(_page: any, params: any, data: any, args: Record<string, any>): Promise<any>;