@jackwener/opencli 0.2.0 → 0.4.0

package/src/pipeline/steps/browser.ts

@@ -31,13 +31,10 @@ export async function stepWait(page: IPage, params: any, data: any, args: Record
   if (typeof params === 'number') await page.wait(params);
   else if (typeof params === 'object' && params) {
     if ('text' in params) {
-      const timeout = params.timeout ?? 10;
-      const start = Date.now();
-      while ((Date.now() - start) / 1000 < timeout) {
-        const snap = await page.snapshot({ raw: true });
-        if (typeof snap === 'string' && snap.includes(params.text)) break;
-        await page.wait(0.5);
-      }
+      await page.wait({
+        text: String(render(params.text, { args, data })),
+        timeout: params.timeout
+      });
     } else if ('time' in params) await page.wait(Number(params.time));
   } else if (typeof params === 'string') await page.wait(Number(render(params, { args, data })));
   return data;

package/src/pipeline/steps/fetch.ts

@@ -5,6 +5,23 @@
 import type { IPage } from '../../types.js';
 import { render } from '../template.js';
 
+/** Simple async concurrency limiter */
+async function mapConcurrent<T, R>(items: T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]> {
+  const results: R[] = new Array(items.length);
+  let index = 0;
+
+  async function worker() {
+    while (index < items.length) {
+      const i = index++;
+      results[i] = await fn(items[i], i);
+    }
+  }
+
+  const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
+  await Promise.all(workers);
+  return results;
+}
+
 /** Single URL fetch helper */
 async function fetchSingle(
   page: IPage | null, url: string, method: string,
@@ -48,12 +65,11 @@ export async function stepFetch(page: IPage | null, params: any, data: any, args
 
   // Per-item fetch when data is array and URL references item
   if (Array.isArray(data) && urlTemplate.includes('item')) {
-    const results: any[] = [];
-    for (let i = 0; i < data.length; i++) {
-      const itemUrl = String(render(urlTemplate, { args, data, item: data[i], index: i }));
-      results.push(await fetchSingle(page, itemUrl, method, queryParams, headers, args, data));
-    }
-    return results;
+    const concurrency = typeof params?.concurrency === 'number' ? params.concurrency : 5;
+    return mapConcurrent(data, concurrency, async (item, index) => {
+      const itemUrl = String(render(urlTemplate, { args, data, item, index }));
+      return fetchSingle(page, itemUrl, method, queryParams, headers, args, data);
+    });
   }
   const url = render(urlOrObj, { args, data });
   return fetchSingle(page, String(url), method, queryParams, headers, args, data);

package/src/pipeline/steps/intercept.ts

@@ -14,7 +14,55 @@ export async function stepIntercept(page: IPage, params: any, data: any, args: R
 
   if (!capturePattern) return data;
 
-  // Step 1: Execute the trigger action
+  // Step 1: Inject fetch/XHR interceptor BEFORE trigger
+  await page.evaluate(`
+    () => {
+      window.__opencli_intercepted = window.__opencli_intercepted || [];
+      const pattern = ${JSON.stringify(capturePattern)};
+      
+      if (!window.__opencli_fetch_patched) {
+        const origFetch = window.fetch;
+        window.fetch = async function(...args) {
+          const reqUrl = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
+          const response = await origFetch.apply(this, args);
+          setTimeout(async () => {
+            try {
+              if (reqUrl.includes(pattern)) {
+                const clone = response.clone();
+                const json = await clone.json();
+                window.__opencli_intercepted.push(json);
+              }
+            } catch(e) {}
+          }, 0);
+          return response;
+        };
+        window.__opencli_fetch_patched = true;
+      }
+
+      if (!window.__opencli_xhr_patched) {
+        const XHR = XMLHttpRequest.prototype;
+        const open = XHR.open;
+        const send = XHR.send;
+        XHR.open = function(method, url, ...args) {
+          this._reqUrl = url;
+          return open.call(this, method, url, ...args);
+        };
+        XHR.send = function(...args) {
+          this.addEventListener('load', function() {
+            try {
+              if (this._reqUrl && this._reqUrl.includes(pattern)) {
+                window.__opencli_intercepted.push(JSON.parse(this.responseText));
+              }
+            } catch(e) {}
+          });
+          return send.apply(this, args);
+        };
+        window.__opencli_xhr_patched = true;
+      }
+    }
+  `);
+
+  // Step 2: Execute the trigger action
   if (trigger.startsWith('navigate:')) {
     const url = render(trigger.slice('navigate:'.length), { args, data });
     await page.goto(String(url));
@@ -29,36 +77,18 @@ export async function stepIntercept(page: IPage, params: any, data: any, args: R
     await page.scroll('down');
   }
 
-  // Step 2: Wait a bit for network requests to fire
+  // Step 3: Wait a bit for network requests to fire
   await page.wait(Math.min(timeout, 3));
 
-  // Step 3: Get network requests and find matching ones
-  const rawNetwork = await page.networkRequests(false);
-  const matchingResponses: any[] = [];
-
-  if (typeof rawNetwork === 'string') {
-    const lines = rawNetwork.split('\n');
-    for (const line of lines) {
-      const match = line.match(/\[?(GET|POST)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i);
-      if (match) {
-        const [, , url, status] = match;
-        if (url.includes(capturePattern) && status === '200') {
-          try {
-            const body = await page.evaluate(`
-              async () => {
-                try {
-                  const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
-                  if (!resp.ok) return null;
-                  return await resp.json();
-                } catch { return null; }
-              }
-            `);
-            if (body) matchingResponses.push(body);
-          } catch {}
-        }
-      }
+  // Step 4: Retrieve captured data
+  const matchingResponses = await page.evaluate(`
+    () => {
+      const data = window.__opencli_intercepted || [];
+      window.__opencli_intercepted = []; // clear after reading
+      return data;
     }
-  }
+  `);
+
 
   // Step 4: Select from response if specified
   let result = matchingResponses.length === 1 ? matchingResponses[0] :

package/src/pipeline/steps/tap.ts

@@ -42,6 +42,8 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record<
     async () => {
       // ── 1. Setup capture proxy (fetch + XHR dual interception) ──
       let captured = null;
+      let captureResolve;
+      const capturePromise = new Promise(r => { captureResolve = r; });
       const capturePattern = ${JSON.stringify(capturePattern)};
 
       // Intercept fetch API
@@ -52,7 +54,7 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record<
           const url = typeof fetchArgs[0] === 'string' ? fetchArgs[0]
             : fetchArgs[0] instanceof Request ? fetchArgs[0].url : String(fetchArgs[0]);
           if (capturePattern && url.includes(capturePattern) && !captured) {
-            try { captured = await resp.clone().json(); } catch {}
+            try { captured = await resp.clone().json(); captureResolve(); } catch {}
           }
         } catch {}
         return resp;
@@ -71,13 +73,13 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record<
           const origHandler = xhr.onreadystatechange;
           xhr.onreadystatechange = function() {
             if (xhr.readyState === 4 && !captured) {
-              try { captured = JSON.parse(xhr.responseText); } catch {}
+              try { captured = JSON.parse(xhr.responseText); captureResolve(); } catch {}
             }
             if (origHandler) origHandler.apply(this, arguments);
           };
           const origOnload = xhr.onload;
           xhr.onload = function() {
-            if (!captured) { try { captured = JSON.parse(xhr.responseText); } catch {} }
+            if (!captured) { try { captured = JSON.parse(xhr.responseText); captureResolve(); } catch {} }
             if (origOnload) origOnload.apply(this, arguments);
           };
         }
@@ -117,9 +119,9 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record<
         await ${actionCall};
 
         // ── 4. Wait for network response ──
-        const deadline = Date.now() + ${timeout} * 1000;
-        while (!captured && Date.now() < deadline) {
-          await new Promise(r => setTimeout(r, 200));
+        if (!captured) {
+          const timeoutPromise = new Promise(r => setTimeout(r, ${timeout} * 1000));
+          await Promise.race([capturePromise, timeoutPromise]);
         }
       } finally {
         // ── 5. Always restore originals ──

package/src/pipeline/template.test.ts

@@ -75,6 +75,12 @@ describe('evalExpr', () => {
   it('applies length filter', () => {
     expect(evalExpr('item.items | length', { item: { items: [1, 2, 3] } })).toBe(3);
   });
+  it('applies json filter to strings with quotes', () => {
+    expect(evalExpr('args.keyword | json', { args: { keyword: "O'Reilly" } })).toBe('"O\'Reilly"');
+  });
+  it('applies json filter to nullish values', () => {
+    expect(evalExpr('args.keyword | json', { args: {} })).toBe('null');
+  });
 });
 
 describe('render', () => {

package/src/pipeline/template.ts

@@ -75,7 +75,7 @@ export function evalExpr(expr: string, ctx: RenderContext): any {
  * Apply a named filter to a value.
  * Supported filters:
  *   default(val), join(sep), upper, lower, truncate(n), trim,
- *   replace(old,new), keys, length, first, last
+ *   replace(old,new), keys, length, first, last, json
  */
 function applyFilter(filterExpr: string, value: any): any {
   const match = filterExpr.match(/^(\w+)(?:\((.+)\))?$/);
@@ -117,6 +117,8 @@ function applyFilter(filterExpr: string, value: any): any {
       return Array.isArray(value) ? value[0] : value;
     case 'last':
       return Array.isArray(value) ? value[value.length - 1] : value;
+    case 'json':
+      return JSON.stringify(value ?? null);
     default:
       return value;
   }

package/src/types.ts

@@ -12,7 +12,7 @@ export interface IPage {
   click(ref: string): Promise<void>;
   typeText(ref: string, text: string): Promise<void>;
   pressKey(key: string): Promise<void>;
-  wait(seconds: number): Promise<void>;
+  wait(options: number | { text?: string; time?: number; timeout?: number }): Promise<void>;
   tabs(): Promise<any>;
   closeTab(index?: number): Promise<void>;
   newTab(): Promise<void>;
@@ -20,4 +20,7 @@ export interface IPage {
   networkRequests(includeStatic?: boolean): Promise<any>;
   consoleMessages(level?: string): Promise<any>;
   scroll(direction?: string, amount?: number): Promise<void>;
+  autoScroll(options?: { times?: number; delayMs?: number }): Promise<void>;
+  installInterceptor(pattern: string): Promise<void>;
+  getInterceptedRequests(): Promise<any[]>;
 }

package/dist/clis/index.d.ts

@@ -1,22 +0,0 @@
-/**
- * Import all TypeScript CLI adapters so they self-register.
- *
- * Each TS adapter calls cli() on import, which adds itself to the global registry.
- */
-import './bilibili/search.js';
-import './bilibili/me.js';
-import './bilibili/favorite.js';
-import './bilibili/history.js';
-import './bilibili/feed.js';
-import './bilibili/user-videos.js';
-import './github/search.js';
-import './zhihu/question.js';
-import './xiaohongshu/search.js';
-import './bbc/news.js';
-import './weibo/hot.js';
-import './boss/search.js';
-import './yahoo-finance/quote.js';
-import './reuters/search.js';
-import './smzdm/search.js';
-import './ctrip/search.js';
-import './youtube/search.js';

package/dist/clis/index.js

@@ -1,34 +0,0 @@
-/**
- * Import all TypeScript CLI adapters so they self-register.
- *
- * Each TS adapter calls cli() on import, which adds itself to the global registry.
- */
-// bilibili
-import './bilibili/search.js';
-import './bilibili/me.js';
-import './bilibili/favorite.js';
-import './bilibili/history.js';
-import './bilibili/feed.js';
-import './bilibili/user-videos.js';
-// github
-import './github/search.js';
-// zhihu
-import './zhihu/question.js';
-// xiaohongshu
-import './xiaohongshu/search.js';
-// bbc
-import './bbc/news.js';
-// weibo
-import './weibo/hot.js';
-// boss
-import './boss/search.js';
-// yahoo-finance
-import './yahoo-finance/quote.js';
-// reuters
-import './reuters/search.js';
-// smzdm
-import './smzdm/search.js';
-// ctrip
-import './ctrip/search.js';
-// youtube
-import './youtube/search.js';

package/src/clis/index.ts

@@ -1,46 +0,0 @@
-/**
- * Import all TypeScript CLI adapters so they self-register.
- *
- * Each TS adapter calls cli() on import, which adds itself to the global registry.
- */
-
-// bilibili
-import './bilibili/search.js';
-import './bilibili/me.js';
-import './bilibili/favorite.js';
-import './bilibili/history.js';
-import './bilibili/feed.js';
-import './bilibili/user-videos.js';
-
-// github
-import './github/search.js';
-
-// zhihu
-import './zhihu/question.js';
-
-// xiaohongshu
-import './xiaohongshu/search.js';
-
-// bbc
-import './bbc/news.js';
-
-// weibo
-import './weibo/hot.js';
-
-// boss
-import './boss/search.js';
-
-// yahoo-finance
-import './yahoo-finance/quote.js';
-
-// reuters
-import './reuters/search.js';
-
-// smzdm
-import './smzdm/search.js';
-
-// ctrip
-import './ctrip/search.js';
-
-// youtube
-import './youtube/search.js';