@jackwener/opencli 0.3.0 → 0.4.1

package/src/pipeline/steps/fetch.ts

@@ -5,6 +5,23 @@
 import type { IPage } from '../../types.js';
 import { render } from '../template.js';
 
+/** Simple async concurrency limiter */
+async function mapConcurrent<T, R>(items: T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]> {
+  const results: R[] = new Array(items.length);
+  let index = 0;
+
+  async function worker() {
+    while (index < items.length) {
+      const i = index++;
+      results[i] = await fn(items[i], i);
+    }
+  }
+
+  const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
+  await Promise.all(workers);
+  return results;
+}
+
 /** Single URL fetch helper */
 async function fetchSingle(
   page: IPage | null, url: string, method: string,
@@ -39,6 +56,46 @@ async function fetchSingle(
   `);
 }
 
+/**
+ * Batch fetch: send all URLs into the browser as a single evaluate() call.
+ * This eliminates N-1 cross-process IPC round trips, performing all fetches
+ * inside the V8 engine and returning results as one JSON array.
+ */
+async function fetchBatchInBrowser(
+  page: IPage, urls: string[], method: string,
+  headers: Record<string, string>, concurrency: number,
+): Promise<any[]> {
+  const headersJs = JSON.stringify(headers);
+  const urlsJs = JSON.stringify(urls);
+  return page.evaluate(`
+    async () => {
+      const urls = ${urlsJs};
+      const method = "${method}";
+      const headers = ${headersJs};
+      const concurrency = ${concurrency};
+
+      const results = new Array(urls.length);
+      let idx = 0;
+
+      async function worker() {
+        while (idx < urls.length) {
+          const i = idx++;
+          try {
+            const resp = await fetch(urls[i], { method, headers, credentials: "include" });
+            results[i] = await resp.json();
+          } catch (e) {
+            results[i] = { error: e.message };
+          }
+        }
+      }
+
+      const workers = Array.from({ length: Math.min(concurrency, urls.length) }, () => worker());
+      await Promise.all(workers);
+      return results;
+    }
+  `);
+}
+
 export async function stepFetch(page: IPage | null, params: any, data: any, args: Record<string, any>): Promise<any> {
   const urlOrObj = typeof params === 'string' ? params : (params?.url ?? '');
   const method = params?.method ?? 'GET';
@@ -48,12 +105,33 @@ export async function stepFetch(page: IPage | null, params: any, data: any, args
 
   // Per-item fetch when data is array and URL references item
   if (Array.isArray(data) && urlTemplate.includes('item')) {
-    const results: any[] = [];
-    for (let i = 0; i < data.length; i++) {
-      const itemUrl = String(render(urlTemplate, { args, data, item: data[i], index: i }));
-      results.push(await fetchSingle(page, itemUrl, method, queryParams, headers, args, data));
+    const concurrency = typeof params?.concurrency === 'number' ? params.concurrency : 5;
+
+    // Render all URLs upfront
+    const renderedHeaders: Record<string, string> = {};
+    for (const [k, v] of Object.entries(headers)) renderedHeaders[k] = String(render(v, { args, data }));
+    const renderedParams: Record<string, string> = {};
+    for (const [k, v] of Object.entries(queryParams)) renderedParams[k] = String(render(v, { args, data }));
+
+    const urls = data.map((item: any, index: number) => {
+      let url = String(render(urlTemplate, { args, data, item, index }));
+      if (Object.keys(renderedParams).length > 0) {
+        const qs = new URLSearchParams(renderedParams).toString();
+        url = `${url}${url.includes('?') ? '&' : '?'}${qs}`;
+      }
+      return url;
+    });
+
+    // BATCH IPC: if browser is available, batch all fetches into a single evaluate() call
+    if (page !== null) {
+      return fetchBatchInBrowser(page, urls, method.toUpperCase(), renderedHeaders, concurrency);
     }
-    return results;
+
+    // Non-browser: use concurrent pool (already optimized)
+    return mapConcurrent(data, concurrency, async (item, index) => {
+      const itemUrl = String(render(urlTemplate, { args, data, item, index }));
+      return fetchSingle(null, itemUrl, method, queryParams, headers, args, data);
+    });
   }
   const url = render(urlOrObj, { args, data });
   return fetchSingle(page, String(url), method, queryParams, headers, args, data);

package/src/pipeline/steps/tap.ts

@@ -42,6 +42,8 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record<
     async () => {
       // ── 1. Setup capture proxy (fetch + XHR dual interception) ──
       let captured = null;
+      let captureResolve;
+      const capturePromise = new Promise(r => { captureResolve = r; });
       const capturePattern = ${JSON.stringify(capturePattern)};
 
       // Intercept fetch API
@@ -52,7 +54,7 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record<
           const url = typeof fetchArgs[0] === 'string' ? fetchArgs[0]
             : fetchArgs[0] instanceof Request ? fetchArgs[0].url : String(fetchArgs[0]);
           if (capturePattern && url.includes(capturePattern) && !captured) {
-            try { captured = await resp.clone().json(); } catch {}
+            try { captured = await resp.clone().json(); captureResolve(); } catch {}
           }
         } catch {}
         return resp;
@@ -71,13 +73,13 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record<
           const origHandler = xhr.onreadystatechange;
           xhr.onreadystatechange = function() {
             if (xhr.readyState === 4 && !captured) {
-              try { captured = JSON.parse(xhr.responseText); } catch {}
+              try { captured = JSON.parse(xhr.responseText); captureResolve(); } catch {}
             }
             if (origHandler) origHandler.apply(this, arguments);
           };
           const origOnload = xhr.onload;
           xhr.onload = function() {
-            if (!captured) { try { captured = JSON.parse(xhr.responseText); } catch {} }
+            if (!captured) { try { captured = JSON.parse(xhr.responseText); captureResolve(); } catch {} }
             if (origOnload) origOnload.apply(this, arguments);
           };
         }
@@ -117,9 +119,9 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record<
         await ${actionCall};
 
         // ── 4. Wait for network response ──
-        const deadline = Date.now() + ${timeout} * 1000;
-        while (!captured && Date.now() < deadline) {
-          await new Promise(r => setTimeout(r, 200));
+        if (!captured) {
+          const timeoutPromise = new Promise(r => setTimeout(r, ${timeout} * 1000));
+          await Promise.race([capturePromise, timeoutPromise]);
         }
       } finally {
         // ── 5. Always restore originals ──

package/src/registry.ts

@@ -34,6 +34,9 @@ export interface CliCommand {
   pipeline?: any[];
   timeoutSeconds?: number;
   source?: string;
+  /** Internal: lazy-loaded TS module support */
+  _lazy?: boolean;
+  _modulePath?: string;
 }
 
 export interface CliOptions {

package/src/types.ts

@@ -12,7 +12,7 @@ export interface IPage {
   click(ref: string): Promise<void>;
   typeText(ref: string, text: string): Promise<void>;
   pressKey(key: string): Promise<void>;
-  wait(seconds: number): Promise<void>;
+  wait(options: number | { text?: string; time?: number; timeout?: number }): Promise<void>;
   tabs(): Promise<any>;
   closeTab(index?: number): Promise<void>;
   newTab(): Promise<void>;