@letsscrapedata/controller 0.0.50 → 0.0.52

package/dist/index.js

@@ -1,14 +1,14 @@
-// src/playwright/browser.ts
-import EventEmitter3 from "events";
-import { getCurrentUnixTime as getCurrentUnixTime3, getPerformanceOfPidTree } from "@letsscrapedata/utils";
-
-// src/playwright/context.ts
-import EventEmitter2 from "events";
-import { getCurrentUnixTime as getCurrentUnixTime2, sleep } from "@letsscrapedata/utils";
-
 // src/utils/log.ts
 import { log, LogLevel } from "@letsscrapedata/utils";
 var pkgLog = log;
+function setControllerLogFun(logFun) {
+  if (typeof logFun === "function") {
+    pkgLog = logFun;
+    return true;
+  } else {
+    return false;
+  }
+}
 async function loginfo(...args) {
   await pkgLog(LogLevel.INF, ...args);
 }
@@ -19,10 +19,44 @@ async function logerr(...args) {
   await pkgLog(LogLevel.ERR, ...args);
 }
 
+// src/playwright/browser.ts
+import EventEmitter3 from "events";
+import { getCurrentUnixTime as getCurrentUnixTime3, getPerformanceOfPidTree } from "@letsscrapedata/utils";
+
+// src/playwright/context.ts
+import EventEmitter2 from "events";
+import { getCurrentUnixTime as getCurrentUnixTime2, sleep } from "@letsscrapedata/utils";
+
 // src/playwright/page.ts
 import EventEmitter from "events";
 import { getCurrentUnixTime, unreachable as unreachable2 } from "@letsscrapedata/utils";
 
+// src/utils/common.ts
+function convertDataAttributeName(attr) {
+  if (!attr.startsWith("data-")) {
+    return "";
+  }
+  const parts = attr.split("-");
+  let name = parts[1];
+  for (const part of parts.slice(2)) {
+    if (!part) {
+      continue;
+    }
+    name = `${name}${part[1].toUpperCase()}${part.slice(1).toLowerCase()}`;
+  }
+  return name;
+}
+function getIframeSelector(iframeOption) {
+  const { src = "", id = "", selector = "" } = iframeOption;
+  if (typeof src === "string" && src) {
+    return `iframe[src^="${src}"]`;
+  } else if (typeof id === "string" && id) {
+    return `iframe[id="${id}"]`;
+  } else {
+    return selector;
+  }
+}
+
 // src/playwright/element.ts
 import { unreachable } from "@letsscrapedata/utils";
 var PlaywrightElement = class _PlaywrightElement {
@@ -43,6 +77,29 @@ var PlaywrightElement = class _PlaywrightElement {
     const names = await this.#locator.evaluate((node) => node.getAttributeNames());
     return names;
   }
+  async dataset() {
+    try {
+      const dataset = await this.#locator.evaluate((node) => node.dataset);
+      return dataset;
+    } catch (err) {
+      return {};
+    }
+  }
+  async evaluate(func, args) {
+    try {
+      const frame = this.#frame;
+      ;
+      if (typeof frame.parentFrame === "function") {
+        return await frame.evaluate(func, args);
+      } else {
+        const locator = this.#frame.owner();
+        return await locator.evaluate(func, args);
+      }
+    } catch (err) {
+      logerr(err);
+      return "";
+    }
+  }
   /*
     async #getChildFrame(parentFrame: Frame, iframeOption: IframeOption): Promise<Frame | null> {
       if (!parentFrame) {
@@ -76,22 +133,15 @@ var PlaywrightElement = class _PlaywrightElement {
       return null;
     }
       */
-  #getIframeSelector(iframeOption) {
-    const { src = "", selector = "" } = iframeOption;
-    if (!src && !selector) {
-      throw new Error("Invalid parent frame");
-    }
-    return selector ? selector : `iframe[src^="${src}"]`;
-  }
   async #getChildFrameLocator(parent, iframeOption) {
-    return parent.frameLocator(this.#getIframeSelector(iframeOption));
+    return parent.frameLocator(getIframeSelector(iframeOption));
   }
   async #getDescendantFrame(parent, iframeOptions) {
     try {
       if (iframeOptions.length <= 0) {
         return null;
       }
-      let frameLocator = parent.frameLocator(this.#getIframeSelector(iframeOptions[0]));
+      let frameLocator = parent.frameLocator(getIframeSelector(iframeOptions[0]));
       for (const iframeOption of iframeOptions.slice(1)) {
         if (!frameLocator) {
           return null;
@@ -108,12 +158,12 @@ var PlaywrightElement = class _PlaywrightElement {
     let frame = this.#frame;
     const retObj = { frame, locators: [] };
     if (iframeOptions.length > 0) {
-      frame = await this.#getDescendantFrame(frame, iframeOptions);
-      if (!frame) {
+      const childFrame = await this.#getDescendantFrame(frame, iframeOptions);
+      if (!childFrame) {
         return retObj;
       }
-      retObj.frame = frame;
-      parent = frame;
+      retObj.frame = childFrame;
+      parent = childFrame;
     }
     try {
       let locators = [];
@@ -298,6 +348,7 @@ var PlaywrightPage = class extends EventEmitter {
   #page;
   #status;
   #pageId;
+  #closeWhenFree;
   #resquestInterceptionOptions;
   #responseInterceptionOptions;
   #client;
@@ -428,29 +479,57 @@ var PlaywrightPage = class extends EventEmitter {
       return null;
     }
     */
-  #getIframeSelector(iframeOption) {
-    const { src = "", selector = "" } = iframeOption;
-    if (!src && !selector) {
-      throw new Error("Invalid parent frame");
+  async #findDescendantFrame(src, id) {
+    if (!this.#page) {
+      throw new Error("No valid page");
     }
-    return selector ? selector : `iframe[src^="${src}"]`;
+    const frames = this.#page.frames();
+    for (const frame of frames) {
+      const url = frame.url();
+      if (typeof src === "string" && src) {
+        if (url.startsWith(src)) {
+          return frame;
+        } else if (url.toLowerCase().startsWith(src)) {
+          return frame;
+        }
+      } else if (src instanceof RegExp) {
+        if (url.match(src)) {
+          return frame;
+        }
+      } else if (id) {
+        const element = await frame.frameElement();
+        if (element) {
+          const frameId = await frame.evaluate(([ele, attr]) => ele.getAttribute(attr), [element, "id"]);
+          if (frameId === id) {
+            return frame;
+          }
+        }
+      }
+    }
+    return null;
   }
   async #getChildFrameLocator(parent, iframeOption) {
-    return parent.frameLocator(this.#getIframeSelector(iframeOption));
+    return parent.frameLocator(getIframeSelector(iframeOption));
   }
-  async #getDescendantFrameLocator(mainFrame, iframeOptions) {
+  async #getDescendantFrame(mainFrame, iframeOptions) {
     try {
       if (iframeOptions.length <= 0) {
         return null;
       }
-      let frameLocator = mainFrame.frameLocator(this.#getIframeSelector(iframeOptions[0]));
-      for (const iframeOption of iframeOptions.slice(1)) {
-        if (!frameLocator) {
-          return null;
+      if (iframeOptions.length === 1 && !iframeOptions[0].selector) {
+        const { src = "", id = "" } = iframeOptions[0];
+        const frame = await this.#findDescendantFrame(src, id);
+        return frame;
+      } else {
+        let frameLocator = mainFrame.frameLocator(getIframeSelector(iframeOptions[0]));
+        for (const iframeOption of iframeOptions.slice(1)) {
+          if (!frameLocator) {
+            return null;
+          }
+          frameLocator = await this.#getChildFrameLocator(frameLocator, iframeOption);
         }
-        frameLocator = await this.#getChildFrameLocator(frameLocator, iframeOption);
+        return frameLocator;
       }
-      return frameLocator;
     } catch (err) {
       throw new Error(`No child iframe: ${JSON.stringify(iframeOptions)}`);
     }
@@ -462,7 +541,7 @@ var PlaywrightPage = class extends EventEmitter {
     let frame = this.#page.mainFrame();
     const retObj = { frame, locators: [] };
     if (iframeOptions.length > 0) {
-      frame = await this.#getDescendantFrameLocator(frame, iframeOptions);
+      frame = await this.#getDescendantFrame(frame, iframeOptions);
       if (!frame) {
         return retObj;
       }
@@ -535,12 +614,32 @@ var PlaywrightPage = class extends EventEmitter {
     const { browserIdx = 0, browserContextIdx = 0, pageIdx = 0, openType = "other", openTime = currentTime, lastStatusUpdateTime = currentTime, taskId = 0, relatedId = 0, misc = {} } = pageInfo ? pageInfo : {};
     this.#page.pageInfo = { browserIdx, browserContextIdx, pageIdx, openType, openTime, lastStatusUpdateTime, taskId, relatedId, misc };
     this.#pageId = `page-${browserIdx}-${browserContextIdx}-${pageIdx}`;
+    this.#closeWhenFree = false;
     this.#resquestInterceptionOptions = [];
     this.#responseInterceptionOptions = [];
     this.#client = null;
     this.#responseCb = null;
     this.#addPageOn();
   }
+  async addPreloadScript(scriptOrFunc, arg) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    if (typeof scriptOrFunc === "string") {
+      await this.#page.addInitScript({ content: scriptOrFunc });
+    } else if (typeof scriptOrFunc === "function") {
+      await this.#page.addInitScript(scriptOrFunc, arg);
+    } else {
+      throw new Error(`Invalid type of scriptOrFunc ${typeof scriptOrFunc}`);
+    }
+    return true;
+  }
+  async addScriptTag(options) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    return this.#page.addScriptTag(options);
+  }
   apiContext() {
     return this.browserContext().apiContext();
   }
@@ -600,7 +699,7 @@ var PlaywrightPage = class extends EventEmitter {
   }
   async close() {
     if (this.#status === "closed") {
-      console.warn(`Page ${this.#pageId} is already closed.`);
+      logwarn(`Page ${this.#pageId} is already closed.`);
       return true;
     } else if (this.#status === "busy") {
       throw new Error(`Page ${this.#pageId} cannot be closed because it is busy.`);
@@ -613,13 +712,16 @@ var PlaywrightPage = class extends EventEmitter {
     this.#status = "closed";
     return true;
   }
+  closeWhenFree() {
+    return this.#closeWhenFree;
+  }
   async content(iframeOptions = []) {
     if (!this.#page) {
       throw new Error("No valid page");
     }
     let content = "";
     if (iframeOptions.length > 0) {
-      const frameLocator = await this.#getDescendantFrameLocator(this.#page.mainFrame(), iframeOptions);
+      const frameLocator = await this.#getDescendantFrame(this.#page.mainFrame(), iframeOptions);
       if (frameLocator) {
         content = await frameLocator.locator(":root").evaluate(() => document.documentElement.outerHTML);
       }
@@ -641,11 +743,18 @@ var PlaywrightPage = class extends EventEmitter {
     const height = await this.#page.evaluate(() => document.documentElement.scrollHeight);
     return height;
   }
-  async evalute(fun, args) {
+  async evaluate(func, args) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    return this.#page.evaluate(func, args);
+  }
+  async exposeFunction(name, callbackFunction) {
     if (!this.#page) {
       throw new Error("No valid page");
     }
-    return this.#page.evaluate(fun, args);
+    await this.#page.exposeFunction(name, callbackFunction);
+    return;
   }
   async findElement(selectorOrXpath, iframeOptions = []) {
     if (!this.#page) {
@@ -683,7 +792,7 @@ var PlaywrightPage = class extends EventEmitter {
   }
   async free() {
     if (this.#status === "free") {
-      console.warn(`Page ${this.#pageId} is already free.`);
+      logwarn(`Page ${this.#pageId} is already free.`);
     }
     this.#status = "free";
     await this.clearRequestInterceptions();
@@ -823,6 +932,10 @@ var PlaywrightPage = class extends EventEmitter {
     }
     return response;
   }
+  setCloseWhenFree(closeWhenFree) {
+    this.#closeWhenFree = closeWhenFree;
+    return true;
+  }
   async setCookies(cookies) {
     if (!this.#page) {
       throw new Error("No valid page");
@@ -1006,7 +1119,7 @@ var PlaywrightPage = class extends EventEmitter {
     }
     const actOptions = Array.isArray(options) ? options : [options];
     if (actOptions.length <= 0) {
-      console.warn("Invalid paras in setResponseInterception");
+      logwarn("Invalid paras in setResponseInterception");
       return false;
     }
     const firstResponseInterception = this.#responseInterceptionOptions.length <= 0;
@@ -1708,6 +1821,26 @@ var PuppeteerElement = class _PuppeteerElement {
     const names = await this.#frame.evaluate((ele) => ele.getAttributeNames(), this.#$ele);
     return names;
   }
+  async dataset() {
+    try {
+      const attributeNames = await this.attributeNames();
+      const dataset = {};
+      for (const attributeName of attributeNames) {
+        if (!attributeName.startsWith("data-")) {
+          continue;
+        }
+        const val = await this.attribute(attributeName);
+        const key = convertDataAttributeName(attributeName);
+        dataset[key] = val;
+      }
+      return dataset;
+    } catch (err) {
+      return {};
+    }
+  }
+  async evaluate(func, args) {
+    return await this.#frame.evaluate(func, args);
+  }
   async #getChildFrame(parentFrame, iframeOption) {
     if (!parentFrame) {
       throw new Error("Invalid parent frame");
@@ -1757,13 +1890,13 @@ var PuppeteerElement = class _PuppeteerElement {
     let frame = this.#frame;
     const retObj = { frame, elementHandles: [] };
     if (iframeOptions.length > 0) {
-      frame = await this.#getDescendantFrame(frame, iframeOptions);
-      if (!frame) {
+      const childFrame = await this.#getDescendantFrame(frame, iframeOptions);
+      if (!childFrame) {
         return retObj;
       }
-      retObj.frame = frame;
+      retObj.frame = childFrame;
       absolute = true;
-      parent = frame;
+      parent = childFrame;
     }
     try {
       if (selector.startsWith("./") || selector.startsWith("/") || selector.startsWith("..")) {
@@ -1965,6 +2098,7 @@ var PuppeteerPage = class extends EventEmitter4 {
   #page;
   #status;
   #pageId;
+  #closeWhenFree;
   #requestInterceptionNum;
   #responseInterceptionNum;
   #client;
@@ -2039,15 +2173,41 @@ var PuppeteerPage = class extends EventEmitter4 {
     });
     return true;
   }
+  async #findDescendantFrame(src, id) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    const frames = this.#page.frames();
+    for (const frame of frames) {
+      const url = frame.url();
+      if (typeof src === "string" && src) {
+        if (url.startsWith(src)) {
+          return frame;
+        } else if (url.toLowerCase().startsWith(src)) {
+          return frame;
+        }
+      } else if (src instanceof RegExp) {
+        if (url.match(src)) {
+          return frame;
+        }
+      } else if (id) {
+        const element = await frame.frameElement();
+        if (element) {
+          const frameId = await frame.evaluate((ele, attr) => ele.getAttribute(attr), element, "id");
+          if (frameId === id) {
+            return frame;
+          }
+        }
+      }
+    }
+    return null;
+  }
   async #getChildFrame(parentFrame, iframeOption) {
     if (!parentFrame) {
       throw new Error("Invalid parent frame");
     }
     let iframe = null;
-    let { src = "", selector = "" } = iframeOption;
-    if (!src && !selector) {
-      throw new Error("Invalid IframeOption");
-    }
+    let { src = "" } = iframeOption;
     if (src) {
       const childFrames = parentFrame.childFrames();
       for (const childFrame of childFrames) {
@@ -2065,7 +2225,8 @@ var PuppeteerPage = class extends EventEmitter4 {
         }
       }
     } else {
-      const $eleIframe = await parentFrame.$(selector);
+      const frameSelector = getIframeSelector(iframeOption);
+      const $eleIframe = await parentFrame.$(frameSelector);
       if ($eleIframe) {
         iframe = await $eleIframe.contentFrame();
         return iframe;
@@ -2075,11 +2236,16 @@ var PuppeteerPage = class extends EventEmitter4 {
   }
   async #getDescendantFrame(parentFrame, iframeOptions) {
     let iframe = parentFrame;
-    for (const iframeOption of iframeOptions) {
-      if (!iframe) {
-        return null;
+    if (iframeOptions.length === 1 && !iframeOptions[0].selector) {
+      const { src = "", id = "" } = iframeOptions[0];
+      iframe = await this.#findDescendantFrame(src, id);
+    } else {
+      for (const iframeOption of iframeOptions) {
+        if (!iframe) {
+          return null;
+        }
+        iframe = await this.#getChildFrame(iframe, iframeOption);
       }
-      iframe = await this.#getChildFrame(iframe, iframeOption);
     }
     return iframe;
   }
@@ -2108,7 +2274,7 @@ var PuppeteerPage = class extends EventEmitter4 {
       }
       return retObj;
     } catch (err) {
-      console.log(err);
+      loginfo(err);
       return retObj;
     }
   }
@@ -2161,11 +2327,31 @@ var PuppeteerPage = class extends EventEmitter4 {
     const { browserIdx = 0, browserContextIdx = 0, pageIdx = 0, openType = "other", openTime = currentTime, lastStatusUpdateTime = currentTime, taskId = 0, relatedId = 0, misc = {} } = pageInfo ? pageInfo : {};
     this.#page.pageInfo = { browserIdx, browserContextIdx, pageIdx, openType, openTime, lastStatusUpdateTime, taskId, relatedId, misc };
     this.#pageId = `page-${browserIdx}-${browserContextIdx}-${pageIdx}`;
+    this.#closeWhenFree = false;
     this.#requestInterceptionNum = 0;
     this.#responseInterceptionNum = 0;
     this.#client = null;
     this.#addPageOn();
   }
+  async addPreloadScript(scriptOrFunc, arg) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    if (typeof scriptOrFunc === "string") {
+      await this.#page.evaluateOnNewDocument(scriptOrFunc);
+    } else if (typeof scriptOrFunc === "function") {
+      await this.#page.evaluateOnNewDocument(scriptOrFunc, arg);
+    } else {
+      throw new Error(`Invalid type of scriptOrFunc ${typeof scriptOrFunc}`);
+    }
+    return true;
+  }
+  async addScriptTag(options) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    return this.#page.addScriptTag(options);
+  }
   apiContext() {
     throw new Error("Not supported in PuppeteerPage.");
   }
@@ -2222,7 +2408,7 @@ var PuppeteerPage = class extends EventEmitter4 {
   }
   async close() {
     if (this.#status === "closed") {
-      console.warn(`Page ${this.#pageId} is already closed.`);
+      logwarn(`Page ${this.#pageId} is already closed.`);
       return true;
     } else if (this.#status === "busy") {
       throw new Error(`Page ${this.#pageId} cannot be closed because it is busy.`);
@@ -2235,6 +2421,9 @@ var PuppeteerPage = class extends EventEmitter4 {
     this.#status = "closed";
     return true;
   }
+  closeWhenFree() {
+    return this.#closeWhenFree;
+  }
   async content(iframeOptions = []) {
     if (!this.#page) {
       throw new Error("No valid page");
@@ -2263,11 +2452,18 @@ var PuppeteerPage = class extends EventEmitter4 {
     const height = await this.#page.evaluate(() => document.documentElement.scrollHeight);
     return height;
   }
-  async evalute(fun, args) {
+  async evaluate(func, args) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    return this.#page.evaluate(func, args);
+  }
+  async exposeFunction(name, callbackFunction) {
     if (!this.#page) {
       throw new Error("No valid page");
     }
-    return this.#page.evaluate(fun, args);
+    await this.#page.exposeFunction(name, callbackFunction);
+    return;
   }
   async findElement(selectorOrXpath, iframeOptions = []) {
     if (!this.#page) {
@@ -2305,7 +2501,7 @@ var PuppeteerPage = class extends EventEmitter4 {
   }
   async free() {
     if (this.#status === "free") {
-      console.warn(`Page ${this.#pageId} is already free.`);
+      logwarn(`Page ${this.#pageId} is already free.`);
     }
     this.#status = "free";
     await this.clearRequestInterceptions();
@@ -2448,6 +2644,10 @@ var PuppeteerPage = class extends EventEmitter4 {
     }
     return response;
   }
+  setCloseWhenFree(closeWhenFree) {
+    this.#closeWhenFree = closeWhenFree;
+    return true;
+  }
   async setCookies(cookies) {
     if (!this.#page) {
       throw new Error("No valid page");
@@ -2523,7 +2723,7 @@ var PuppeteerPage = class extends EventEmitter4 {
     }
     const actOptions = Array.isArray(options) ? options : [options];
     if (actOptions.length <= 0) {
-      console.warn("Invalid paras in setRequestInterception");
+      logwarn("Invalid paras in setRequestInterception");
       return false;
     }
     if (this.#requestInterceptionNum <= 0) {
@@ -2559,7 +2759,7 @@ var PuppeteerPage = class extends EventEmitter4 {
         await request.continue();
         return true;
       } catch (err) {
-        console.error(err);
+        logerr(err);
         return false;
       }
     });
@@ -2571,7 +2771,7 @@ var PuppeteerPage = class extends EventEmitter4 {
     }
     const actOptions = Array.isArray(options) ? options : [options];
     if (actOptions.length <= 0) {
-      console.warn("Invalid paras in setResponseInterception");
+      logwarn("Invalid paras in setResponseInterception");
       return false;
     }
     this.#responseInterceptionNum++;
@@ -2626,7 +2826,7 @@ var PuppeteerPage = class extends EventEmitter4 {
         }
         return true;
       } catch (err) {
-        console.error(err);
+        logerr(err);
         return false;
       }
     });
@@ -3286,6 +3486,22 @@ var CheerioElement = class _CheerioElement {
       return Array.from(Object.keys(element.attribs));
     }
   }
+  async dataset() {
+    const attributeNames = await this.attributeNames();
+    const dataset = {};
+    for (const attributeName of attributeNames) {
+      if (!attributeName.startsWith("data-")) {
+        continue;
+      }
+      const val = await this.attribute(attributeName);
+      const key = convertDataAttributeName(attributeName);
+      dataset[key] = val;
+    }
+    return dataset;
+  }
+  async evaluate() {
+    throw new Error("Not supported in CheerioPage.");
+  }
   #findNodes(selector, absolute) {
     if (selector.startsWith("./") || selector.startsWith("/")) {
       throw new Error("Do not support XPath in cheerio.");
@@ -3406,6 +3622,12 @@ var CheerioPage = class extends EventEmitter7 {
   _origPage() {
     throw new Error("Method not implemented.");
   }
+  async addPreloadScript() {
+    throw new Error("Not supported in CheerioPage.");
+  }
+  addScriptTag() {
+    throw new Error("Not supported in CheerioPage.");
+  }
   apiContext() {
     throw new Error("Not supported in CheerioPage.");
   }
@@ -3433,13 +3655,19 @@ var CheerioPage = class extends EventEmitter7 {
   async close() {
     throw new Error("Not supported in CheerioPage.");
   }
+  closeWhenFree() {
+    throw new Error("Not supported in CheerioPage.");
+  }
   async content() {
     throw new Error("Not supported in CheerioPage.");
   }
   async cookies() {
     throw new Error("Not supported in CheerioPage.");
   }
-  async evalute() {
+  async evaluate() {
+    throw new Error("Not supported in CheerioPage.");
+  }
+  exposeFunction() {
     throw new Error("Not supported in CheerioPage.");
   }
   #findNodes(selector) {
@@ -3536,6 +3764,9 @@ var CheerioPage = class extends EventEmitter7 {
   async sendCDPMessage() {
     throw new Error("Method not implemented.");
   }
+  setCloseWhenFree() {
+    throw new Error("Not supported in CheerioPage.");
+  }
   async setCookies() {
     throw new Error("Not supported in CheerioPage.");
   }
@@ -3872,5 +4103,6 @@ export {
   PuppeteerBrowserContext,
   PuppeteerElement,
   PuppeteerPage,
-  controller
+  controller,
+  setControllerLogFun
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@letsscrapedata/controller",
-  "version": "0.0.50",
+  "version": "0.0.52",
   "description": "Unified browser / HTML controller interfaces that support playwright, puppeteer and cheerio",
   "type": "module",
   "main": "./dist/index.cjs",