@letsscrapedata/controller 0.0.51 → 0.0.52

package/dist/index.cjs

@@ -78,6 +78,32 @@ var import_utils4 = require("@letsscrapedata/utils");
 var import_node_events = __toESM(require("events"), 1);
 var import_utils3 = require("@letsscrapedata/utils");
 
+// src/utils/common.ts
+function convertDataAttributeName(attr) {
+  if (!attr.startsWith("data-")) {
+    return "";
+  }
+  const parts = attr.split("-");
+  let name = parts[1];
+  for (const part of parts.slice(2)) {
+    if (!part) {
+      continue;
+    }
+    name = `${name}${part[1].toUpperCase()}${part.slice(1).toLowerCase()}`;
+  }
+  return name;
+}
+function getIframeSelector(iframeOption) {
+  const { src = "", id = "", selector = "" } = iframeOption;
+  if (typeof src === "string" && src) {
+    return `iframe[src^="${src}"]`;
+  } else if (typeof id === "string" && id) {
+    return `iframe[id="${id}"]`;
+  } else {
+    return selector;
+  }
+}
+
 // src/playwright/element.ts
 var import_utils2 = require("@letsscrapedata/utils");
 var PlaywrightElement = class _PlaywrightElement {
@@ -98,6 +124,29 @@ var PlaywrightElement = class _PlaywrightElement {
     const names = await this.#locator.evaluate((node) => node.getAttributeNames());
     return names;
   }
+  async dataset() {
+    try {
+      const dataset = await this.#locator.evaluate((node) => node.dataset);
+      return dataset;
+    } catch (err) {
+      return {};
+    }
+  }
+  async evaluate(func, args) {
+    try {
+      const frame = this.#frame;
+      ;
+      if (typeof frame.parentFrame === "function") {
+        return await frame.evaluate(func, args);
+      } else {
+        const locator = this.#frame.owner();
+        return await locator.evaluate(func, args);
+      }
+    } catch (err) {
+      logerr(err);
+      return "";
+    }
+  }
   /*
     async #getChildFrame(parentFrame: Frame, iframeOption: IframeOption): Promise<Frame | null> {
       if (!parentFrame) {
@@ -131,22 +180,15 @@ var PlaywrightElement = class _PlaywrightElement {
       return null;
     }
       */
-  #getIframeSelector(iframeOption) {
-    const { src = "", selector = "" } = iframeOption;
-    if (!src && !selector) {
-      throw new Error("Invalid parent frame");
-    }
-    return selector ? selector : `iframe[src^="${src}"]`;
-  }
   async #getChildFrameLocator(parent, iframeOption) {
-    return parent.frameLocator(this.#getIframeSelector(iframeOption));
+    return parent.frameLocator(getIframeSelector(iframeOption));
   }
   async #getDescendantFrame(parent, iframeOptions) {
     try {
       if (iframeOptions.length <= 0) {
         return null;
       }
-      let frameLocator = parent.frameLocator(this.#getIframeSelector(iframeOptions[0]));
+      let frameLocator = parent.frameLocator(getIframeSelector(iframeOptions[0]));
       for (const iframeOption of iframeOptions.slice(1)) {
         if (!frameLocator) {
           return null;
@@ -163,12 +205,12 @@ var PlaywrightElement = class _PlaywrightElement {
     let frame = this.#frame;
     const retObj = { frame, locators: [] };
     if (iframeOptions.length > 0) {
-      frame = await this.#getDescendantFrame(frame, iframeOptions);
-      if (!frame) {
+      const childFrame = await this.#getDescendantFrame(frame, iframeOptions);
+      if (!childFrame) {
         return retObj;
       }
-      retObj.frame = frame;
-      parent = frame;
+      retObj.frame = childFrame;
+      parent = childFrame;
     }
     try {
       let locators = [];
@@ -353,6 +395,7 @@ var PlaywrightPage = class extends import_node_events.default {
   #page;
   #status;
   #pageId;
+  #closeWhenFree;
   #resquestInterceptionOptions;
   #responseInterceptionOptions;
   #client;
@@ -483,29 +526,57 @@ var PlaywrightPage = class extends import_node_events.default {
       return null;
     }
     */
-  #getIframeSelector(iframeOption) {
-    const { src = "", selector = "" } = iframeOption;
-    if (!src && !selector) {
-      throw new Error("Invalid parent frame");
+  async #findDescendantFrame(src, id) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    const frames = this.#page.frames();
+    for (const frame of frames) {
+      const url = frame.url();
+      if (typeof src === "string" && src) {
+        if (url.startsWith(src)) {
+          return frame;
+        } else if (url.toLowerCase().startsWith(src)) {
+          return frame;
+        }
+      } else if (src instanceof RegExp) {
+        if (url.match(src)) {
+          return frame;
+        }
+      } else if (id) {
+        const element = await frame.frameElement();
+        if (element) {
+          const frameId = await frame.evaluate(([ele, attr]) => ele.getAttribute(attr), [element, "id"]);
+          if (frameId === id) {
+            return frame;
+          }
+        }
+      }
     }
-    return selector ? selector : `iframe[src^="${src}"]`;
+    return null;
   }
   async #getChildFrameLocator(parent, iframeOption) {
-    return parent.frameLocator(this.#getIframeSelector(iframeOption));
+    return parent.frameLocator(getIframeSelector(iframeOption));
   }
-  async #getDescendantFrameLocator(mainFrame, iframeOptions) {
+  async #getDescendantFrame(mainFrame, iframeOptions) {
     try {
       if (iframeOptions.length <= 0) {
         return null;
       }
-      let frameLocator = mainFrame.frameLocator(this.#getIframeSelector(iframeOptions[0]));
-      for (const iframeOption of iframeOptions.slice(1)) {
-        if (!frameLocator) {
-          return null;
+      if (iframeOptions.length === 1 && !iframeOptions[0].selector) {
+        const { src = "", id = "" } = iframeOptions[0];
+        const frame = await this.#findDescendantFrame(src, id);
+        return frame;
+      } else {
+        let frameLocator = mainFrame.frameLocator(getIframeSelector(iframeOptions[0]));
+        for (const iframeOption of iframeOptions.slice(1)) {
+          if (!frameLocator) {
+            return null;
+          }
+          frameLocator = await this.#getChildFrameLocator(frameLocator, iframeOption);
         }
-        frameLocator = await this.#getChildFrameLocator(frameLocator, iframeOption);
+        return frameLocator;
       }
-      return frameLocator;
     } catch (err) {
       throw new Error(`No child iframe: ${JSON.stringify(iframeOptions)}`);
     }
@@ -517,7 +588,7 @@ var PlaywrightPage = class extends import_node_events.default {
     let frame = this.#page.mainFrame();
     const retObj = { frame, locators: [] };
     if (iframeOptions.length > 0) {
-      frame = await this.#getDescendantFrameLocator(frame, iframeOptions);
+      frame = await this.#getDescendantFrame(frame, iframeOptions);
       if (!frame) {
         return retObj;
       }
@@ -590,12 +661,32 @@ var PlaywrightPage = class extends import_node_events.default {
     const { browserIdx = 0, browserContextIdx = 0, pageIdx = 0, openType = "other", openTime = currentTime, lastStatusUpdateTime = currentTime, taskId = 0, relatedId = 0, misc = {} } = pageInfo ? pageInfo : {};
     this.#page.pageInfo = { browserIdx, browserContextIdx, pageIdx, openType, openTime, lastStatusUpdateTime, taskId, relatedId, misc };
     this.#pageId = `page-${browserIdx}-${browserContextIdx}-${pageIdx}`;
+    this.#closeWhenFree = false;
     this.#resquestInterceptionOptions = [];
     this.#responseInterceptionOptions = [];
     this.#client = null;
     this.#responseCb = null;
     this.#addPageOn();
   }
+  async addPreloadScript(scriptOrFunc, arg) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    if (typeof scriptOrFunc === "string") {
+      await this.#page.addInitScript({ content: scriptOrFunc });
+    } else if (typeof scriptOrFunc === "function") {
+      await this.#page.addInitScript(scriptOrFunc, arg);
+    } else {
+      throw new Error(`Invalid type of scriptOrFunc ${typeof scriptOrFunc}`);
+    }
+    return true;
+  }
+  async addScriptTag(options) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    return this.#page.addScriptTag(options);
+  }
   apiContext() {
     return this.browserContext().apiContext();
   }
@@ -655,7 +746,7 @@ var PlaywrightPage = class extends import_node_events.default {
   }
   async close() {
     if (this.#status === "closed") {
-      console.warn(`Page ${this.#pageId} is already closed.`);
+      logwarn(`Page ${this.#pageId} is already closed.`);
       return true;
     } else if (this.#status === "busy") {
       throw new Error(`Page ${this.#pageId} cannot be closed because it is busy.`);
@@ -668,13 +759,16 @@ var PlaywrightPage = class extends import_node_events.default {
     this.#status = "closed";
     return true;
   }
+  closeWhenFree() {
+    return this.#closeWhenFree;
+  }
   async content(iframeOptions = []) {
     if (!this.#page) {
       throw new Error("No valid page");
     }
     let content = "";
     if (iframeOptions.length > 0) {
-      const frameLocator = await this.#getDescendantFrameLocator(this.#page.mainFrame(), iframeOptions);
+      const frameLocator = await this.#getDescendantFrame(this.#page.mainFrame(), iframeOptions);
       if (frameLocator) {
         content = await frameLocator.locator(":root").evaluate(() => document.documentElement.outerHTML);
       }
@@ -696,11 +790,18 @@ var PlaywrightPage = class extends import_node_events.default {
     const height = await this.#page.evaluate(() => document.documentElement.scrollHeight);
     return height;
   }
-  async evalute(fun, args) {
+  async evaluate(func, args) {
     if (!this.#page) {
       throw new Error("No valid page");
     }
-    return this.#page.evaluate(fun, args);
+    return this.#page.evaluate(func, args);
+  }
+  async exposeFunction(name, callbackFunction) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    await this.#page.exposeFunction(name, callbackFunction);
+    return;
   }
   async findElement(selectorOrXpath, iframeOptions = []) {
     if (!this.#page) {
@@ -738,7 +839,7 @@ var PlaywrightPage = class extends import_node_events.default {
   }
   async free() {
     if (this.#status === "free") {
-      console.warn(`Page ${this.#pageId} is already free.`);
+      logwarn(`Page ${this.#pageId} is already free.`);
     }
     this.#status = "free";
     await this.clearRequestInterceptions();
@@ -878,6 +979,10 @@ var PlaywrightPage = class extends import_node_events.default {
     }
     return response;
   }
+  setCloseWhenFree(closeWhenFree) {
+    this.#closeWhenFree = closeWhenFree;
+    return true;
+  }
   async setCookies(cookies) {
     if (!this.#page) {
       throw new Error("No valid page");
@@ -1061,7 +1166,7 @@ var PlaywrightPage = class extends import_node_events.default {
     }
     const actOptions = Array.isArray(options) ? options : [options];
     if (actOptions.length <= 0) {
-      console.warn("Invalid paras in setResponseInterception");
+      logwarn("Invalid paras in setResponseInterception");
       return false;
     }
     const firstResponseInterception = this.#responseInterceptionOptions.length <= 0;
@@ -1763,6 +1868,26 @@ var PuppeteerElement = class _PuppeteerElement {
     const names = await this.#frame.evaluate((ele) => ele.getAttributeNames(), this.#$ele);
     return names;
   }
+  async dataset() {
+    try {
+      const attributeNames = await this.attributeNames();
+      const dataset = {};
+      for (const attributeName of attributeNames) {
+        if (!attributeName.startsWith("data-")) {
+          continue;
+        }
+        const val = await this.attribute(attributeName);
+        const key = convertDataAttributeName(attributeName);
+        dataset[key] = val;
+      }
+      return dataset;
+    } catch (err) {
+      return {};
+    }
+  }
+  async evaluate(func, args) {
+    return await this.#frame.evaluate(func, args);
+  }
   async #getChildFrame(parentFrame, iframeOption) {
     if (!parentFrame) {
       throw new Error("Invalid parent frame");
@@ -1812,13 +1937,13 @@ var PuppeteerElement = class _PuppeteerElement {
     let frame = this.#frame;
     const retObj = { frame, elementHandles: [] };
     if (iframeOptions.length > 0) {
-      frame = await this.#getDescendantFrame(frame, iframeOptions);
-      if (!frame) {
+      const childFrame = await this.#getDescendantFrame(frame, iframeOptions);
+      if (!childFrame) {
         return retObj;
       }
-      retObj.frame = frame;
+      retObj.frame = childFrame;
       absolute = true;
-      parent = frame;
+      parent = childFrame;
     }
     try {
       if (selector.startsWith("./") || selector.startsWith("/") || selector.startsWith("..")) {
@@ -2020,6 +2145,7 @@ var PuppeteerPage = class extends import_node_events4.default {
   #page;
   #status;
   #pageId;
+  #closeWhenFree;
   #requestInterceptionNum;
   #responseInterceptionNum;
   #client;
@@ -2094,15 +2220,41 @@ var PuppeteerPage = class extends import_node_events4.default {
     });
     return true;
   }
+  async #findDescendantFrame(src, id) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    const frames = this.#page.frames();
+    for (const frame of frames) {
+      const url = frame.url();
+      if (typeof src === "string" && src) {
+        if (url.startsWith(src)) {
+          return frame;
+        } else if (url.toLowerCase().startsWith(src)) {
+          return frame;
+        }
+      } else if (src instanceof RegExp) {
+        if (url.match(src)) {
+          return frame;
+        }
+      } else if (id) {
+        const element = await frame.frameElement();
+        if (element) {
+          const frameId = await frame.evaluate((ele, attr) => ele.getAttribute(attr), element, "id");
+          if (frameId === id) {
+            return frame;
+          }
+        }
+      }
+    }
+    return null;
+  }
   async #getChildFrame(parentFrame, iframeOption) {
     if (!parentFrame) {
       throw new Error("Invalid parent frame");
     }
     let iframe = null;
-    let { src = "", selector = "" } = iframeOption;
-    if (!src && !selector) {
-      throw new Error("Invalid IframeOption");
-    }
+    let { src = "" } = iframeOption;
     if (src) {
       const childFrames = parentFrame.childFrames();
       for (const childFrame of childFrames) {
@@ -2120,7 +2272,8 @@ var PuppeteerPage = class extends import_node_events4.default {
         }
       }
     } else {
-      const $eleIframe = await parentFrame.$(selector);
+      const frameSelector = getIframeSelector(iframeOption);
+      const $eleIframe = await parentFrame.$(frameSelector);
       if ($eleIframe) {
         iframe = await $eleIframe.contentFrame();
         return iframe;
@@ -2130,11 +2283,16 @@ var PuppeteerPage = class extends import_node_events4.default {
   }
   async #getDescendantFrame(parentFrame, iframeOptions) {
     let iframe = parentFrame;
-    for (const iframeOption of iframeOptions) {
-      if (!iframe) {
-        return null;
+    if (iframeOptions.length === 1 && !iframeOptions[0].selector) {
+      const { src = "", id = "" } = iframeOptions[0];
+      iframe = await this.#findDescendantFrame(src, id);
+    } else {
+      for (const iframeOption of iframeOptions) {
+        if (!iframe) {
+          return null;
+        }
+        iframe = await this.#getChildFrame(iframe, iframeOption);
       }
-      iframe = await this.#getChildFrame(iframe, iframeOption);
     }
     return iframe;
   }
@@ -2163,7 +2321,7 @@ var PuppeteerPage = class extends import_node_events4.default {
       }
       return retObj;
     } catch (err) {
-      console.log(err);
+      loginfo(err);
       return retObj;
     }
   }
@@ -2216,11 +2374,31 @@ var PuppeteerPage = class extends import_node_events4.default {
     const { browserIdx = 0, browserContextIdx = 0, pageIdx = 0, openType = "other", openTime = currentTime, lastStatusUpdateTime = currentTime, taskId = 0, relatedId = 0, misc = {} } = pageInfo ? pageInfo : {};
     this.#page.pageInfo = { browserIdx, browserContextIdx, pageIdx, openType, openTime, lastStatusUpdateTime, taskId, relatedId, misc };
     this.#pageId = `page-${browserIdx}-${browserContextIdx}-${pageIdx}`;
+    this.#closeWhenFree = false;
     this.#requestInterceptionNum = 0;
     this.#responseInterceptionNum = 0;
     this.#client = null;
     this.#addPageOn();
   }
+  async addPreloadScript(scriptOrFunc, arg) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    if (typeof scriptOrFunc === "string") {
+      await this.#page.evaluateOnNewDocument(scriptOrFunc);
+    } else if (typeof scriptOrFunc === "function") {
+      await this.#page.evaluateOnNewDocument(scriptOrFunc, arg);
+    } else {
+      throw new Error(`Invalid type of scriptOrFunc ${typeof scriptOrFunc}`);
+    }
+    return true;
+  }
+  async addScriptTag(options) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    return this.#page.addScriptTag(options);
+  }
   apiContext() {
     throw new Error("Not supported in PuppeteerPage.");
   }
@@ -2277,7 +2455,7 @@ var PuppeteerPage = class extends import_node_events4.default {
   }
   async close() {
     if (this.#status === "closed") {
-      console.warn(`Page ${this.#pageId} is already closed.`);
+      logwarn(`Page ${this.#pageId} is already closed.`);
       return true;
     } else if (this.#status === "busy") {
       throw new Error(`Page ${this.#pageId} cannot be closed because it is busy.`);
@@ -2290,6 +2468,9 @@ var PuppeteerPage = class extends import_node_events4.default {
     this.#status = "closed";
     return true;
   }
+  closeWhenFree() {
+    return this.#closeWhenFree;
+  }
   async content(iframeOptions = []) {
     if (!this.#page) {
       throw new Error("No valid page");
@@ -2318,11 +2499,18 @@ var PuppeteerPage = class extends import_node_events4.default {
     const height = await this.#page.evaluate(() => document.documentElement.scrollHeight);
     return height;
   }
-  async evalute(fun, args) {
+  async evaluate(func, args) {
+    if (!this.#page) {
+      throw new Error("No valid page");
+    }
+    return this.#page.evaluate(func, args);
+  }
+  async exposeFunction(name, callbackFunction) {
     if (!this.#page) {
       throw new Error("No valid page");
     }
-    return this.#page.evaluate(fun, args);
+    await this.#page.exposeFunction(name, callbackFunction);
+    return;
   }
   async findElement(selectorOrXpath, iframeOptions = []) {
     if (!this.#page) {
@@ -2360,7 +2548,7 @@ var PuppeteerPage = class extends import_node_events4.default {
   }
   async free() {
     if (this.#status === "free") {
-      console.warn(`Page ${this.#pageId} is already free.`);
+      logwarn(`Page ${this.#pageId} is already free.`);
     }
     this.#status = "free";
     await this.clearRequestInterceptions();
@@ -2503,6 +2691,10 @@ var PuppeteerPage = class extends import_node_events4.default {
     }
     return response;
   }
+  setCloseWhenFree(closeWhenFree) {
+    this.#closeWhenFree = closeWhenFree;
+    return true;
+  }
   async setCookies(cookies) {
     if (!this.#page) {
       throw new Error("No valid page");
@@ -2578,7 +2770,7 @@ var PuppeteerPage = class extends import_node_events4.default {
     }
     const actOptions = Array.isArray(options) ? options : [options];
     if (actOptions.length <= 0) {
-      console.warn("Invalid paras in setRequestInterception");
+      logwarn("Invalid paras in setRequestInterception");
       return false;
     }
     if (this.#requestInterceptionNum <= 0) {
@@ -2614,7 +2806,7 @@ var PuppeteerPage = class extends import_node_events4.default {
         await request.continue();
         return true;
       } catch (err) {
-        console.error(err);
+        logerr(err);
         return false;
       }
     });
@@ -2626,7 +2818,7 @@ var PuppeteerPage = class extends import_node_events4.default {
     }
     const actOptions = Array.isArray(options) ? options : [options];
     if (actOptions.length <= 0) {
-      console.warn("Invalid paras in setResponseInterception");
+      logwarn("Invalid paras in setResponseInterception");
       return false;
     }
     this.#responseInterceptionNum++;
@@ -2681,7 +2873,7 @@ var PuppeteerPage = class extends import_node_events4.default {
         }
         return true;
       } catch (err) {
-        console.error(err);
+        logerr(err);
         return false;
       }
     });
@@ -3341,6 +3533,22 @@ var CheerioElement = class _CheerioElement {
       return Array.from(Object.keys(element.attribs));
     }
   }
+  async dataset() {
+    const attributeNames = await this.attributeNames();
+    const dataset = {};
+    for (const attributeName of attributeNames) {
+      if (!attributeName.startsWith("data-")) {
+        continue;
+      }
+      const val = await this.attribute(attributeName);
+      const key = convertDataAttributeName(attributeName);
+      dataset[key] = val;
+    }
+    return dataset;
+  }
+  async evaluate() {
+    throw new Error("Not supported in CheerioPage.");
+  }
   #findNodes(selector, absolute) {
     if (selector.startsWith("./") || selector.startsWith("/")) {
       throw new Error("Do not support XPath in cheerio.");
@@ -3461,6 +3669,12 @@ var CheerioPage = class extends import_node_events7.default {
   _origPage() {
     throw new Error("Method not implemented.");
   }
+  async addPreloadScript() {
+    throw new Error("Not supported in CheerioPage.");
+  }
+  addScriptTag() {
+    throw new Error("Not supported in CheerioPage.");
+  }
   apiContext() {
     throw new Error("Not supported in CheerioPage.");
   }
@@ -3488,13 +3702,19 @@ var CheerioPage = class extends import_node_events7.default {
   async close() {
     throw new Error("Not supported in CheerioPage.");
   }
+  closeWhenFree() {
+    throw new Error("Not supported in CheerioPage.");
+  }
   async content() {
     throw new Error("Not supported in CheerioPage.");
   }
   async cookies() {
     throw new Error("Not supported in CheerioPage.");
   }
-  async evalute() {
+  async evaluate() {
+    throw new Error("Not supported in CheerioPage.");
+  }
+  exposeFunction() {
     throw new Error("Not supported in CheerioPage.");
   }
   #findNodes(selector) {
@@ -3591,6 +3811,9 @@ var CheerioPage = class extends import_node_events7.default {
   async sendCDPMessage() {
     throw new Error("Method not implemented.");
   }
+  setCloseWhenFree() {
+    throw new Error("Not supported in CheerioPage.");
+  }
   async setCookies() {
     throw new Error("Not supported in CheerioPage.");
   }