@blocklet/crawler 2.2.11 → 2.2.13

package/lib/cjs/middlewares/crawler.js

@@ -40,8 +40,12 @@ const getPageContent = async ({
   let pageContent = null;
   try {
     const response = await page.goto(url, {
-      timeout: 20 * 1e3
+      timeout: 60 * 1e3
+      // 60s
     });
+    if (!response) {
+      throw new Error(`Failed to load page: response is null for ${url}`);
+    }
     const statusCode = response.status();
     if (![200, 304].includes(statusCode)) {
       throw new Error(`Request failed with status ${statusCode}, in ${url}`);
@@ -127,7 +131,7 @@ const crawlUrl = async ({
       try {
         if (index % autoCloseBrowserCount === 0) {
           await (0, _utils.closeBrowser)({
-            trimCache: index % (autoCloseBrowserCount * 5) === 0
+            trimCache: false
           });
         }
         const canCrawl = await (0, _utils.isAcceptCrawler)(url);
@@ -180,7 +184,16 @@ const crawlUrl = async ({
   }
 };
 exports.crawlUrl = crawlUrl;
+const crawlBlockletRunningMap = /* @__PURE__ */new Map();
 const crawlBlocklet = async () => {
+  const {
+    mountPoint,
+    did
+  } = (0, _utils.getComponentInfo)();
+  if (crawlBlockletRunningMap.has(did) && crawlBlockletRunningMap.get(did)) {
+    _utils.logger.info(`Crawler blocklet ${did} is running, skip it`);
+    return;
+  }
   try {
     const browser = await (0, _utils.getBrowser)();
     if (!browser) {
@@ -197,10 +210,6 @@ const crawlBlocklet = async () => {
   if (!appUrl) {
     throw new Error("appUrl not found");
   }
-  const {
-    mountPoint,
-    did
-  } = (0, _utils.getComponentInfo)();
   const sitemapList = await (0, _utils.getSitemapList)(appUrl);
   const matchMountPoint = (0, _ufo.joinURL)(appUrl, !mountPoint || mountPoint === "/" ? "" : mountPoint);
   const otherMountPointList = _utils.components.filter(item => item.mountPoint && item.mountPoint !== mountPoint).map(item => item.mountPoint);
@@ -253,39 +262,46 @@ const crawlBlocklet = async () => {
     lastmodMapTotal: lastmodMap.size
   }];
   _utils.logger.info(...crawlerLogText("start"));
-  await crawlUrl({
-    // @ts-ignore
-    urls: canUseBlockletLocList,
-    lastmodMap,
-    formatPageContent: async ({
-      page,
-      url
-    }) => {
-      const pageContent = await page.evaluate(() => {
-        const removeElements = tagName => {
-          const elements = document.querySelectorAll(tagName);
-          for (let i = elements.length - 1; i >= 0; i--) {
-            try {
-              elements[i]?.parentNode?.removeChild(elements[i]);
-            } catch (error) {}
-          }
-        };
-        removeElements('[id="uploader-container"]');
-        removeElements('[class^="uppy-"]');
-        removeElements('[id="point-up-component"]');
-        const meta = document.createElement("meta");
-        meta.name = "blocklet-crawler";
-        meta.content = "true";
-        document.head.appendChild(meta);
-        return document.documentElement.outerHTML;
-      });
-      return pageContent;
-    }
-  });
-  _utils.logger.info(...crawlerLogText("success"));
-  await (0, _utils.closeBrowser)({
-    trimCache: true
-  });
+  try {
+    crawlBlockletRunningMap.set(did, true);
+    await crawlUrl({
+      // @ts-ignore
+      urls: canUseBlockletLocList,
+      lastmodMap,
+      formatPageContent: async ({
+        page,
+        url
+      }) => {
+        const pageContent = await page.evaluate(() => {
+          const removeElements = tagName => {
+            const elements = document.querySelectorAll(tagName);
+            for (let i = elements.length - 1; i >= 0; i--) {
+              try {
+                elements[i]?.parentNode?.removeChild(elements[i]);
+              } catch (error) {}
+            }
+          };
+          removeElements('[id="uploader-container"]');
+          removeElements('[class^="uppy-"]');
+          removeElements('[id="point-up-component"]');
+          const meta = document.createElement("meta");
+          meta.name = "blocklet-crawler";
+          meta.content = "true";
+          document.head.appendChild(meta);
+          return document.documentElement.outerHTML;
+        });
+        return pageContent;
+      }
+    });
+    _utils.logger.info(...crawlerLogText("success"));
+    await (0, _utils.closeBrowser)({
+      trimCache: true
+    });
+  } catch (error) {
+    _utils.logger.info(`Crawler blocklet abort by error`, error);
+  } finally {
+    crawlBlockletRunningMap.delete(did);
+  }
 };
 exports.crawlBlocklet = crawlBlocklet;
 const CRON_CRAWL_BLOCKLET_KEY = "cron-crawl-blocklet";

package/lib/cjs/middlewares/utils.js

@@ -121,6 +121,7 @@ const closeBrowser = async ({
       if (temporaryDirectory) {
         _fsExtra.default.emptyDirSync(temporaryDirectory);
       }
+      logger.info("Trim cache success");
     }
     logger.info("Close browser success");
     if (global.gc) {

package/lib/es/middlewares/crawler.js

@@ -32,8 +32,12 @@ export const getPageContent = async ({ url, formatPageContent }) => {
   let pageContent = null;
   try {
     const response = await page.goto(url, {
-      timeout: 20 * 1e3
+      timeout: 60 * 1e3
+      // 60s
     });
+    if (!response) {
+      throw new Error(`Failed to load page: response is null for ${url}`);
+    }
     const statusCode = response.status();
     if (![200, 304].includes(statusCode)) {
       throw new Error(`Request failed with status ${statusCode}, in ${url}`);
@@ -112,7 +116,7 @@ export const crawlUrl = async ({
       try {
         if (index % autoCloseBrowserCount === 0) {
           await closeBrowser({
-            trimCache: index % (autoCloseBrowserCount * 5) === 0
+            trimCache: false
           });
         }
         const canCrawl = await isAcceptCrawler(url);
@@ -157,7 +161,13 @@ export const crawlUrl = async ({
     await crawlQueue.add(crawlUrlJob({ url, index: index + 1 }));
   }
 };
+const crawlBlockletRunningMap = /* @__PURE__ */ new Map();
 export const crawlBlocklet = async () => {
+  const { mountPoint, did } = getComponentInfo();
+  if (crawlBlockletRunningMap.has(did) && crawlBlockletRunningMap.get(did)) {
+    logger.info(`Crawler blocklet ${did} is running, skip it`);
+    return;
+  }
   try {
     const browser = await getBrowser();
     if (!browser) {
@@ -172,7 +182,6 @@ export const crawlBlocklet = async () => {
   if (!appUrl) {
     throw new Error("appUrl not found");
   }
-  const { mountPoint, did } = getComponentInfo();
   const sitemapList = await getSitemapList(appUrl);
   const matchMountPoint = joinURL(appUrl, !mountPoint || mountPoint === "/" ? "" : mountPoint);
   const otherMountPointList = components.filter((item) => item.mountPoint && item.mountPoint !== mountPoint).map((item) => item.mountPoint);
@@ -230,37 +239,44 @@ export const crawlBlocklet = async () => {
     }
   ];
   logger.info(...crawlerLogText("start"));
-  await crawlUrl({
-    // @ts-ignore
-    urls: canUseBlockletLocList,
-    lastmodMap,
-    formatPageContent: async ({ page, url }) => {
-      const pageContent = await page.evaluate(() => {
-        const removeElements = (tagName) => {
-          const elements = document.querySelectorAll(tagName);
-          for (let i = elements.length - 1; i >= 0; i--) {
-            try {
-              elements[i]?.parentNode?.removeChild(elements[i]);
-            } catch (error) {
+  try {
+    crawlBlockletRunningMap.set(did, true);
+    await crawlUrl({
+      // @ts-ignore
+      urls: canUseBlockletLocList,
+      lastmodMap,
+      formatPageContent: async ({ page, url }) => {
+        const pageContent = await page.evaluate(() => {
+          const removeElements = (tagName) => {
+            const elements = document.querySelectorAll(tagName);
+            for (let i = elements.length - 1; i >= 0; i--) {
+              try {
+                elements[i]?.parentNode?.removeChild(elements[i]);
+              } catch (error) {
+              }
             }
-          }
-        };
-        removeElements('[id="uploader-container"]');
-        removeElements('[class^="uppy-"]');
-        removeElements('[id="point-up-component"]');
-        const meta = document.createElement("meta");
-        meta.name = "blocklet-crawler";
-        meta.content = "true";
-        document.head.appendChild(meta);
-        return document.documentElement.outerHTML;
-      });
-      return pageContent;
-    }
-  });
-  logger.info(...crawlerLogText("success"));
-  await closeBrowser({
-    trimCache: true
-  });
+          };
+          removeElements('[id="uploader-container"]');
+          removeElements('[class^="uppy-"]');
+          removeElements('[id="point-up-component"]');
+          const meta = document.createElement("meta");
+          meta.name = "blocklet-crawler";
+          meta.content = "true";
+          document.head.appendChild(meta);
+          return document.documentElement.outerHTML;
+        });
+        return pageContent;
+      }
+    });
+    logger.info(...crawlerLogText("success"));
+    await closeBrowser({
+      trimCache: true
+    });
+  } catch (error) {
+    logger.info(`Crawler blocklet abort by error`, error);
+  } finally {
+    crawlBlockletRunningMap.delete(did);
+  }
 };
 const CRON_CRAWL_BLOCKLET_KEY = "cron-crawl-blocklet";
 let cronCrawlBlockletJob = null;

package/lib/es/middlewares/utils.js

@@ -61,6 +61,7 @@ export const closeBrowser = async ({ trimCache = true } = {}) => {
       if (temporaryDirectory) {
         fs.emptyDirSync(temporaryDirectory);
       }
+      logger.info("Trim cache success");
     }
     logger.info("Close browser success");
     if (global.gc) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blocklet/crawler",
-  "version": "2.2.11",
+  "version": "2.2.13",
   "description": "blocklet crawler lib",
   "publishConfig": {
     "access": "public"