@blocklet/crawler 2.1.231 → 2.1.233

package/lib/cjs/middlewares/crawler.js

@@ -32,7 +32,9 @@ const getPageContent = async ({
   let page = await (0, _utils.initPage)();
   let pageContent = null;
   try {
-    const response = await page.goto(url);
+    const response = await page.goto(url, {
+      timeout: 20 * 1e3
+    });
     const statusCode = response.status();
     if (![200, 304].includes(statusCode)) {
       throw new Error(`Request failed with status ${statusCode}, in ${url}`);
@@ -231,18 +233,14 @@ const crawlBlocklet = async () => {
     }
   });
   _utils.logger.info(...crawlerLogText("success"));
-  await _utils.puppeteer.trimCache();
-  _utils.logger.info("Puppeteer trim cache success");
-  if (global.gc) {
-    global.gc();
-  }
+  await (0, _utils.closeBrowser)();
 };
 exports.crawlBlocklet = crawlBlocklet;
 const CRON_CRAWL_BLOCKLET_KEY = "cron-crawl-blocklet";
 let cronCrawlBlockletJob = null;
 const initCronCrawlBlocklet = ({
-  time = "0 0 */12 * * *",
-  // every 12 hours
+  time = "0 0 */24 * * *",
+  // every 24 hours
   options
 } = {}) => {
   if (!cronCrawlBlockletJob) {

package/lib/cjs/middlewares/utils.js

@@ -97,7 +97,11 @@ exports.clearCheckBrowserTimer = clearCheckBrowserTimer;
 const closeBrowser = async () => {
   try {
     if (browser) {
-      await browser.close();
+      const pages = await browser.pages().catch(() => []);
+      await Promise.all(pages.map(page => page.close().catch(() => {})));
+      await browser.close().catch(err => {
+        logger.warn("Browser close failed with error:", err);
+      });
       browser = null;
       await useCache.remove(BROWSER_WS_ENDPOINT_KEY);
     }
@@ -107,7 +111,9 @@ const closeBrowser = async () => {
     if (global.gc) {
       global.gc();
     }
-  } catch (error) {}
+  } catch (error) {
+    logger.error("Failed to close browser:", error);
+  }
 };
 exports.closeBrowser = closeBrowser;
 const getBrowser = async () => {
@@ -134,16 +140,17 @@ const getBrowser = async () => {
       try {
         browser = await _puppeteer.default.launch({
           headless: true,
-          // stable headless
-          // headless: false, // debug
-          // dumpio: true,
           args: [
           // docs: https://peter.sh/experiments/chromium-command-line-switches/
           "--no-first-run",
           // '--no-startup-window',
           "--hide-scrollbars", "--no-sandbox", "--no-zygote",
           // '--single-process',
-          "--disable-setuid-sandbox", "--disable-gpu", "--disable-dev-shm-usage", "--disable-site-isolation-trials", "--disable-accelerated-2d-canvas"]
+          "--disable-setuid-sandbox", "--disable-gpu", "--disable-dev-shm-usage", "--disable-site-isolation-trials", "--disable-accelerated-2d-canvas", "--disable-extensions", "--js-flags=--max_old_space_size=512",
+          // 限制V8内存
+          "--disable-background-networking", "--disable-default-apps", "--disable-web-security",
+          // 允许跨域请求
+          "--disable-software-rasterizer", "--disable-crash-reporter"]
         });
         logger.info("Launch browser success");
         const browserWSEndpoint = await browser.wsEndpoint();
@@ -158,11 +165,15 @@ const getBrowser = async () => {
         let count = 0;
         checkBrowserTimer = setInterval(async () => {
           if (browser) {
-            const pages = await browser.pages();
+            const pages = await browser.pages().catch(() => []);
             if (pages.length === 1 && pages[0].url() === "about:blank") {
               count++;
+              logger.debug(`Browser inactive count: ${count}/3`);
+            } else {
+              count = 0;
             }
             if (count >= 3) {
+              logger.info("Browser inactive for 3 minutes, closing...");
               await closeBrowser();
             }
           }
@@ -398,8 +409,9 @@ const cachePool = exports.cachePool = (0, _genericPool.createPool)({
     }
   }
 }, {
-  max: 10,
-  min: 1
+  max: 2,
+  // 2 clients
+  min: 0
   // evictionRunIntervalMillis: 0,
 });
 const withCache = async cb => {

package/lib/es/middlewares/crawler.js

@@ -8,7 +8,6 @@ import {
   isAcceptCrawler,
   env,
   components,
-  puppeteer,
   getComponentInfo,
   sleep,
   closeBrowser,
@@ -31,7 +30,9 @@ export const getPageContent = async ({ url, formatPageContent }) => {
   let page = await initPage();
   let pageContent = null;
   try {
-    const response = await page.goto(url);
+    const response = await page.goto(url, {
+      timeout: 20 * 1e3
+    });
     const statusCode = response.status();
     if (![200, 304].includes(statusCode)) {
       throw new Error(`Request failed with status ${statusCode}, in ${url}`);
@@ -212,17 +213,13 @@ export const crawlBlocklet = async () => {
     }
   });
   logger.info(...crawlerLogText("success"));
-  await puppeteer.trimCache();
-  logger.info("Puppeteer trim cache success");
-  if (global.gc) {
-    global.gc();
-  }
+  await closeBrowser();
 };
 const CRON_CRAWL_BLOCKLET_KEY = "cron-crawl-blocklet";
 let cronCrawlBlockletJob = null;
 export const initCronCrawlBlocklet = ({
-  time = "0 0 */12 * * *",
-  // every 12 hours
+  time = "0 0 */24 * * *",
+  // every 24 hours
   options
 } = {}) => {
   if (!cronCrawlBlockletJob) {

package/lib/es/middlewares/utils.js

@@ -41,7 +41,12 @@ export const clearCheckBrowserTimer = () => {
 export const closeBrowser = async () => {
   try {
     if (browser) {
-      await browser.close();
+      const pages = await browser.pages().catch(() => []);
+      await Promise.all(pages.map((page) => page.close().catch(() => {
+      })));
+      await browser.close().catch((err) => {
+        logger.warn("Browser close failed with error:", err);
+      });
       browser = null;
       await useCache.remove(BROWSER_WS_ENDPOINT_KEY);
     }
@@ -52,6 +57,7 @@ export const closeBrowser = async () => {
       global.gc();
     }
   } catch (error) {
+    logger.error("Failed to close browser:", error);
   }
 };
 export const getBrowser = async () => {
@@ -78,9 +84,6 @@ export const getBrowser = async () => {
       try {
         browser = await puppeteer.launch({
           headless: true,
-          // stable headless
-          // headless: false, // debug
-          // dumpio: true,
           args: [
             // docs: https://peter.sh/experiments/chromium-command-line-switches/
             "--no-first-run",
@@ -93,7 +96,16 @@ export const getBrowser = async () => {
             "--disable-gpu",
             "--disable-dev-shm-usage",
             "--disable-site-isolation-trials",
-            "--disable-accelerated-2d-canvas"
+            "--disable-accelerated-2d-canvas",
+            "--disable-extensions",
+            "--js-flags=--max_old_space_size=512",
+            // 限制V8内存
+            "--disable-background-networking",
+            "--disable-default-apps",
+            "--disable-web-security",
+            // 允许跨域请求
+            "--disable-software-rasterizer",
+            "--disable-crash-reporter"
           ]
         });
         logger.info("Launch browser success");
@@ -109,11 +121,15 @@ export const getBrowser = async () => {
         let count = 0;
         checkBrowserTimer = setInterval(async () => {
           if (browser) {
-            const pages = await browser.pages();
+            const pages = await browser.pages().catch(() => []);
             if (pages.length === 1 && pages[0].url() === "about:blank") {
               count++;
+              logger.debug(`Browser inactive count: ${count}/3`);
+            } else {
+              count = 0;
             }
             if (count >= 3) {
+              logger.info("Browser inactive for 3 minutes, closing...");
               await closeBrowser();
             }
           }
@@ -423,8 +439,9 @@ export const cachePool = createPool(
     }
   },
   {
-    max: 10,
-    min: 1
+    max: 2,
+    // 2 clients
+    min: 0
     // evictionRunIntervalMillis: 0,
   }
 );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blocklet/crawler",
-  "version": "2.1.231",
+  "version": "2.1.233",
   "description": "blocklet crawler lib",
   "publishConfig": {
     "access": "public"