@blocklet/crawler 2.1.232 → 2.1.234

package/lib/cjs/middlewares/crawler.js

@@ -32,7 +32,9 @@ const getPageContent = async ({
   let page = await (0, _utils.initPage)();
   let pageContent = null;
   try {
-    const response = await page.goto(url);
+    const response = await page.goto(url, {
+      timeout: 20 * 1e3
+    });
     const statusCode = response.status();
     if (![200, 304].includes(statusCode)) {
       throw new Error(`Request failed with status ${statusCode}, in ${url}`);
@@ -231,18 +233,14 @@ const crawlBlocklet = async () => {
     }
   });
   _utils.logger.info(...crawlerLogText("success"));
-  await _utils.puppeteer.trimCache();
-  _utils.logger.info("Puppeteer trim cache success");
-  if (global.gc) {
-    global.gc();
-  }
+  await (0, _utils.closeBrowser)();
 };
 exports.crawlBlocklet = crawlBlocklet;
 const CRON_CRAWL_BLOCKLET_KEY = "cron-crawl-blocklet";
 let cronCrawlBlockletJob = null;
 const initCronCrawlBlocklet = ({
-  time = "0 0 */12 * * *",
-  // every 12 hours
+  time = "0 0 */24 * * *",
+  // every 24 hours
   options
 } = {}) => {
   if (!cronCrawlBlockletJob) {

package/lib/cjs/middlewares/utils.js

@@ -63,6 +63,7 @@ Object.keys(_config).forEach(function (key) {
 });
 var _util = require("util");
 var _child_process = require("child_process");
+var _component = require("@blocklet/sdk/lib/component");
 function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
 function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
 function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
@@ -97,7 +98,11 @@ exports.clearCheckBrowserTimer = clearCheckBrowserTimer;
 const closeBrowser = async () => {
   try {
     if (browser) {
-      await browser.close();
+      const pages = await browser.pages().catch(() => []);
+      await Promise.all(pages.map(page => page.close().catch(() => {})));
+      await browser.close().catch(err => {
+        logger.warn("Browser close failed with error:", err);
+      });
       browser = null;
       await useCache.remove(BROWSER_WS_ENDPOINT_KEY);
     }
@@ -107,7 +112,9 @@ const closeBrowser = async () => {
     if (global.gc) {
       global.gc();
     }
-  } catch (error) {}
+  } catch (error) {
+    logger.error("Failed to close browser:", error);
+  }
 };
 exports.closeBrowser = closeBrowser;
 const getBrowser = async () => {
@@ -134,16 +141,17 @@ const getBrowser = async () => {
       try {
         browser = await _puppeteer.default.launch({
           headless: true,
-          // stable headless
-          // headless: false, // debug
-          // dumpio: true,
           args: [
           // docs: https://peter.sh/experiments/chromium-command-line-switches/
           "--no-first-run",
           // '--no-startup-window',
           "--hide-scrollbars", "--no-sandbox", "--no-zygote",
           // '--single-process',
-          "--disable-setuid-sandbox", "--disable-gpu", "--disable-dev-shm-usage", "--disable-site-isolation-trials", "--disable-accelerated-2d-canvas"]
+          "--disable-setuid-sandbox", "--disable-gpu", "--disable-dev-shm-usage", "--disable-site-isolation-trials", "--disable-accelerated-2d-canvas", "--disable-extensions", "--js-flags=--max_old_space_size=512",
+          // 限制V8内存
+          "--disable-background-networking", "--disable-default-apps", "--disable-web-security",
+          // 允许跨域请求
+          "--disable-software-rasterizer", "--disable-crash-reporter"]
         });
         logger.info("Launch browser success");
         const browserWSEndpoint = await browser.wsEndpoint();
@@ -158,11 +166,15 @@ const getBrowser = async () => {
         let count = 0;
         checkBrowserTimer = setInterval(async () => {
           if (browser) {
-            const pages = await browser.pages();
+            const pages = await browser.pages().catch(() => []);
             if (pages.length === 1 && pages[0].url() === "about:blank") {
               count++;
+              logger.debug(`Browser inactive count: ${count}/3`);
+            } else {
+              count = 0;
             }
             if (count >= 3) {
+              logger.info("Browser inactive for 3 minutes, closing...");
               await closeBrowser();
             }
           }
@@ -338,6 +350,19 @@ async function detectBestRedisUrl() {
   const possibleUrls = [
   // environment variable priority
   process.env.REDIS_URL,
+  // by web component endpoint
+  (() => {
+    try {
+      const endpoint = (0, _component.getComponentWebEndpoint)(_config.default.env.componentDid);
+      if (endpoint) {
+        const url = new URL(endpoint);
+        return `redis://${url.hostname}:6379`;
+      }
+    } catch (err) {
+      logger.warn(`Failed to get component endpoint: ${err.message}`);
+    }
+    return null;
+  })(),
   // default gateway
   defaultGateway ? `redis://${defaultGateway}:6379` : null,
   // common Docker gateway
@@ -354,7 +379,7 @@ async function detectBestRedisUrl() {
       });
       testClient.on("error", () => {});
       await Promise.race([testClient.connect(), new Promise((_, reject) => setTimeout(() => reject(new Error("Connection timeout")), 2e3))]);
-      const pingResult = await testClient.ping();
+      await testClient.ping();
       await testClient.disconnect();
       logger.info(`\u2705 Found available Redis connection: ${url}`);
       cachedRedisUrl = url;
@@ -398,8 +423,9 @@ const cachePool = exports.cachePool = (0, _genericPool.createPool)({
     }
   }
 }, {
-  max: 10,
-  min: 1
+  max: 2,
+  // 2 clients
+  min: 0
   // evictionRunIntervalMillis: 0,
 });
 const withCache = async cb => {

package/lib/es/middlewares/crawler.js

@@ -8,7 +8,6 @@ import {
   isAcceptCrawler,
   env,
   components,
-  puppeteer,
   getComponentInfo,
   sleep,
   closeBrowser,
@@ -31,7 +30,9 @@ export const getPageContent = async ({ url, formatPageContent }) => {
   let page = await initPage();
   let pageContent = null;
   try {
-    const response = await page.goto(url);
+    const response = await page.goto(url, {
+      timeout: 20 * 1e3
+    });
     const statusCode = response.status();
     if (![200, 304].includes(statusCode)) {
       throw new Error(`Request failed with status ${statusCode}, in ${url}`);
@@ -212,17 +213,13 @@ export const crawlBlocklet = async () => {
     }
   });
   logger.info(...crawlerLogText("success"));
-  await puppeteer.trimCache();
-  logger.info("Puppeteer trim cache success");
-  if (global.gc) {
-    global.gc();
-  }
+  await closeBrowser();
 };
 const CRON_CRAWL_BLOCKLET_KEY = "cron-crawl-blocklet";
 let cronCrawlBlockletJob = null;
 export const initCronCrawlBlocklet = ({
-  time = "0 0 */12 * * *",
-  // every 12 hours
+  time = "0 0 */24 * * *",
+  // every 24 hours
   options
 } = {}) => {
   if (!cronCrawlBlockletJob) {

package/lib/es/middlewares/utils.js

@@ -13,6 +13,7 @@ import uniq from "lodash/uniq";
 import config from "@blocklet/sdk/lib/config";
 import { promisify } from "util";
 import { exec } from "child_process";
+import { getComponentWebEndpoint } from "@blocklet/sdk/lib/component";
 export * from "@blocklet/sdk/lib/config";
 const { logger } = config;
 const execAsync = promisify(exec);
@@ -41,7 +42,12 @@ export const clearCheckBrowserTimer = () => {
 export const closeBrowser = async () => {
   try {
     if (browser) {
-      await browser.close();
+      const pages = await browser.pages().catch(() => []);
+      await Promise.all(pages.map((page) => page.close().catch(() => {
+      })));
+      await browser.close().catch((err) => {
+        logger.warn("Browser close failed with error:", err);
+      });
       browser = null;
       await useCache.remove(BROWSER_WS_ENDPOINT_KEY);
     }
@@ -52,6 +58,7 @@ export const closeBrowser = async () => {
       global.gc();
     }
   } catch (error) {
+    logger.error("Failed to close browser:", error);
   }
 };
 export const getBrowser = async () => {
@@ -78,9 +85,6 @@ export const getBrowser = async () => {
       try {
         browser = await puppeteer.launch({
           headless: true,
-          // stable headless
-          // headless: false, // debug
-          // dumpio: true,
           args: [
             // docs: https://peter.sh/experiments/chromium-command-line-switches/
             "--no-first-run",
@@ -93,7 +97,16 @@ export const getBrowser = async () => {
             "--disable-gpu",
             "--disable-dev-shm-usage",
             "--disable-site-isolation-trials",
-            "--disable-accelerated-2d-canvas"
+            "--disable-accelerated-2d-canvas",
+            "--disable-extensions",
+            "--js-flags=--max_old_space_size=512",
+            // 限制V8内存
+            "--disable-background-networking",
+            "--disable-default-apps",
+            "--disable-web-security",
+            // 允许跨域请求
+            "--disable-software-rasterizer",
+            "--disable-crash-reporter"
           ]
         });
         logger.info("Launch browser success");
@@ -109,11 +122,15 @@ export const getBrowser = async () => {
         let count = 0;
         checkBrowserTimer = setInterval(async () => {
           if (browser) {
-            const pages = await browser.pages();
+            const pages = await browser.pages().catch(() => []);
             if (pages.length === 1 && pages[0].url() === "about:blank") {
               count++;
+              logger.debug(`Browser inactive count: ${count}/3`);
+            } else {
+              count = 0;
             }
             if (count >= 3) {
+              logger.info("Browser inactive for 3 minutes, closing...");
               await closeBrowser();
             }
           }
@@ -358,6 +375,19 @@ async function detectBestRedisUrl() {
   const possibleUrls = [
     // environment variable priority
     process.env.REDIS_URL,
+    // by web component endpoint
+    (() => {
+      try {
+        const endpoint = getComponentWebEndpoint(config.env.componentDid);
+        if (endpoint) {
+          const url = new URL(endpoint);
+          return `redis://${url.hostname}:6379`;
+        }
+      } catch (err) {
+        logger.warn(`Failed to get component endpoint: ${err.message}`);
+      }
+      return null;
+    })(),
     // default gateway
     defaultGateway ? `redis://${defaultGateway}:6379` : null,
     // common Docker gateway
@@ -377,7 +407,7 @@ async function detectBestRedisUrl() {
         testClient.connect(),
         new Promise((_, reject) => setTimeout(() => reject(new Error("Connection timeout")), 2e3))
       ]);
-      const pingResult = await testClient.ping();
+      await testClient.ping();
       await testClient.disconnect();
       logger.info(`\u2705 Found available Redis connection: ${url}`);
       cachedRedisUrl = url;
@@ -423,8 +453,9 @@ export const cachePool = createPool(
     }
   },
   {
-    max: 10,
-    min: 1
+    max: 2,
+    // 2 clients
+    min: 0
     // evictionRunIntervalMillis: 0,
   }
 );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blocklet/crawler",
-  "version": "2.1.232",
+  "version": "2.1.234",
   "description": "blocklet crawler lib",
   "publishConfig": {
     "access": "public"