@blocklet/crawler 2.1.219 → 2.1.221

package/lib/cjs/middlewares/utils.js

@@ -203,7 +203,9 @@ const initPage = async ({
   return page;
 };
 exports.initPage = initPage;
-const botUserAgents = [/bot/i, /spider/i, /facebookexternalhit/i, /simplepie/i, /yahooseeker/i, /embedly/i, /quora link preview/i, /outbrain/i, /vkshare/i, /monit/i, /Pingability/i, /Monitoring/i, /WinHttpRequest/i, /Apache-HttpClient/i, /getprismatic.com/i, /python-requests/i, /Twurly/i, /yandex/i, /browserproxy/i, /crawler/i, /Qwantify/i, /Yahoo/i, /pinterest/i, /Tumblr/i, /Tumblr Agent/i, /WhatsApp/i, /Google-Structured-Data-Testing-Tool/i, /Google-InspectionTool/i, /Googlebot/i, /GPTBot/i, /Applebot/i];
+const botUserAgents = [/bot/i, /spider/i, /facebookexternalhit/i, /simplepie/i, /yahooseeker/i, /embedly/i, /quora link preview/i, /outbrain/i, /vkshare/i, /monit/i, /Pingability/i, /Monitoring/i, /WinHttpRequest/i, /Apache-HttpClient/i, /getprismatic.com/i, /python-requests/i, /Twurly/i, /yandex/i, /browserproxy/i, /crawler/i, /Qwantify/i, /Yahoo/i, /pinterest/i, /Tumblr/i, /Tumblr Agent/i, /WhatsApp/i, /Google-Structured-Data-Testing-Tool/i, /Google-InspectionTool/i, /Googlebot/i, /GPTBot/i, /Applebot/i,
+// AI bots
+/Anthropic-ai/i, /Claude-Web/i, /anthropic-ai-scraper/i, /Google-Extended/i, /GoogleOther/i, /CCBot\/\d/i, /Bytespider/i, /BingBot/i, /Baiduspider/i, /Sogou/i, /Perplexity/i, /Cohere-ai/i, /xlts-bot/i, /THAAS/i, /YisouSpider/i, /AlibabaGroup/i, /adaptive-edge-crawler/i];
 const isSpider = ua => botUserAgents.some(spider => {
   return spider.test(ua);
 });
@@ -386,16 +388,35 @@ const useCache = exports.useCache = {
 };
 const checkBrowserDownloaded = async () => {
   try {
+    const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH || "/usr/bin/chromium";
+    if (_fsExtra.default.existsSync(executablePath)) {
+      try {
+        const puppeteer2 = await Promise.resolve().then(() => require("puppeteer"));
+        const browser2 = await puppeteer2.launch({
+          executablePath,
+          args: ["--no-sandbox", "--disable-setuid-sandbox"],
+          headless: true
+        });
+        await browser2.close();
+        logger.info(`System Chromium found and tested successfully: ${executablePath}`);
+        return;
+      } catch (err) {
+        logger.warn(`System Chromium exists but test failed, will try to download`);
+      }
+    }
     const {
       downloadBrowser
     } = await (async () => {
       try {
         return await Promise.resolve().then(() => require("@blocklet/puppeteer/internal/node/install.js"));
-      } catch {
+      } catch (err) {
         logger.warn("Skipping browser installation because the Puppeteer build is not available. Run `npm install` again after you have re-built Puppeteer.");
       }
     })();
-    await downloadBrowser();
+    if (downloadBrowser) {
+      await downloadBrowser();
+      logger.info("Browser download completed successfully");
+    }
   } catch (error) {
     logger.warn("Browser download failed", error);
   }

package/lib/es/middlewares/utils.js

@@ -179,7 +179,25 @@ const botUserAgents = [
   /Google-InspectionTool/i,
   /Googlebot/i,
   /GPTBot/i,
-  /Applebot/i
+  /Applebot/i,
+  // AI bots
+  /Anthropic-ai/i,
+  /Claude-Web/i,
+  /anthropic-ai-scraper/i,
+  /Google-Extended/i,
+  /GoogleOther/i,
+  /CCBot\/\d/i,
+  /Bytespider/i,
+  /BingBot/i,
+  /Baiduspider/i,
+  /Sogou/i,
+  /Perplexity/i,
+  /Cohere-ai/i,
+  /xlts-bot/i,
+  /THAAS/i,
+  /YisouSpider/i,
+  /AlibabaGroup/i,
+  /adaptive-edge-crawler/i
 ];
 const isSpider = (ua) => botUserAgents.some((spider) => {
   return spider.test(ua);
@@ -386,16 +404,35 @@ export const useCache = {
 };
 export const checkBrowserDownloaded = async () => {
   try {
+    const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH || "/usr/bin/chromium";
+    if (fs.existsSync(executablePath)) {
+      try {
+        const puppeteer2 = await import("puppeteer");
+        const browser2 = await puppeteer2.launch({
+          executablePath,
+          args: ["--no-sandbox", "--disable-setuid-sandbox"],
+          headless: true
+        });
+        await browser2.close();
+        logger.info(`System Chromium found and tested successfully: ${executablePath}`);
+        return;
+      } catch (err) {
+        logger.warn(`System Chromium exists but test failed, will try to download`);
+      }
+    }
     const { downloadBrowser } = await (async () => {
       try {
         return await import("@blocklet/puppeteer/internal/node/install.js");
-      } catch {
+      } catch (err) {
         logger.warn(
           "Skipping browser installation because the Puppeteer build is not available. Run `npm install` again after you have re-built Puppeteer."
         );
       }
     })();
-    await downloadBrowser();
+    if (downloadBrowser) {
+      await downloadBrowser();
+      logger.info("Browser download completed successfully");
+    }
   } catch (error) {
     logger.warn("Browser download failed", error);
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blocklet/crawler",
-  "version": "2.1.219",
+  "version": "2.1.221",
   "description": "blocklet crawler lib",
   "publishConfig": {
     "access": "public"