@blocklet/crawler 2.1.229 → 2.1.231

package/lib/cjs/middlewares/utils.js

@@ -61,15 +61,19 @@ Object.keys(_config).forEach(function (key) {
     }
   });
 });
+var _util = require("util");
+var _child_process = require("child_process");
 function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
 function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
 function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 const {
   logger
 } = _config.default;
+const execAsync = (0, _util.promisify)(_child_process.exec);
 let browser;
 let checkBrowserTimer;
 const BROWSER_WS_ENDPOINT_KEY = `browserWSEndpoint-${process.env.BLOCKLET_DID || "unknown"}`;
+let cachedRedisUrl = null;
 const api = exports.api = _axios.default.create({
   timeout: 1e3 * 10,
   // 10s
@@ -310,11 +314,65 @@ const getRelativePath = url => {
 exports.getRelativePath = getRelativePath;
 let ttl = 1e3 * 60 * 60 * 24 * 7;
 const MAX_REDIS_RETRY = 3;
+async function getDefaultGateway() {
+  try {
+    const {
+      stdout
+    } = await execAsync("cat /proc/net/route | grep '^eth0' | head -1 | awk '{print $3}'");
+    if (stdout.trim()) {
+      const hex = stdout.trim();
+      const ip = [parseInt(hex.slice(6, 8), 16), parseInt(hex.slice(4, 6), 16), parseInt(hex.slice(2, 4), 16), parseInt(hex.slice(0, 2), 16)].join(".");
+      logger.info(`Detected default gateway address: ${ip}`);
+      return ip;
+    }
+  } catch (err) {}
+  return null;
+}
+async function detectBestRedisUrl() {
+  if (cachedRedisUrl) {
+    logger.info(`Using cached Redis connection: ${cachedRedisUrl}`);
+    return cachedRedisUrl;
+  }
+  logger.info("Starting to detect best Redis connection...");
+  const defaultGateway = await getDefaultGateway();
+  const possibleUrls = [
+  // environment variable priority
+  process.env.REDIS_URL,
+  // default gateway
+  defaultGateway ? `redis://${defaultGateway}:6379` : null,
+  // common Docker gateway
+  "redis://192.168.0.1:6379",
+  // Docker special DNS
+  "redis://host.docker.internal:6379",
+  // local Redis (last try)
+  "redis://localhost:6379"].filter(Boolean);
+  for (const url of possibleUrls) {
+    try {
+      logger.debug(`Testing Redis connection: ${url}`);
+      const testClient = (0, _redis.createClient)({
+        url
+      });
+      testClient.on("error", () => {});
+      await Promise.race([testClient.connect(), new Promise((_, reject) => setTimeout(() => reject(new Error("Connection timeout")), 2e3))]);
+      const pingResult = await testClient.ping();
+      await testClient.disconnect();
+      logger.info(`\u2705 Found available Redis connection: ${url}`);
+      cachedRedisUrl = url;
+      return url;
+    } catch (err) {
+      logger.debug(`Redis connection failed ${url}: ${err.message}`);
+    }
+  }
+  logger.warn("\u26A0\uFE0F No available Redis connection found, using default value");
+  return "redis://localhost:6379";
+}
 const cachePool = exports.cachePool = (0, _genericPool.createPool)({
   create: async () => {
     let cache = null;
     try {
+      const redisUrl = await detectBestRedisUrl();
       const redisClient = (0, _redis.createClient)({
+        url: redisUrl,
         socket: {
           // @ts-ignore
           reconnectStrategy: retries => {
@@ -325,9 +383,13 @@ const cachePool = exports.cachePool = (0, _genericPool.createPool)({
           }
         }
       });
+      redisClient.on("error", err => logger.warn("Redis Client Error:", err));
       await redisClient.connect();
+      logger.info(`Successfully connected to Redis: ${redisUrl}`);
       cache = redisClient;
-    } catch (error) {}
+    } catch (error) {
+      logger.warn("Redis connection failed", error);
+    }
     return cache;
   },
   destroy: async client => {

package/lib/es/middlewares/utils.js

@@ -11,11 +11,15 @@ import { parseSitemap } from "sitemap";
 import flattenDeep from "lodash/flattenDeep";
 import uniq from "lodash/uniq";
 import config from "@blocklet/sdk/lib/config";
+import { promisify } from "util";
+import { exec } from "child_process";
 export * from "@blocklet/sdk/lib/config";
 const { logger } = config;
+const execAsync = promisify(exec);
 let browser;
 let checkBrowserTimer;
 const BROWSER_WS_ENDPOINT_KEY = `browserWSEndpoint-${process.env.BLOCKLET_DID || "unknown"}`;
+let cachedRedisUrl = null;
 export const api = axios.create({
   timeout: 1e3 * 10,
   // 10s
@@ -326,12 +330,73 @@ export const getRelativePath = (url) => {
 };
 let ttl = 1e3 * 60 * 60 * 24 * 7;
 const MAX_REDIS_RETRY = 3;
+async function getDefaultGateway() {
+  try {
+    const { stdout } = await execAsync("cat /proc/net/route | grep '^eth0' | head -1 | awk '{print $3}'");
+    if (stdout.trim()) {
+      const hex = stdout.trim();
+      const ip = [
+        parseInt(hex.slice(6, 8), 16),
+        parseInt(hex.slice(4, 6), 16),
+        parseInt(hex.slice(2, 4), 16),
+        parseInt(hex.slice(0, 2), 16)
+      ].join(".");
+      logger.info(`Detected default gateway address: ${ip}`);
+      return ip;
+    }
+  } catch (err) {
+  }
+  return null;
+}
+async function detectBestRedisUrl() {
+  if (cachedRedisUrl) {
+    logger.info(`Using cached Redis connection: ${cachedRedisUrl}`);
+    return cachedRedisUrl;
+  }
+  logger.info("Starting to detect best Redis connection...");
+  const defaultGateway = await getDefaultGateway();
+  const possibleUrls = [
+    // environment variable priority
+    process.env.REDIS_URL,
+    // default gateway
+    defaultGateway ? `redis://${defaultGateway}:6379` : null,
+    // common Docker gateway
+    "redis://192.168.0.1:6379",
+    // Docker special DNS
+    "redis://host.docker.internal:6379",
+    // local Redis (last try)
+    "redis://localhost:6379"
+  ].filter(Boolean);
+  for (const url of possibleUrls) {
+    try {
+      logger.debug(`Testing Redis connection: ${url}`);
+      const testClient = createClient({ url });
+      testClient.on("error", () => {
+      });
+      await Promise.race([
+        testClient.connect(),
+        new Promise((_, reject) => setTimeout(() => reject(new Error("Connection timeout")), 2e3))
+      ]);
+      const pingResult = await testClient.ping();
+      await testClient.disconnect();
+      logger.info(`\u2705 Found available Redis connection: ${url}`);
+      cachedRedisUrl = url;
+      return url;
+    } catch (err) {
+      logger.debug(`Redis connection failed ${url}: ${err.message}`);
+    }
+  }
+  logger.warn("\u26A0\uFE0F No available Redis connection found, using default value");
+  return "redis://localhost:6379";
+}
 export const cachePool = createPool(
   {
     create: async () => {
       let cache = null;
       try {
+        const redisUrl = await detectBestRedisUrl();
         const redisClient = createClient({
+          url: redisUrl,
           socket: {
             // @ts-ignore
             reconnectStrategy: (retries) => {
@@ -342,9 +407,12 @@ export const cachePool = createPool(
             }
           }
         });
+        redisClient.on("error", (err) => logger.warn("Redis Client Error:", err));
         await redisClient.connect();
+        logger.info(`Successfully connected to Redis: ${redisUrl}`);
         cache = redisClient;
       } catch (error) {
+        logger.warn("Redis connection failed", error);
       }
       return cache;
     },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blocklet/crawler",
-  "version": "2.1.229",
+  "version": "2.1.231",
   "description": "blocklet crawler lib",
   "publishConfig": {
     "access": "public"