@blocklet/crawler 2.1.233 → 2.1.235

package/lib/cjs/middlewares/seo.js

@@ -5,7 +5,6 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.initSEOMiddleware = void 0;
 var _utils = require("./utils");
-var _stream = require("stream");
 const initSEOMiddleware = ({
   autoReturnHtml = true,
   allowCrawler = true
@@ -29,11 +28,7 @@ const initSEOMiddleware = ({
       res.setHeader("Last-Modified", req.cachedLastmod);
     }
     if (autoReturnHtml && req.cachedHtml) {
-      const stream = new _stream.Readable();
-      stream.push(req.cachedHtml);
-      stream.push(null);
-      res.set("Content-Type", "text/html, charset=utf-8");
-      stream.pipe(res);
+      res.send(req.cachedHtml);
       return;
     }
     next();

package/lib/cjs/middlewares/utils.js

@@ -63,6 +63,7 @@ Object.keys(_config).forEach(function (key) {
 });
 var _util = require("util");
 var _child_process = require("child_process");
+var _component = require("@blocklet/sdk/lib/component");
 function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
 function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
 function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
@@ -349,6 +350,19 @@ async function detectBestRedisUrl() {
   const possibleUrls = [
   // environment variable priority
   process.env.REDIS_URL,
+  // by web component endpoint
+  (() => {
+    try {
+      const endpoint = (0, _component.getComponentWebEndpoint)(_config.default.env.componentDid);
+      if (endpoint) {
+        const url = new URL(endpoint);
+        return `redis://${url.hostname}:6379`;
+      }
+    } catch (err) {
+      logger.warn(`Failed to get component endpoint: ${err.message}`);
+    }
+    return null;
+  })(),
   // default gateway
   defaultGateway ? `redis://${defaultGateway}:6379` : null,
   // common Docker gateway
@@ -365,7 +379,7 @@ async function detectBestRedisUrl() {
       });
       testClient.on("error", () => {});
       await Promise.race([testClient.connect(), new Promise((_, reject) => setTimeout(() => reject(new Error("Connection timeout")), 2e3))]);
-      const pingResult = await testClient.ping();
+      await testClient.ping();
       await testClient.disconnect();
       logger.info(`\u2705 Found available Redis connection: ${url}`);
       cachedRedisUrl = url;

package/lib/es/middlewares/seo.js

@@ -1,5 +1,4 @@
 import { isBotUserAgent, getFullUrl, useCache, isAcceptCrawler, isSelfCrawler, getRelativePath } from "./utils.js";
-import { Readable } from "stream";
 export const initSEOMiddleware = ({ autoReturnHtml = true, allowCrawler = true } = {}) => {
   return async (req, res, next) => {
     const isBot = isBotUserAgent(req);
@@ -20,11 +19,7 @@ export const initSEOMiddleware = ({ autoReturnHtml = true, allowCrawler = true }
       res.setHeader("Last-Modified", req.cachedLastmod);
     }
     if (autoReturnHtml && req.cachedHtml) {
-      const stream = new Readable();
-      stream.push(req.cachedHtml);
-      stream.push(null);
-      res.set("Content-Type", "text/html, charset=utf-8");
-      stream.pipe(res);
+      res.send(req.cachedHtml);
       return;
     }
     next();

package/lib/es/middlewares/utils.js

@@ -13,6 +13,7 @@ import uniq from "lodash/uniq";
 import config from "@blocklet/sdk/lib/config";
 import { promisify } from "util";
 import { exec } from "child_process";
+import { getComponentWebEndpoint } from "@blocklet/sdk/lib/component";
 export * from "@blocklet/sdk/lib/config";
 const { logger } = config;
 const execAsync = promisify(exec);
@@ -374,6 +375,19 @@ async function detectBestRedisUrl() {
   const possibleUrls = [
     // environment variable priority
     process.env.REDIS_URL,
+    // by web component endpoint
+    (() => {
+      try {
+        const endpoint = getComponentWebEndpoint(config.env.componentDid);
+        if (endpoint) {
+          const url = new URL(endpoint);
+          return `redis://${url.hostname}:6379`;
+        }
+      } catch (err) {
+        logger.warn(`Failed to get component endpoint: ${err.message}`);
+      }
+      return null;
+    })(),
     // default gateway
     defaultGateway ? `redis://${defaultGateway}:6379` : null,
     // common Docker gateway
@@ -393,7 +407,7 @@ async function detectBestRedisUrl() {
         testClient.connect(),
         new Promise((_, reject) => setTimeout(() => reject(new Error("Connection timeout")), 2e3))
       ]);
-      const pingResult = await testClient.ping();
+      await testClient.ping();
       await testClient.disconnect();
       logger.info(`\u2705 Found available Redis connection: ${url}`);
       cachedRedisUrl = url;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blocklet/crawler",
-  "version": "2.1.233",
+  "version": "2.1.235",
   "description": "blocklet crawler lib",
   "publishConfig": {
     "access": "public"