@blocklet/crawler 2.1.235 → 2.1.237

package/lib/cjs/middlewares/crawler.d.ts

@@ -1,9 +1,18 @@
+import { getRelativePath } from './utils';
 import { PageOptions } from './types';
+export { getRelativePath };
 export declare const getPageContent: ({ url, formatPageContent }: PageOptions) => Promise<string>;
-export declare const crawlUrl: ({ urls, lastmodMap, formatPageContent, }: {
+export declare const getUrlInfoFromCache: (url: string) => Promise<any>;
+export declare const setUrlInfoToCache: ({ url, content, lastmod, }: {
+    url: string;
+    content: string;
+    lastmod?: string;
+}) => Promise<any>;
+export declare const crawlUrl: ({ urls, lastmodMap, formatPageContent, autoCloseBrowserCount, }: {
     urls: string[] | string;
     lastmodMap?: Map<string, string>;
     formatPageContent?: Function | null | undefined;
+    autoCloseBrowserCount?: number;
 }) => Promise<void>;
 export declare const crawlBlocklet: () => Promise<void>;
 export declare const initCronCrawlBlocklet: ({ time, options, }?: {

package/lib/cjs/middlewares/crawler.js

@@ -3,7 +3,14 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.initCronCrawlBlocklet = exports.getPageContent = exports.crawlUrl = exports.crawlBlocklet = exports.cancelCronCrawlBlocklet = void 0;
+exports.getPageContent = exports.crawlUrl = exports.crawlBlocklet = exports.cancelCronCrawlBlocklet = void 0;
+Object.defineProperty(exports, "getRelativePath", {
+  enumerable: true,
+  get: function () {
+    return _utils.getRelativePath;
+  }
+});
+exports.setUrlInfoToCache = exports.initCronCrawlBlocklet = exports.getUrlInfoFromCache = void 0;
 var PQueue = _interopRequireWildcard(require("p-queue"));
 var _ufo = require("ufo");
 var _utils = require("./utils");
@@ -59,10 +66,31 @@ const getPageContent = async ({
   return formatHtml(pageContent);
 };
 exports.getPageContent = getPageContent;
+const getUrlInfoFromCache = async url => {
+  const cache = await _utils.useCache.get((0, _utils.getRelativePath)(url));
+  return cache;
+};
+exports.getUrlInfoFromCache = getUrlInfoFromCache;
+const setUrlInfoToCache = async ({
+  url,
+  content,
+  lastmod
+}) => {
+  if (!content || !url) {
+    return;
+  }
+  return await _utils.useCache.set((0, _utils.getRelativePath)(url), {
+    content,
+    lastmod: lastmod || ( /* @__PURE__ */new Date()).toISOString(),
+    updatedAt: ( /* @__PURE__ */new Date()).toISOString()
+  });
+};
+exports.setUrlInfoToCache = setUrlInfoToCache;
 const crawlUrl = async ({
   urls,
   lastmodMap,
-  formatPageContent
+  formatPageContent,
+  autoCloseBrowserCount = 30
 }) => {
   if (typeof urls === "string") {
     urls = [urls];
@@ -74,8 +102,10 @@ const crawlUrl = async ({
   }) => {
     return async () => {
       try {
-        if (index % 50 === 0) {
-          await (0, _utils.closeBrowser)();
+        if (index % autoCloseBrowserCount === 0) {
+          await (0, _utils.closeBrowser)({
+            trimCache: false
+          });
         }
         const canCrawl = await (0, _utils.isAcceptCrawler)(url);
         if (canCrawl) {
@@ -84,11 +114,10 @@ const crawlUrl = async ({
             formatPageContent
           });
           if (pageContent) {
-            await _utils.useCache.set((0, _utils.getRelativePath)(url), {
+            await setUrlInfoToCache({
+              url,
               content: pageContent,
-              lastmod: lastmodMap?.get(url),
-              updatedAt: ( /* @__PURE__ */new Date()).toISOString(),
-              nextDate: cronCrawlBlockletJob.jobs[CRON_CRAWL_BLOCKLET_KEY].nextDate()
+              lastmod: lastmodMap?.get(url)
             });
             _utils.logger.info(`Crawler[${index}] ${url} success`);
           } else if (retryCount < 3) {
@@ -239,8 +268,8 @@ exports.crawlBlocklet = crawlBlocklet;
 const CRON_CRAWL_BLOCKLET_KEY = "cron-crawl-blocklet";
 let cronCrawlBlockletJob = null;
 const initCronCrawlBlocklet = ({
-  time = "0 0 */24 * * *",
-  // every 24 hours
+  time = "0 0 */12 * * *",
+  // every 12 hours
   options
 } = {}) => {
   if (!cronCrawlBlockletJob) {

package/lib/cjs/middlewares/utils.d.ts

@@ -3,7 +3,9 @@ export * from '@blocklet/sdk/lib/config';
 export declare const api: import("axios").AxiosInstance;
 export declare const sleep: (ms: number) => Promise<unknown>;
 export declare const clearCheckBrowserTimer: () => void;
-export declare const closeBrowser: () => Promise<void>;
+export declare const closeBrowser: ({ trimCache }?: {
+    trimCache?: boolean;
+}) => Promise<void>;
 export declare const getBrowser: () => Promise<any>;
 export declare const CRAWLER_FLAG = "x-crawler";
 export declare const isSelfCrawler: (req: any) => boolean;

package/lib/cjs/middlewares/utils.js

@@ -95,7 +95,9 @@ const clearCheckBrowserTimer = () => {
   }
 };
 exports.clearCheckBrowserTimer = clearCheckBrowserTimer;
-const closeBrowser = async () => {
+const closeBrowser = async ({
+  trimCache = true
+} = {}) => {
   try {
     if (browser) {
       const pages = await browser.pages().catch(() => []);
@@ -107,7 +109,9 @@ const closeBrowser = async () => {
       await useCache.remove(BROWSER_WS_ENDPOINT_KEY);
     }
     clearCheckBrowserTimer();
-    await _puppeteer.default.trimCache();
+    if (trimCache) {
+      await _puppeteer.default.trimCache();
+    }
     logger.info("Close browser success");
     if (global.gc) {
       global.gc();
@@ -175,7 +179,9 @@ const getBrowser = async () => {
             }
             if (count >= 3) {
               logger.info("Browser inactive for 3 minutes, closing...");
-              await closeBrowser();
+              await closeBrowser({
+                trimCache: true
+              });
             }
           }
         }, 1e3 * 60);

package/lib/es/middlewares/crawler.d.ts

@@ -1,9 +1,18 @@
+import { getRelativePath } from './utils';
 import { PageOptions } from './types';
+export { getRelativePath };
 export declare const getPageContent: ({ url, formatPageContent }: PageOptions) => Promise<string>;
-export declare const crawlUrl: ({ urls, lastmodMap, formatPageContent, }: {
+export declare const getUrlInfoFromCache: (url: string) => Promise<any>;
+export declare const setUrlInfoToCache: ({ url, content, lastmod, }: {
+    url: string;
+    content: string;
+    lastmod?: string;
+}) => Promise<any>;
+export declare const crawlUrl: ({ urls, lastmodMap, formatPageContent, autoCloseBrowserCount, }: {
     urls: string[] | string;
     lastmodMap?: Map<string, string>;
     formatPageContent?: Function | null | undefined;
+    autoCloseBrowserCount?: number;
 }) => Promise<void>;
 export declare const crawlBlocklet: () => Promise<void>;
 export declare const initCronCrawlBlocklet: ({ time, options, }?: {

package/lib/es/middlewares/crawler.js

@@ -16,6 +16,7 @@ import {
 } from "./utils.js";
 import Cron from "@abtnode/cron";
 import debounce from "lodash/debounce";
+export { getRelativePath };
 const formatHtml = (htmlString) => {
   if (typeof htmlString !== "string") {
     return "";
@@ -53,10 +54,29 @@ export const getPageContent = async ({ url, formatPageContent }) => {
   }
   return formatHtml(pageContent);
 };
+export const getUrlInfoFromCache = async (url) => {
+  const cache = await useCache.get(getRelativePath(url));
+  return cache;
+};
+export const setUrlInfoToCache = async ({
+  url,
+  content,
+  lastmod
+}) => {
+  if (!content || !url) {
+    return;
+  }
+  return await useCache.set(getRelativePath(url), {
+    content,
+    lastmod: lastmod || (/* @__PURE__ */ new Date()).toISOString(),
+    updatedAt: (/* @__PURE__ */ new Date()).toISOString()
+  });
+};
 export const crawlUrl = async ({
   urls,
   lastmodMap,
-  formatPageContent
+  formatPageContent,
+  autoCloseBrowserCount = 30
 }) => {
   if (typeof urls === "string") {
     urls = [urls];
@@ -64,8 +84,10 @@ export const crawlUrl = async ({
   const crawlUrlJob = ({ url, retryCount = 0, index }) => {
     return async () => {
       try {
-        if (index % 50 === 0) {
-          await closeBrowser();
+        if (index % autoCloseBrowserCount === 0) {
+          await closeBrowser({
+            trimCache: false
+          });
         }
         const canCrawl = await isAcceptCrawler(url);
         if (canCrawl) {
@@ -74,12 +96,7 @@ export const crawlUrl = async ({
             formatPageContent
           });
           if (pageContent) {
-            await useCache.set(getRelativePath(url), {
-              content: pageContent,
-              lastmod: lastmodMap?.get(url),
-              updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
-              nextDate: cronCrawlBlockletJob.jobs[CRON_CRAWL_BLOCKLET_KEY].nextDate()
-            });
+            await setUrlInfoToCache({ url, content: pageContent, lastmod: lastmodMap?.get(url) });
             logger.info(`Crawler[${index}] ${url} success`);
           } else if (retryCount < 3) {
             retryCount++;
@@ -218,8 +235,8 @@ export const crawlBlocklet = async () => {
 const CRON_CRAWL_BLOCKLET_KEY = "cron-crawl-blocklet";
 let cronCrawlBlockletJob = null;
 export const initCronCrawlBlocklet = ({
-  time = "0 0 */24 * * *",
-  // every 24 hours
+  time = "0 0 */12 * * *",
+  // every 12 hours
   options
 } = {}) => {
   if (!cronCrawlBlockletJob) {

package/lib/es/middlewares/utils.d.ts

@@ -3,7 +3,9 @@ export * from '@blocklet/sdk/lib/config';
 export declare const api: import("axios").AxiosInstance;
 export declare const sleep: (ms: number) => Promise<unknown>;
 export declare const clearCheckBrowserTimer: () => void;
-export declare const closeBrowser: () => Promise<void>;
+export declare const closeBrowser: ({ trimCache }?: {
+    trimCache?: boolean;
+}) => Promise<void>;
 export declare const getBrowser: () => Promise<any>;
 export declare const CRAWLER_FLAG = "x-crawler";
 export declare const isSelfCrawler: (req: any) => boolean;

package/lib/es/middlewares/utils.js

@@ -39,7 +39,7 @@ export const clearCheckBrowserTimer = () => {
     checkBrowserTimer = null;
   }
 };
-export const closeBrowser = async () => {
+export const closeBrowser = async ({ trimCache = true } = {}) => {
   try {
     if (browser) {
       const pages = await browser.pages().catch(() => []);
@@ -52,7 +52,9 @@ export const closeBrowser = async () => {
       await useCache.remove(BROWSER_WS_ENDPOINT_KEY);
     }
     clearCheckBrowserTimer();
-    await puppeteer.trimCache();
+    if (trimCache) {
+      await puppeteer.trimCache();
+    }
     logger.info("Close browser success");
     if (global.gc) {
       global.gc();
@@ -131,7 +133,9 @@ export const getBrowser = async () => {
             }
             if (count >= 3) {
               logger.info("Browser inactive for 3 minutes, closing...");
-              await closeBrowser();
+              await closeBrowser({
+                trimCache: true
+              });
             }
           }
         }, 1e3 * 60);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blocklet/crawler",
-  "version": "2.1.235",
+  "version": "2.1.237",
   "description": "blocklet crawler lib",
   "publishConfig": {
     "access": "public"