@blocklet/crawler 2.0.172 → 2.0.175

package/lib/cjs/middlewares/crawler.js

@@ -9,9 +9,9 @@ var _ufo = require("ufo");
 var _utils = require("./utils");
 var _cron = _interopRequireDefault(require("@abtnode/cron"));
 var _debounce = _interopRequireDefault(require("lodash/debounce"));
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
-function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
+function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
 const formatHtml = htmlString => {
   if (typeof htmlString !== "string") {
     return "";
@@ -41,7 +41,6 @@ const getPageContent = async ({
       idleTime: 1 * 1e3
       // 1s
     });
-
     if (formatPageContent) {
       pageContent = await formatPageContent({
         page,
@@ -86,7 +85,7 @@ const crawlUrl = async ({
             await _utils.useCache.set((0, _utils.getRelativePath)(url), {
               content: pageContent,
               lastmod: lastmodMap?.get(url),
-              updatedAt: /* @__PURE__ */new Date().toISOString(),
+              updatedAt: ( /* @__PURE__ */new Date()).toISOString(),
               nextDate: cronCrawlBlockletJob.jobs[CRON_CRAWL_BLOCKLET_KEY].nextDate()
             });
             _utils.logger.info(`Crawler[${index}] ${url} success`);
@@ -178,7 +177,7 @@ const crawlBlocklet = async () => {
           skipBlockletLocTotal++;
           return false;
         }
-        if (nextDate && new Date(nextDate).getTime() >= /* @__PURE__ */new Date().getTime()) {
+        if (nextDate && new Date(nextDate).getTime() >= ( /* @__PURE__ */new Date()).getTime()) {
           skipBlockletLocTotal++;
           return false;
         }

package/lib/cjs/middlewares/utils.js

@@ -62,8 +62,8 @@ Object.keys(_config).forEach(function (key) {
   });
 });
 function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
-function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 const {
   logger
 } = _config.default;
@@ -338,7 +338,6 @@ const cachePool = exports.cachePool = (0, _genericPool.createPool)({
   min: 1
   // evictionRunIntervalMillis: 0,
 });
-
 const withCache = async cb => {
   const client = await cachePool.acquire();
   if (client) {

package/lib/es/middlewares/crawler.js

@@ -165,8 +165,7 @@ export const crawlBlocklet = async () => {
         })
       )).filter(Boolean);
       tempLocList.forEach((loc) => {
-        if (item.lastmod)
-          lastmodMap.set(loc, item.lastmod);
+        if (item.lastmod) lastmodMap.set(loc, item.lastmod);
       });
       canUseBlockletLocList.push(...tempLocList);
     })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blocklet/crawler",
-  "version": "2.0.172",
+  "version": "2.0.175",
   "description": "blocklet crawler lib",
   "publishConfig": {
     "access": "public"
@@ -40,15 +40,6 @@
     "lib",
     "*.d.ts"
   ],
-  "scripts": {
-    "coverage": "yarn test -- --coverage",
-    "test": "vitest tests",
-    "build": "unbuild",
-    "build:watch": "npx nodemon --ext 'ts,tsx,json,js,jsx' --exec 'yarn run build' --ignore 'lib/*' ",
-    "dev": "yarn run build:watch",
-    "prepublish": "yarn run build",
-    "prebuild:dep": "yarn run build"
-  },
   "keywords": [
     "blocklet",
     "crawler"
@@ -59,37 +50,45 @@
     "@abtnode/cron": "1.16.30",
     "@blocklet/puppeteer": "^22.11.3",
     "@blocklet/sdk": "1.16.30",
-    "axios": "^1.5.1",
+    "axios": "^1.7.7",
     "crypto": "^1.0.1",
-    "fs-extra": "^11.1.1",
+    "fs-extra": "^11.2.0",
     "generic-pool": "^3.9.0",
     "lodash": "^4.17.21",
-    "lru-cache": "^10.0.1",
+    "lru-cache": "^10.4.3",
     "p-queue": "6.6.2",
     "p-wait-for": "^5.0.2",
-    "prettier": "^3.3.2",
-    "redis": "^4.6.10",
+    "prettier": "^3.3.3",
+    "redis": "^4.7.0",
     "robots-parser": "^3.0.1",
-    "sitemap": "^7.1.1",
-    "ufo": "^1.3.1",
+    "sitemap": "^7.1.2",
+    "ufo": "^1.5.4",
     "url-join": "^4.0.1"
   },
   "devDependencies": {
     "@arcblock/eslint-config-ts": "^0.3.2",
-    "@types/express": "^4.17.18",
-    "@types/fs-extra": "^11.0.2",
-    "@types/lodash": "^4.14.199",
-    "@types/mime-types": "^2.1.2",
-    "@types/node": "^20.8.3",
+    "@types/express": "^4.17.21",
+    "@types/fs-extra": "^11.0.4",
+    "@types/lodash": "^4.17.7",
+    "@types/mime-types": "^2.1.4",
+    "@types/node": "^20.16.5",
     "@types/puppeteer": "^7.0.4",
-    "@types/react": "^18.2.25",
-    "@types/url-join": "^4.0.1",
+    "@types/react": "^18.3.5",
+    "@types/url-join": "^4.0.3",
     "@vitest/coverage-c8": "^0.33.0",
     "jsdom": "^22.1.0",
-    "typescript": "^5.2.2",
+    "typescript": "^5.6.2",
     "unbuild": "^1.2.1",
-    "vitest": "^1.4.0",
+    "vitest": "^1.6.0",
     "vitest-fetch-mock": "^0.2.2"
   },
-  "gitHead": "749dd785b5ab83dbb7b7575b2e7f20cc21084129"
-}
+  "scripts": {
+    "coverage": "pnpm test -- --coverage",
+    "test": "vitest tests",
+    "build": "unbuild",
+    "build:watch": "npx nodemon --ext 'ts,tsx,json,js,jsx' --exec 'pnpm run build' --ignore 'lib/*' ",
+    "dev": "pnpm run build:watch",
+    "prepublish": "pnpm run build",
+    "prebuild:dep": "pnpm run build"
+  }
+}