@arcblock/crawler-middleware 1.3.0 → 1.3.2

package/lib/cjs/cache.d.ts

@@ -19,6 +19,7 @@ export declare class CacheManager {
     private initializedPromise;
     private updateQueue;
     constructor(options: CacheManagerOptions);
+    private initializeQueue;
     waitReady(): Promise<void>;
     getSnapshot(url: string): Promise<SnapshotModel | null>;
     setSnapshot(url: string, snapshot: SnapshotModel): Promise<void>;

package/lib/cjs/cache.js

@@ -8,14 +8,10 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CacheManager = void 0;
-const crawler_1 = require("@arcblock/crawler");
+const utils_1 = require("@arcblock/crawler/utils");
 const lru_cache_1 = require("lru-cache");
-const queue_1 = __importDefault(require("queue"));
 const ufo_1 = require("ufo");
 const env_1 = require("./env");
 const index_1 = require("./store/index");
@@ -26,11 +22,16 @@ class CacheManager {
         if (this.options.cacheMax > 0) {
             this.cache = new lru_cache_1.LRUCache({ max: this.options.cacheMax || 500 });
         }
-        this.updateQueue = new queue_1.default({
-            autostart: true,
-            concurrency: this.options.updatedConcurrency,
+        this.initializedPromise = Promise.all([(0, index_1.initDatabase)(), this.initializeQueue()]);
+    }
+    initializeQueue() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { default: Queue } = yield import('queue');
+            this.updateQueue = new Queue({
+                autostart: true,
+                concurrency: this.options.updatedConcurrency,
+            });
         });
-        this.initializedPromise = Promise.all([(0, index_1.initDatabase)()]);
     }
     waitReady() {
         return __awaiter(this, void 0, void 0, function* () {
@@ -66,7 +67,7 @@ class CacheManager {
             const api = (0, ufo_1.joinURL)(endpoint, 'api/crawl');
             env_1.logger.debug('Fetching snapshot from SnapKit', { url, api });
             try {
-                const { data } = yield crawler_1.utils.axios.get(api, {
+                const { data } = yield utils_1.axios.get(api, {
                     params: {
                         url,
                     },

package/lib/cjs/index.js

@@ -10,11 +10,10 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createSnapshotMiddleware = createSnapshotMiddleware;
-const crawler_1 = require("@arcblock/crawler");
+const utils_1 = require("@arcblock/crawler/utils");
 const ufo_1 = require("ufo");
 const cache_1 = require("./cache");
 const env_1 = require("./env");
-const { isSelfCrawler, isSpider, isStaticFile } = crawler_1.utils;
 function getFullUrl(req) {
     const blockletPathname = req.headers['x-path-prefix']
         ? (0, ufo_1.joinURL)(req.headers['x-path-prefix'], req.originalUrl)
@@ -39,7 +38,7 @@ function createSnapshotMiddleware({ endpoint, accessKey, cacheMax = 0, updateInt
             return next();
         }
         const fullUrl = getFullUrl(req);
-        if (!isSpider(req) || isSelfCrawler(req) || isStaticFile(req)) {
+        if (!(0, utils_1.isSpider)(req) || (0, utils_1.isSelfCrawler)(req) || (0, utils_1.isStaticFile)(req)) {
             return next();
         }
         // fetch content from SnapKit and cache it

package/lib/esm/cache.d.ts

@@ -19,6 +19,7 @@ export declare class CacheManager {
     private initializedPromise;
     private updateQueue;
     constructor(options: CacheManagerOptions);
+    private initializeQueue;
     waitReady(): Promise<void>;
     getSnapshot(url: string): Promise<SnapshotModel | null>;
     setSnapshot(url: string, snapshot: SnapshotModel): Promise<void>;

package/lib/esm/cache.js

@@ -7,9 +7,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-import { utils } from '@arcblock/crawler';
+import { axios } from '@arcblock/crawler/utils';
 import { LRUCache } from 'lru-cache';
-import Queue from 'queue';
 import { joinURL } from 'ufo';
 import { logger } from './env';
 import { Snapshot, initDatabase } from './store/index';
@@ -20,11 +19,16 @@ export class CacheManager {
         if (this.options.cacheMax > 0) {
             this.cache = new LRUCache({ max: this.options.cacheMax || 500 });
         }
-        this.updateQueue = new Queue({
-            autostart: true,
-            concurrency: this.options.updatedConcurrency,
+        this.initializedPromise = Promise.all([initDatabase(), this.initializeQueue()]);
+    }
+    initializeQueue() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { default: Queue } = yield import('queue');
+            this.updateQueue = new Queue({
+                autostart: true,
+                concurrency: this.options.updatedConcurrency,
+            });
         });
-        this.initializedPromise = Promise.all([initDatabase()]);
     }
     waitReady() {
         return __awaiter(this, void 0, void 0, function* () {
@@ -60,7 +64,7 @@ export class CacheManager {
             const api = joinURL(endpoint, 'api/crawl');
             logger.debug('Fetching snapshot from SnapKit', { url, api });
             try {
-                const { data } = yield utils.axios.get(api, {
+                const { data } = yield axios.get(api, {
                     params: {
                         url,
                     },

package/lib/esm/index.js

@@ -7,11 +7,10 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-import { utils } from '@arcblock/crawler';
+import { isSelfCrawler, isSpider, isStaticFile } from '@arcblock/crawler/utils';
 import { joinURL } from 'ufo';
 import { CacheManager } from './cache';
 import { env, logger } from './env';
-const { isSelfCrawler, isSpider, isStaticFile } = utils;
 function getFullUrl(req) {
     const blockletPathname = req.headers['x-path-prefix']
         ? joinURL(req.headers['x-path-prefix'], req.originalUrl)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arcblock/crawler-middleware",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "main": "lib/cjs/index.js",
   "module": "lib/esm/index.js",
   "types": "lib/cjs/index.d.ts",
@@ -40,9 +40,9 @@
     ]
   },
   "dependencies": {
-    "@abtnode/cron": "^1.16.44",
-    "@blocklet/logger": "^1.16.44",
-    "@blocklet/sdk": "^1.16.44",
+    "@abtnode/cron": "^1.16.46",
+    "@blocklet/logger": "^1.16.46",
+    "@blocklet/sdk": "^1.16.46",
     "@sequelize/core": "7.0.0-alpha.46",
     "@sequelize/sqlite3": "7.0.0-alpha.46",
     "axios": "^1.7.9",
@@ -50,7 +50,7 @@
     "queue": "^7.0.0",
     "sqlite3": "^5.1.7",
     "ufo": "^1.5.4",
-    "@arcblock/crawler": "1.3.0"
+    "@arcblock/crawler": "1.3.2"
   },
   "devDependencies": {
     "@types/express": "^4.17.21",