@arcblock/crawler-middleware 1.1.1 → 1.1.3

package/README.md

@@ -2,31 +2,43 @@
 
 This express middleware provides pre-rendered HTML generated by SnapKit for Blocklets, enabling them to return complete HTML content to web spider. This is essential for SEO and ensuring that search engines can properly index dynamically generated content.
 
+## How it Works
+
+1.  The middleware intercepts incoming requests.
+2.  It checks if the request is from a web spider.
+3.  Try to read and return HTML from the local cache (Memory LRU Cache + SQLite).
+4.  If the cache is not found, an asynchronous request is made to SnapKit, and the local cache is updated.
+5.  The current request does not return the cached content; the next spider visit will hit step 3 and return the cache directly.
+
+## How to Verify
+
+1. Update your browser's User Agent string to include "spider"
+2. Visit a page that has already been crawled by SnapKit.
+3. First Visit (Cache Miss): On your first visit, the cache should be missed. Check the server logs for a "Cache miss" message. and a request has been sent to SnapKit to cache the page.
+4. Second Visit (Cache Hit): Wait a moment and then revisit the same page. The cache should be hit. The server logs should show a "Cache hit" message, and the returned HTML should include the meta tag: `<meta name="arcblock-crawler" content="true">.`
+
 ## Usage
 
 ```typescript
 import { createSnapshotMiddleware } from '@arcblock/crawler-middleware';
 
 const app = express();
+const snapshotMiddleware = createSnapshotMiddleware({
+  endpoint: process.env.SNAP_KIT_ENDPOINT,
+  accessKey: process.env.SNAP_KIT_ACCESS_KEY,
+  allowCrawler: (req) => {
+    return req.path === '/';
+  },
+});
 
-app.use(
-  createSnapshotMiddleware({
-    endpoint: process.env.SNAP_KIT_ENDPOINT,
-    accessKey: process.env.SNAP_KIT_ACCESS_KEY,
-    allowCrawler: (req) => {
-      return req.path === '/';
-    },
-  }),
-);
-```
-
-## How it Works
+// for all route
+app.use(snapshotMiddleware);
 
-1.  The middleware intercepts incoming requests.
-2.  It checks if the request is from a web crawler.
-3.  Try to read and return HTML from the local cache.
-4.  If the cache is not found, an asynchronous request is made to SnapKit, and the local cache is updated.
-5.  The current request does not return the cached content; the next crawler visit will hit step 3 and return the cache directly.
+// for one route
+app.use('/doc', snapshotMiddleware, (req) => {
+  /* ... */
+});
+```
 
 ## Options
 
@@ -40,11 +52,16 @@ The options for createSnapshotMiddleware:
   accessKey: string;
   /** Max cache size for LRU cache */
   cacheMax?: number;
-  /**
-   * Cache update interval
-   * When cache exceeds this time, it will try to fetch and update cache from SnapKit
-   */
-  cacheUpdateInterval?: number;
+  /** When cache exceeds this time, it will try to fetch and update cache from SnapKit */
+  updateInterval?: number;
+  /** When failed cache exceeds this time, it will try to fetch and update cache from SnapKit */
+  failedUpdateInterval?: number;
+  /** Update queue concurrency */
+  updatedConcurrency?: number;
+  /** Call res.send(html) when cache hit */
+  autoReturnHtml?: boolean;
+  /** Custom function to determine whether to return cached content */
+  allowCrawler?: (req: Request) => boolean;
 };
 ```
 
@@ -52,8 +69,10 @@ The options for createSnapshotMiddleware:
 
 When using this middleware outside of a Blocklet environment, you need to configure the following environment variables:
 
-- `BLOCKLET_APP_DATA_DIR`: (Required) Directory path for storing the sqlite file
+- `BLOCKLET_DATA_DIR`: (Required) Directory path for storing the sqlite file
 - `BLOCKLET_LOG_DIR`: (Required) Directory path for storing @blocklet/logger logs
 - `BLOCKLET_APP_URL`: (Optional) Deployed domain
 
-You can set these variables in your `.env` file.
+## SQLite
+
+When createSnapshotMiddleware is called, it attempts to create an SQLite database at `BLOCKLET_DATA_DIR`. This database is used to cache HTML content retrieved from SnapKit. Please ensure that the deployment environment supports SQLite.

package/lib/cjs/cache.d.ts

@@ -6,16 +6,18 @@ export type CacheManagerOptions = {
     accessKey: string;
     /** Max cache size for LRU cache */
     cacheMax?: number;
-    /**
-     * Cache update interval
-     * When cache exceeds this time, it will try to fetch and update cache from SnapKit
-     */
-    cacheUpdateInterval?: number;
+    /** When cache exceeds this time, it will try to fetch and update cache from SnapKit */
+    updateInterval?: number;
+    /** When failed cache exceeds this time, it will try to fetch and update cache from SnapKit */
+    failedUpdateInterval?: number;
+    /** Update queue concurrency */
+    updatedConcurrency?: number;
 };
 export declare class CacheManager {
     private options;
     private cache;
     private initializedPromise;
+    private updateQueue;
     constructor(options: CacheManagerOptions);
     waitReady(): Promise<void>;
     getSnapshot(url: string): Promise<SnapshotModel | null>;
@@ -23,4 +25,5 @@ export declare class CacheManager {
     fetchSnapKit(url: string): Promise<any>;
     isCacheExpired(url: string): Promise<boolean>;
     updateSnapshot(url: string): Promise<void>;
+    enqueueUpdateSnapshot(url: string): any;
 }

package/lib/cjs/cache.js

@@ -8,17 +8,25 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CacheManager = void 0;
 const crawler_1 = require("@arcblock/crawler");
 const lru_cache_1 = require("lru-cache");
+const queue_1 = __importDefault(require("queue"));
 const ufo_1 = require("ufo");
 const env_1 = require("./env");
 const index_1 = require("./store/index");
 class CacheManager {
     constructor(options) {
-        this.options = Object.assign({ cacheMax: 500, cacheUpdateInterval: 1000 * 60 * 60 * 24 }, options);
+        this.options = Object.assign({ cacheMax: 500, updateInterval: 1000 * 60 * 60 * 24, failedUpdateInterval: 1000 * 60 * 60 * 24, updatedConcurrency: 10 }, options);
         this.cache = new lru_cache_1.LRUCache({ max: this.options.cacheMax || 500 });
+        this.updateQueue = new queue_1.default({
+            autostart: true,
+            concurrency: this.options.updatedConcurrency,
+        });
         this.initializedPromise = Promise.all([(0, index_1.initDatabase)()]);
     }
     waitReady() {
@@ -48,6 +56,7 @@ class CacheManager {
     }
     fetchSnapKit(url) {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a;
             const { endpoint, accessKey } = this.options;
             const api = (0, ufo_1.joinURL)(endpoint, 'api/crawl');
             env_1.logger.debug('Fetching snapshot from SnapKit', { url, api });
@@ -73,7 +82,7 @@ class CacheManager {
                 return snapshotData;
             }
             catch (error) {
-                env_1.logger.error('Failed to fetch content by SnapKit', { url, error });
+                env_1.logger.error('Failed to fetch content by SnapKit', { url, error, data: (_a = error === null || error === void 0 ? void 0 : error.response) === null || _a === void 0 ? void 0 : _a.data });
                 return null;
             }
         });
@@ -84,28 +93,30 @@ class CacheManager {
             if (!snapshot) {
                 return true;
             }
-            return Date.now() - new Date(snapshot.createdAt).getTime() > this.options.cacheUpdateInterval;
+            const interval = snapshot.html ? this.options.updateInterval : this.options.failedUpdateInterval;
+            return Date.now() - new Date(snapshot.updatedAt).getTime() > interval;
         });
     }
     updateSnapshot(url) {
         return __awaiter(this, void 0, void 0, function* () {
             try {
                 const snapshot = yield this.fetchSnapKit(url);
-                if (snapshot) {
-                    // update db
-                    const [updatedSnapshot] = yield index_1.Snapshot.upsert({
-                        url,
-                        html: snapshot.html,
-                        lastModified: snapshot.lastModified,
-                    });
-                    // update cache
-                    this.cache.set(url, updatedSnapshot);
-                }
+                // update db
+                const [updatedSnapshot] = yield index_1.Snapshot.upsert({
+                    url,
+                    html: (snapshot === null || snapshot === void 0 ? void 0 : snapshot.html) || '',
+                    lastModified: snapshot === null || snapshot === void 0 ? void 0 : snapshot.lastModified,
+                });
+                // update cache
+                this.cache.set(url, updatedSnapshot);
             }
             catch (error) {
                 env_1.logger.error('Failed to update snapshot', { url, error });
             }
         });
     }
+    enqueueUpdateSnapshot(url) {
+        return this.updateQueue.push(() => this.updateSnapshot(url));
+    }
 }
 exports.CacheManager = CacheManager;

package/lib/cjs/index.d.ts

@@ -1,16 +1,17 @@
 import { NextFunction, Request, Response } from 'express';
-export declare function createSnapshotMiddleware({ endpoint, accessKey, cacheMax, cacheUpdateInterval, autoReturnHtml, allowCrawler, }: {
+export declare function createSnapshotMiddleware({ endpoint, accessKey, cacheMax, updateInterval, failedUpdateInterval, updatedConcurrency, autoReturnHtml, allowCrawler, }: {
     /** SnapKit endpoint */
     endpoint: string;
     /** SnapKit access key */
     accessKey: string;
     /** Max cache size for LRU cache */
     cacheMax?: number;
-    /**
-     * Cache update interval
-     * When cache exceeds this time, it will try to fetch and update cache from SnapKit
-     */
-    cacheUpdateInterval?: number;
+    /** When cache exceeds this time, it will try to fetch and update cache from SnapKit */
+    updateInterval?: number;
+    /** When failed cache exceeds this time, it will try to fetch and update cache from SnapKit */
+    failedUpdateInterval?: number;
+    /** Update queue concurrency */
+    updatedConcurrency?: number;
     /** Call res.send(html) when cache hit */
     autoReturnHtml?: boolean;
     /** Custom function to determine whether to return cached content */

package/lib/cjs/index.js

@@ -21,7 +21,7 @@ function getFullUrl(req) {
         : req.originalUrl;
     return (0, ufo_1.joinURL)(env_1.env.appUrl || req.get('host'), blockletPathname);
 }
-function createSnapshotMiddleware({ endpoint, accessKey, cacheMax = 500, cacheUpdateInterval = 1000 * 60 * 60 * 24, autoReturnHtml = true, allowCrawler = () => true, }) {
+function createSnapshotMiddleware({ endpoint, accessKey, cacheMax = 500, updateInterval = 1000 * 60 * 60 * 24, failedUpdateInterval = 1000 * 60 * 60 * 24, updatedConcurrency = 10, autoReturnHtml = true, allowCrawler = () => true, }) {
     if (!accessKey || !endpoint) {
         throw new Error('accessKey and endpoint are required');
     }
@@ -29,7 +29,9 @@ function createSnapshotMiddleware({ endpoint, accessKey, cacheMax = 500, cacheUp
         endpoint,
         accessKey,
         cacheMax,
-        cacheUpdateInterval,
+        updateInterval,
+        failedUpdateInterval,
+        updatedConcurrency,
     });
     return (req, res, next) => __awaiter(this, void 0, void 0, function* () {
         yield cacheManager.waitReady();
@@ -37,18 +39,18 @@ function createSnapshotMiddleware({ endpoint, accessKey, cacheMax = 500, cacheUp
             return next();
         }
         const fullUrl = getFullUrl(req);
-        // Always fetch content from SnapKit and cache it, even for non-crawler requests
-        if (yield cacheManager.isCacheExpired(fullUrl)) {
-            env_1.logger.info(`Cache expired for ${fullUrl}, fetching from SnapKit`);
-            // Don't await here, the cache will be effective after the next request
-            cacheManager.updateSnapshot(fullUrl);
-        }
         if (!isSpider(req) || isSelfCrawler(req) || isStaticFile(req)) {
             return next();
         }
+        // fetch content from SnapKit and cache it
+        // Don't await here, the cache will be effective after the next request
+        if (yield cacheManager.isCacheExpired(fullUrl)) {
+            cacheManager.enqueueUpdateSnapshot(fullUrl);
+        }
         // cache hit
         const cachedSnapshot = yield cacheManager.getSnapshot(fullUrl);
-        if (cachedSnapshot) {
+        if (cachedSnapshot === null || cachedSnapshot === void 0 ? void 0 : cachedSnapshot.html) {
+            env_1.logger.info(`Cache hit: ${fullUrl}`);
             // @ts-ignore
             req.cachedHtml = cachedSnapshot.html;
             if (cachedSnapshot.lastModified) {
@@ -66,7 +68,7 @@ function createSnapshotMiddleware({ endpoint, accessKey, cacheMax = 500, cacheUp
             }
             return next();
         }
-        env_1.logger.debug(`Cache not hit: ${fullUrl}`);
+        env_1.logger.info(`Cache miss: ${fullUrl}`);
         return next();
     });
 }

package/lib/cjs/store/index.js

@@ -31,6 +31,7 @@ const model_snapshot_1 = require("./model-snapshot");
 __exportStar(require("./model-snapshot"), exports);
 function initDatabase() {
     return __awaiter(this, void 0, void 0, function* () {
+        env_1.logger.debug(`Init database at ${env_1.env.databasePath}`);
         const sequelize = new core_1.Sequelize({
             dialect: sqlite3_1.SqliteDialect,
             storage: env_1.env.databasePath,

package/lib/esm/cache.d.ts

@@ -6,16 +6,18 @@ export type CacheManagerOptions = {
     accessKey: string;
     /** Max cache size for LRU cache */
     cacheMax?: number;
-    /**
-     * Cache update interval
-     * When cache exceeds this time, it will try to fetch and update cache from SnapKit
-     */
-    cacheUpdateInterval?: number;
+    /** When cache exceeds this time, it will try to fetch and update cache from SnapKit */
+    updateInterval?: number;
+    /** When failed cache exceeds this time, it will try to fetch and update cache from SnapKit */
+    failedUpdateInterval?: number;
+    /** Update queue concurrency */
+    updatedConcurrency?: number;
 };
 export declare class CacheManager {
     private options;
     private cache;
     private initializedPromise;
+    private updateQueue;
     constructor(options: CacheManagerOptions);
     waitReady(): Promise<void>;
     getSnapshot(url: string): Promise<SnapshotModel | null>;
@@ -23,4 +25,5 @@ export declare class CacheManager {
     fetchSnapKit(url: string): Promise<any>;
     isCacheExpired(url: string): Promise<boolean>;
     updateSnapshot(url: string): Promise<void>;
+    enqueueUpdateSnapshot(url: string): any;
 }

package/lib/esm/cache.js

@@ -9,13 +9,18 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 import { utils } from '@arcblock/crawler';
 import { LRUCache } from 'lru-cache';
+import Queue from 'queue';
 import { joinURL } from 'ufo';
 import { logger } from './env';
 import { Snapshot, initDatabase } from './store/index';
 export class CacheManager {
     constructor(options) {
-        this.options = Object.assign({ cacheMax: 500, cacheUpdateInterval: 1000 * 60 * 60 * 24 }, options);
+        this.options = Object.assign({ cacheMax: 500, updateInterval: 1000 * 60 * 60 * 24, failedUpdateInterval: 1000 * 60 * 60 * 24, updatedConcurrency: 10 }, options);
         this.cache = new LRUCache({ max: this.options.cacheMax || 500 });
+        this.updateQueue = new Queue({
+            autostart: true,
+            concurrency: this.options.updatedConcurrency,
+        });
         this.initializedPromise = Promise.all([initDatabase()]);
     }
     waitReady() {
@@ -45,6 +50,7 @@ export class CacheManager {
     }
     fetchSnapKit(url) {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a;
             const { endpoint, accessKey } = this.options;
             const api = joinURL(endpoint, 'api/crawl');
             logger.debug('Fetching snapshot from SnapKit', { url, api });
@@ -70,7 +76,7 @@ export class CacheManager {
                 return snapshotData;
             }
             catch (error) {
-                logger.error('Failed to fetch content by SnapKit', { url, error });
+                logger.error('Failed to fetch content by SnapKit', { url, error, data: (_a = error === null || error === void 0 ? void 0 : error.response) === null || _a === void 0 ? void 0 : _a.data });
                 return null;
             }
         });
@@ -81,27 +87,29 @@ export class CacheManager {
             if (!snapshot) {
                 return true;
             }
-            return Date.now() - new Date(snapshot.createdAt).getTime() > this.options.cacheUpdateInterval;
+            const interval = snapshot.html ? this.options.updateInterval : this.options.failedUpdateInterval;
+            return Date.now() - new Date(snapshot.updatedAt).getTime() > interval;
         });
     }
     updateSnapshot(url) {
         return __awaiter(this, void 0, void 0, function* () {
             try {
                 const snapshot = yield this.fetchSnapKit(url);
-                if (snapshot) {
-                    // update db
-                    const [updatedSnapshot] = yield Snapshot.upsert({
-                        url,
-                        html: snapshot.html,
-                        lastModified: snapshot.lastModified,
-                    });
-                    // update cache
-                    this.cache.set(url, updatedSnapshot);
-                }
+                // update db
+                const [updatedSnapshot] = yield Snapshot.upsert({
+                    url,
+                    html: (snapshot === null || snapshot === void 0 ? void 0 : snapshot.html) || '',
+                    lastModified: snapshot === null || snapshot === void 0 ? void 0 : snapshot.lastModified,
+                });
+                // update cache
+                this.cache.set(url, updatedSnapshot);
             }
             catch (error) {
                 logger.error('Failed to update snapshot', { url, error });
             }
         });
     }
+    enqueueUpdateSnapshot(url) {
+        return this.updateQueue.push(() => this.updateSnapshot(url));
+    }
 }

package/lib/esm/index.d.ts

@@ -1,16 +1,17 @@
 import { NextFunction, Request, Response } from 'express';
-export declare function createSnapshotMiddleware({ endpoint, accessKey, cacheMax, cacheUpdateInterval, autoReturnHtml, allowCrawler, }: {
+export declare function createSnapshotMiddleware({ endpoint, accessKey, cacheMax, updateInterval, failedUpdateInterval, updatedConcurrency, autoReturnHtml, allowCrawler, }: {
     /** SnapKit endpoint */
     endpoint: string;
     /** SnapKit access key */
     accessKey: string;
     /** Max cache size for LRU cache */
     cacheMax?: number;
-    /**
-     * Cache update interval
-     * When cache exceeds this time, it will try to fetch and update cache from SnapKit
-     */
-    cacheUpdateInterval?: number;
+    /** When cache exceeds this time, it will try to fetch and update cache from SnapKit */
+    updateInterval?: number;
+    /** When failed cache exceeds this time, it will try to fetch and update cache from SnapKit */
+    failedUpdateInterval?: number;
+    /** Update queue concurrency */
+    updatedConcurrency?: number;
     /** Call res.send(html) when cache hit */
     autoReturnHtml?: boolean;
     /** Custom function to determine whether to return cached content */

package/lib/esm/index.js

@@ -18,7 +18,7 @@ function getFullUrl(req) {
         : req.originalUrl;
     return joinURL(env.appUrl || req.get('host'), blockletPathname);
 }
-export function createSnapshotMiddleware({ endpoint, accessKey, cacheMax = 500, cacheUpdateInterval = 1000 * 60 * 60 * 24, autoReturnHtml = true, allowCrawler = () => true, }) {
+export function createSnapshotMiddleware({ endpoint, accessKey, cacheMax = 500, updateInterval = 1000 * 60 * 60 * 24, failedUpdateInterval = 1000 * 60 * 60 * 24, updatedConcurrency = 10, autoReturnHtml = true, allowCrawler = () => true, }) {
     if (!accessKey || !endpoint) {
         throw new Error('accessKey and endpoint are required');
     }
@@ -26,7 +26,9 @@ export function createSnapshotMiddleware({ endpoint, accessKey, cacheMax = 500,
         endpoint,
         accessKey,
         cacheMax,
-        cacheUpdateInterval,
+        updateInterval,
+        failedUpdateInterval,
+        updatedConcurrency,
     });
     return (req, res, next) => __awaiter(this, void 0, void 0, function* () {
         yield cacheManager.waitReady();
@@ -34,18 +36,18 @@ export function createSnapshotMiddleware({ endpoint, accessKey, cacheMax = 500,
             return next();
         }
         const fullUrl = getFullUrl(req);
-        // Always fetch content from SnapKit and cache it, even for non-crawler requests
-        if (yield cacheManager.isCacheExpired(fullUrl)) {
-            logger.info(`Cache expired for ${fullUrl}, fetching from SnapKit`);
-            // Don't await here, the cache will be effective after the next request
-            cacheManager.updateSnapshot(fullUrl);
-        }
         if (!isSpider(req) || isSelfCrawler(req) || isStaticFile(req)) {
             return next();
         }
+        // fetch content from SnapKit and cache it
+        // Don't await here, the cache will be effective after the next request
+        if (yield cacheManager.isCacheExpired(fullUrl)) {
+            cacheManager.enqueueUpdateSnapshot(fullUrl);
+        }
         // cache hit
         const cachedSnapshot = yield cacheManager.getSnapshot(fullUrl);
-        if (cachedSnapshot) {
+        if (cachedSnapshot === null || cachedSnapshot === void 0 ? void 0 : cachedSnapshot.html) {
+            logger.info(`Cache hit: ${fullUrl}`);
             // @ts-ignore
             req.cachedHtml = cachedSnapshot.html;
             if (cachedSnapshot.lastModified) {
@@ -63,7 +65,7 @@ export function createSnapshotMiddleware({ endpoint, accessKey, cacheMax = 500,
             }
             return next();
         }
-        logger.debug(`Cache not hit: ${fullUrl}`);
+        logger.info(`Cache miss: ${fullUrl}`);
         return next();
     });
 }

package/lib/esm/store/index.js

@@ -14,6 +14,7 @@ import { Snapshot } from './model-snapshot';
 export * from './model-snapshot';
 export function initDatabase() {
     return __awaiter(this, void 0, void 0, function* () {
+        logger.debug(`Init database at ${env.databasePath}`);
         const sequelize = new Sequelize({
             dialect: SqliteDialect,
             storage: env.databasePath,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arcblock/crawler-middleware",
-  "version": "1.1.1",
+  "version": "1.1.3",
   "main": "lib/cjs/index.js",
   "module": "lib/esm/index.js",
   "types": "lib/cjs/index.d.ts",
@@ -41,38 +41,24 @@
   },
   "dependencies": {
     "@abtnode/cron": "^1.16.43",
-    "@abtnode/models": "^1.16.43",
-    "@abtnode/queue": "^1.16.43",
     "@blocklet/logger": "^1.16.43",
-    "@blocklet/puppeteer": "^22.11.3",
     "@blocklet/sdk": "^1.16.43",
     "@sequelize/core": "7.0.0-alpha.46",
     "@sequelize/sqlite3": "7.0.0-alpha.46",
     "axios": "^1.7.9",
-    "fs-extra": "^11.2.0",
-    "generic-pool": "^3.9.0",
-    "lodash": "^4.17.21",
     "lru-cache": "^10.4.3",
-    "redis": "^4.7.0",
-    "robots-parser": "^3.0.1",
-    "sequelize": "^6.37.7",
-    "sitemap": "^7.1.2",
+    "queue": "^7.0.0",
     "sqlite3": "^5.1.7",
     "ufo": "^1.5.4",
-    "@arcblock/crawler": "1.1.1"
+    "@arcblock/crawler": "1.1.3"
   },
   "devDependencies": {
-    "@blocklet/js-sdk": "^1.16.39",
-    "@types/dotenv-flow": "^3.3.3",
     "@types/express": "^4.17.21",
-    "@types/fs-extra": "^11.0.4",
-    "@types/lodash": "^4.17.16",
     "@types/node": "^20.17.19",
-    "express": "^4.21.2",
     "bumpp": "^9.11.1",
+    "express": "^4.21.2",
     "nodemon": "^3.1.9",
     "npm-run-all": "^4.1.5",
-    "puppeteer": "^24.8.2",
     "tsx": "^4.19.3",
     "zx": "^8.3.2"
   },