@arcblock/crawler-middleware 1.1.1

package/README.md

@@ -0,0 +1,59 @@
+# @arcblock/crawler-middleware
+
+This express middleware provides pre-rendered HTML generated by SnapKit for Blocklets, enabling them to return complete HTML content to web spider. This is essential for SEO and ensuring that search engines can properly index dynamically generated content.
+
+## Usage
+
+```typescript
+import { createSnapshotMiddleware } from '@arcblock/crawler-middleware';
+
+const app = express();
+
+app.use(
+  createSnapshotMiddleware({
+    endpoint: process.env.SNAP_KIT_ENDPOINT,
+    accessKey: process.env.SNAP_KIT_ACCESS_KEY,
+    allowCrawler: (req) => {
+      return req.path === '/';
+    },
+  }),
+);
+```
+
+## How it Works
+
+1.  The middleware intercepts incoming requests.
+2.  It checks if the request is from a web crawler.
+3.  Try to read and return HTML from the local cache.
+4.  If the cache is not found, an asynchronous request is made to SnapKit, and the local cache is updated.
+5.  The current request does not return the cached content; the next crawler visit will hit step 3 and return the cache directly.
+
+## Options
+
+The options for createSnapshotMiddleware:
+
+```typescript
+{
+  /** SnapKit endpoint */
+  endpoint: string;
+  /** SnapKit access key */
+  accessKey: string;
+  /** Max cache size for LRU cache */
+  cacheMax?: number;
+  /**
+   * Cache update interval
+   * When cache exceeds this time, it will try to fetch and update cache from SnapKit
+   */
+  cacheUpdateInterval?: number;
+};
+```
+
+## Environment Variables
+
+When using this middleware outside of a Blocklet environment, you need to configure the following environment variables:
+
+- `BLOCKLET_APP_DATA_DIR`: (Required) Directory path for storing the sqlite file
+- `BLOCKLET_LOG_DIR`: (Required) Directory path for storing @blocklet/logger logs
+- `BLOCKLET_APP_URL`: (Optional) Deployed domain
+
+You can set these variables in your `.env` file.

package/lib/cjs/cache.d.ts

@@ -0,0 +1,26 @@
+import { SnapshotModel } from './store/index';
+export type CacheManagerOptions = {
+    /** SnapKit endpoint */
+    endpoint: string;
+    /** SnapKit access key */
+    accessKey: string;
+    /** Max cache size for LRU cache */
+    cacheMax?: number;
+    /**
+     * Cache update interval
+     * When cache exceeds this time, it will try to fetch and update cache from SnapKit
+     */
+    cacheUpdateInterval?: number;
+};
+export declare class CacheManager {
+    private options;
+    private cache;
+    private initializedPromise;
+    constructor(options: CacheManagerOptions);
+    waitReady(): Promise<void>;
+    getSnapshot(url: string): Promise<SnapshotModel | null>;
+    setSnapshot(url: string, snapshot: SnapshotModel): Promise<void>;
+    fetchSnapKit(url: string): Promise<any>;
+    isCacheExpired(url: string): Promise<boolean>;
+    updateSnapshot(url: string): Promise<void>;
+}

package/lib/cjs/cache.js

@@ -0,0 +1,111 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CacheManager = void 0;
+const crawler_1 = require("@arcblock/crawler");
+const lru_cache_1 = require("lru-cache");
+const ufo_1 = require("ufo");
+const env_1 = require("./env");
+const index_1 = require("./store/index");
+class CacheManager {
+    constructor(options) {
+        this.options = Object.assign({ cacheMax: 500, cacheUpdateInterval: 1000 * 60 * 60 * 24 }, options);
+        this.cache = new lru_cache_1.LRUCache({ max: this.options.cacheMax || 500 });
+        this.initializedPromise = Promise.all([(0, index_1.initDatabase)()]);
+    }
+    waitReady() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.initializedPromise;
+        });
+    }
+    getSnapshot(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const cachedSnapshot = this.cache.get(url);
+            if (cachedSnapshot) {
+                return cachedSnapshot;
+            }
+            const snapshot = yield index_1.Snapshot.findOne({ where: { url } });
+            if (snapshot) {
+                this.cache.set(url, snapshot);
+                return snapshot;
+            }
+            return null;
+        });
+    }
+    setSnapshot(url, snapshot) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield index_1.Snapshot.create(snapshot);
+            this.cache.set(url, snapshot);
+        });
+    }
+    fetchSnapKit(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { endpoint, accessKey } = this.options;
+            const api = (0, ufo_1.joinURL)(endpoint, 'api/crawl');
+            env_1.logger.debug('Fetching snapshot from SnapKit', { url, api });
+            try {
+                const { data } = yield crawler_1.utils.axios.get(api, {
+                    params: {
+                        url,
+                    },
+                    headers: {
+                        Authorization: `Bearer ${accessKey}`,
+                    },
+                });
+                const { data: snapshotData } = data || {};
+                if ((snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.status) !== 'success') {
+                    env_1.logger.info(`No valid HTML found for ${url} from SnapKit`, { snapshotData, data });
+                    return null;
+                }
+                env_1.logger.info('Success to fetch content by SnapKit and cache it', {
+                    url,
+                    jobId: snapshotData.jobId,
+                    lastModified: snapshotData.lastModified,
+                });
+                return snapshotData;
+            }
+            catch (error) {
+                env_1.logger.error('Failed to fetch content by SnapKit', { url, error });
+                return null;
+            }
+        });
+    }
+    isCacheExpired(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const snapshot = yield this.getSnapshot(url);
+            if (!snapshot) {
+                return true;
+            }
+            return Date.now() - new Date(snapshot.createdAt).getTime() > this.options.cacheUpdateInterval;
+        });
+    }
+    updateSnapshot(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const snapshot = yield this.fetchSnapKit(url);
+                if (snapshot) {
+                    // update db
+                    const [updatedSnapshot] = yield index_1.Snapshot.upsert({
+                        url,
+                        html: snapshot.html,
+                        lastModified: snapshot.lastModified,
+                    });
+                    // update cache
+                    this.cache.set(url, updatedSnapshot);
+                }
+            }
+            catch (error) {
+                env_1.logger.error('Failed to update snapshot', { url, error });
+            }
+        });
+    }
+}
+exports.CacheManager = CacheManager;

package/lib/cjs/env.d.ts

@@ -0,0 +1,5 @@
+export declare const env: {
+    databasePath: string;
+    appUrl: string;
+};
+export declare const logger: any;

package/lib/cjs/env.js

@@ -0,0 +1,14 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.logger = exports.env = void 0;
+const logger_1 = __importDefault(require("@blocklet/logger"));
+const config_1 = __importDefault(require("@blocklet/sdk/lib/config"));
+const node_path_1 = __importDefault(require("node:path"));
+exports.env = {
+    databasePath: node_path_1.default.join(config_1.default.env.dataDir, 'crawler-middleware/snapshot.db'),
+    appUrl: config_1.default.env.appUrl,
+};
+exports.logger = (0, logger_1.default)('@arcblock/crawler-middleware', { level: process.env.LOG_LEVEL || 'info' });

package/lib/cjs/index.d.ts

@@ -0,0 +1,18 @@
+import { NextFunction, Request, Response } from 'express';
+export declare function createSnapshotMiddleware({ endpoint, accessKey, cacheMax, cacheUpdateInterval, autoReturnHtml, allowCrawler, }: {
+    /** SnapKit endpoint */
+    endpoint: string;
+    /** SnapKit access key */
+    accessKey: string;
+    /** Max cache size for LRU cache */
+    cacheMax?: number;
+    /**
+     * Cache update interval
+     * When cache exceeds this time, it will try to fetch and update cache from SnapKit
+     */
+    cacheUpdateInterval?: number;
+    /** Call res.send(html) when cache hit */
+    autoReturnHtml?: boolean;
+    /** Custom function to determine whether to return cached content */
+    allowCrawler?: (req: Request) => boolean;
+}): (req: Request, res: Response, next: NextFunction) => Promise<void>;

package/lib/cjs/index.js

@@ -0,0 +1,72 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createSnapshotMiddleware = createSnapshotMiddleware;
+const crawler_1 = require("@arcblock/crawler");
+const ufo_1 = require("ufo");
+const cache_1 = require("./cache");
+const env_1 = require("./env");
+const { isSelfCrawler, isSpider, isStaticFile } = crawler_1.utils;
+function getFullUrl(req) {
+    const blockletPathname = req.headers['x-path-prefix']
+        ? (0, ufo_1.joinURL)(req.headers['x-path-prefix'], req.originalUrl)
+        : req.originalUrl;
+    return (0, ufo_1.joinURL)(env_1.env.appUrl || req.get('host'), blockletPathname);
+}
+function createSnapshotMiddleware({ endpoint, accessKey, cacheMax = 500, cacheUpdateInterval = 1000 * 60 * 60 * 24, autoReturnHtml = true, allowCrawler = () => true, }) {
+    if (!accessKey || !endpoint) {
+        throw new Error('accessKey and endpoint are required');
+    }
+    const cacheManager = new cache_1.CacheManager({
+        endpoint,
+        accessKey,
+        cacheMax,
+        cacheUpdateInterval,
+    });
+    return (req, res, next) => __awaiter(this, void 0, void 0, function* () {
+        yield cacheManager.waitReady();
+        if (!allowCrawler(req)) {
+            return next();
+        }
+        const fullUrl = getFullUrl(req);
+        // Always fetch content from SnapKit and cache it, even for non-crawler requests
+        if (yield cacheManager.isCacheExpired(fullUrl)) {
+            env_1.logger.info(`Cache expired for ${fullUrl}, fetching from SnapKit`);
+            // Don't await here, the cache will be effective after the next request
+            cacheManager.updateSnapshot(fullUrl);
+        }
+        if (!isSpider(req) || isSelfCrawler(req) || isStaticFile(req)) {
+            return next();
+        }
+        // cache hit
+        const cachedSnapshot = yield cacheManager.getSnapshot(fullUrl);
+        if (cachedSnapshot) {
+            // @ts-ignore
+            req.cachedHtml = cachedSnapshot.html;
+            if (cachedSnapshot.lastModified) {
+                // @ts-ignore
+                req.cachedLastmod = new Date(cachedSnapshot.lastModified).toUTCString();
+                res.setHeader('Last-Modified', cachedSnapshot.lastModified);
+            }
+            if (autoReturnHtml) {
+                env_1.logger.debug(`Cache hit: ${fullUrl} `, {
+                    lastModified: cachedSnapshot.lastModified,
+                    createdAt: cachedSnapshot.createdAt,
+                });
+                res.send(cachedSnapshot.html);
+                return;
+            }
+            return next();
+        }
+        env_1.logger.debug(`Cache not hit: ${fullUrl}`);
+        return next();
+    });
+}

package/lib/cjs/store/index.d.ts

@@ -0,0 +1,4 @@
+import { Sequelize } from '@sequelize/core';
+import { SqliteDialect } from '@sequelize/sqlite3';
+export * from './model-snapshot';
+export declare function initDatabase(): Promise<Sequelize<SqliteDialect>>;

package/lib/cjs/store/index.js

@@ -0,0 +1,66 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initDatabase = initDatabase;
+const core_1 = require("@sequelize/core");
+const sqlite3_1 = require("@sequelize/sqlite3");
+const env_1 = require("../env");
+const model_snapshot_1 = require("./model-snapshot");
+__exportStar(require("./model-snapshot"), exports);
+function initDatabase() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const sequelize = new core_1.Sequelize({
+            dialect: sqlite3_1.SqliteDialect,
+            storage: env_1.env.databasePath,
+            logging: (msg) => process.env.SQLITE_LOG && env_1.logger.debug(msg),
+            pool: {
+                min: 0,
+                max: 10,
+                idle: 10000,
+            },
+            retry: {
+                match: [/SQLITE_BUSY/],
+                name: 'query',
+                max: 10,
+            },
+        });
+        model_snapshot_1.Snapshot.initModel(sequelize);
+        try {
+            yield Promise.all([
+                sequelize.query('pragma journal_mode = WAL;'),
+                sequelize.query('pragma synchronous = normal;'),
+                sequelize.query('pragma journal_size_limit = 67108864;'),
+            ]);
+            yield sequelize.authenticate();
+            yield sequelize.sync();
+            env_1.logger.info('Successfully connected to database');
+        }
+        catch (error) {
+            env_1.logger.error('Failed to connect to database:', error);
+            throw error;
+        }
+        return sequelize;
+    });
+}

package/lib/cjs/store/model-snapshot.d.ts

@@ -0,0 +1,16 @@
+import { Model, Sequelize } from '@sequelize/core';
+export interface SnapshotModel {
+    url: string;
+    html: string;
+    lastModified?: string;
+    createdAt?: string;
+    updatedAt?: string;
+}
+export declare class Snapshot extends Model<SnapshotModel> implements SnapshotModel {
+    url: SnapshotModel['url'];
+    html: SnapshotModel['html'];
+    lastModified?: SnapshotModel['lastModified'];
+    createdAt: SnapshotModel['createdAt'];
+    updatedAt: SnapshotModel['updatedAt'];
+    static initModel(sequelize: Sequelize): void;
+}

package/lib/cjs/store/model-snapshot.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Snapshot = void 0;
+const core_1 = require("@sequelize/core");
+class Snapshot extends core_1.Model {
+    static initModel(sequelize) {
+        Snapshot.init({
+            url: {
+                type: core_1.DataTypes.STRING,
+                allowNull: false,
+                primaryKey: true,
+            },
+            html: {
+                type: core_1.DataTypes.TEXT,
+                allowNull: false,
+            },
+            lastModified: {
+                type: core_1.DataTypes.STRING,
+                allowNull: true,
+            },
+        }, {
+            sequelize,
+            modelName: 'snapshot',
+            tableName: 'snap',
+            timestamps: true,
+        });
+    }
+}
+exports.Snapshot = Snapshot;

package/lib/esm/cache.d.ts

@@ -0,0 +1,26 @@
+import { SnapshotModel } from './store/index';
+export type CacheManagerOptions = {
+    /** SnapKit endpoint */
+    endpoint: string;
+    /** SnapKit access key */
+    accessKey: string;
+    /** Max cache size for LRU cache */
+    cacheMax?: number;
+    /**
+     * Cache update interval
+     * When cache exceeds this time, it will try to fetch and update cache from SnapKit
+     */
+    cacheUpdateInterval?: number;
+};
+export declare class CacheManager {
+    private options;
+    private cache;
+    private initializedPromise;
+    constructor(options: CacheManagerOptions);
+    waitReady(): Promise<void>;
+    getSnapshot(url: string): Promise<SnapshotModel | null>;
+    setSnapshot(url: string, snapshot: SnapshotModel): Promise<void>;
+    fetchSnapKit(url: string): Promise<any>;
+    isCacheExpired(url: string): Promise<boolean>;
+    updateSnapshot(url: string): Promise<void>;
+}

package/lib/esm/cache.js

@@ -0,0 +1,107 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { utils } from '@arcblock/crawler';
+import { LRUCache } from 'lru-cache';
+import { joinURL } from 'ufo';
+import { logger } from './env';
+import { Snapshot, initDatabase } from './store/index';
+export class CacheManager {
+    constructor(options) {
+        this.options = Object.assign({ cacheMax: 500, cacheUpdateInterval: 1000 * 60 * 60 * 24 }, options);
+        this.cache = new LRUCache({ max: this.options.cacheMax || 500 });
+        this.initializedPromise = Promise.all([initDatabase()]);
+    }
+    waitReady() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.initializedPromise;
+        });
+    }
+    getSnapshot(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const cachedSnapshot = this.cache.get(url);
+            if (cachedSnapshot) {
+                return cachedSnapshot;
+            }
+            const snapshot = yield Snapshot.findOne({ where: { url } });
+            if (snapshot) {
+                this.cache.set(url, snapshot);
+                return snapshot;
+            }
+            return null;
+        });
+    }
+    setSnapshot(url, snapshot) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield Snapshot.create(snapshot);
+            this.cache.set(url, snapshot);
+        });
+    }
+    fetchSnapKit(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { endpoint, accessKey } = this.options;
+            const api = joinURL(endpoint, 'api/crawl');
+            logger.debug('Fetching snapshot from SnapKit', { url, api });
+            try {
+                const { data } = yield utils.axios.get(api, {
+                    params: {
+                        url,
+                    },
+                    headers: {
+                        Authorization: `Bearer ${accessKey}`,
+                    },
+                });
+                const { data: snapshotData } = data || {};
+                if ((snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.status) !== 'success') {
+                    logger.info(`No valid HTML found for ${url} from SnapKit`, { snapshotData, data });
+                    return null;
+                }
+                logger.info('Success to fetch content by SnapKit and cache it', {
+                    url,
+                    jobId: snapshotData.jobId,
+                    lastModified: snapshotData.lastModified,
+                });
+                return snapshotData;
+            }
+            catch (error) {
+                logger.error('Failed to fetch content by SnapKit', { url, error });
+                return null;
+            }
+        });
+    }
+    isCacheExpired(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const snapshot = yield this.getSnapshot(url);
+            if (!snapshot) {
+                return true;
+            }
+            return Date.now() - new Date(snapshot.createdAt).getTime() > this.options.cacheUpdateInterval;
+        });
+    }
+    updateSnapshot(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const snapshot = yield this.fetchSnapKit(url);
+                if (snapshot) {
+                    // update db
+                    const [updatedSnapshot] = yield Snapshot.upsert({
+                        url,
+                        html: snapshot.html,
+                        lastModified: snapshot.lastModified,
+                    });
+                    // update cache
+                    this.cache.set(url, updatedSnapshot);
+                }
+            }
+            catch (error) {
+                logger.error('Failed to update snapshot', { url, error });
+            }
+        });
+    }
+}

package/lib/esm/env.d.ts

@@ -0,0 +1,5 @@
+export declare const env: {
+    databasePath: string;
+    appUrl: string;
+};
+export declare const logger: any;

package/lib/esm/env.js

@@ -0,0 +1,8 @@
+import createLogger from '@blocklet/logger';
+import config from '@blocklet/sdk/lib/config';
+import path from 'node:path';
+export const env = {
+    databasePath: path.join(config.env.dataDir, 'crawler-middleware/snapshot.db'),
+    appUrl: config.env.appUrl,
+};
+export const logger = createLogger('@arcblock/crawler-middleware', { level: process.env.LOG_LEVEL || 'info' });

package/lib/esm/index.d.ts

@@ -0,0 +1,18 @@
+import { NextFunction, Request, Response } from 'express';
+export declare function createSnapshotMiddleware({ endpoint, accessKey, cacheMax, cacheUpdateInterval, autoReturnHtml, allowCrawler, }: {
+    /** SnapKit endpoint */
+    endpoint: string;
+    /** SnapKit access key */
+    accessKey: string;
+    /** Max cache size for LRU cache */
+    cacheMax?: number;
+    /**
+     * Cache update interval
+     * When cache exceeds this time, it will try to fetch and update cache from SnapKit
+     */
+    cacheUpdateInterval?: number;
+    /** Call res.send(html) when cache hit */
+    autoReturnHtml?: boolean;
+    /** Custom function to determine whether to return cached content */
+    allowCrawler?: (req: Request) => boolean;
+}): (req: Request, res: Response, next: NextFunction) => Promise<void>;

package/lib/esm/index.js

@@ -0,0 +1,69 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { utils } from '@arcblock/crawler';
+import { joinURL } from 'ufo';
+import { CacheManager } from './cache';
+import { env, logger } from './env';
+const { isSelfCrawler, isSpider, isStaticFile } = utils;
+function getFullUrl(req) {
+    const blockletPathname = req.headers['x-path-prefix']
+        ? joinURL(req.headers['x-path-prefix'], req.originalUrl)
+        : req.originalUrl;
+    return joinURL(env.appUrl || req.get('host'), blockletPathname);
+}
+export function createSnapshotMiddleware({ endpoint, accessKey, cacheMax = 500, cacheUpdateInterval = 1000 * 60 * 60 * 24, autoReturnHtml = true, allowCrawler = () => true, }) {
+    if (!accessKey || !endpoint) {
+        throw new Error('accessKey and endpoint are required');
+    }
+    const cacheManager = new CacheManager({
+        endpoint,
+        accessKey,
+        cacheMax,
+        cacheUpdateInterval,
+    });
+    return (req, res, next) => __awaiter(this, void 0, void 0, function* () {
+        yield cacheManager.waitReady();
+        if (!allowCrawler(req)) {
+            return next();
+        }
+        const fullUrl = getFullUrl(req);
+        // Always fetch content from SnapKit and cache it, even for non-crawler requests
+        if (yield cacheManager.isCacheExpired(fullUrl)) {
+            logger.info(`Cache expired for ${fullUrl}, fetching from SnapKit`);
+            // Don't await here, the cache will be effective after the next request
+            cacheManager.updateSnapshot(fullUrl);
+        }
+        if (!isSpider(req) || isSelfCrawler(req) || isStaticFile(req)) {
+            return next();
+        }
+        // cache hit
+        const cachedSnapshot = yield cacheManager.getSnapshot(fullUrl);
+        if (cachedSnapshot) {
+            // @ts-ignore
+            req.cachedHtml = cachedSnapshot.html;
+            if (cachedSnapshot.lastModified) {
+                // @ts-ignore
+                req.cachedLastmod = new Date(cachedSnapshot.lastModified).toUTCString();
+                res.setHeader('Last-Modified', cachedSnapshot.lastModified);
+            }
+            if (autoReturnHtml) {
+                logger.debug(`Cache hit: ${fullUrl} `, {
+                    lastModified: cachedSnapshot.lastModified,
+                    createdAt: cachedSnapshot.createdAt,
+                });
+                res.send(cachedSnapshot.html);
+                return;
+            }
+            return next();
+        }
+        logger.debug(`Cache not hit: ${fullUrl}`);
+        return next();
+    });
+}

package/lib/esm/store/index.d.ts

@@ -0,0 +1,4 @@
+import { Sequelize } from '@sequelize/core';
+import { SqliteDialect } from '@sequelize/sqlite3';
+export * from './model-snapshot';
+export declare function initDatabase(): Promise<Sequelize<SqliteDialect>>;

package/lib/esm/store/index.js

@@ -0,0 +1,49 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { Sequelize } from '@sequelize/core';
+import { SqliteDialect } from '@sequelize/sqlite3';
+import { env, logger } from '../env';
+import { Snapshot } from './model-snapshot';
+export * from './model-snapshot';
+export function initDatabase() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const sequelize = new Sequelize({
+            dialect: SqliteDialect,
+            storage: env.databasePath,
+            logging: (msg) => process.env.SQLITE_LOG && logger.debug(msg),
+            pool: {
+                min: 0,
+                max: 10,
+                idle: 10000,
+            },
+            retry: {
+                match: [/SQLITE_BUSY/],
+                name: 'query',
+                max: 10,
+            },
+        });
+        Snapshot.initModel(sequelize);
+        try {
+            yield Promise.all([
+                sequelize.query('pragma journal_mode = WAL;'),
+                sequelize.query('pragma synchronous = normal;'),
+                sequelize.query('pragma journal_size_limit = 67108864;'),
+            ]);
+            yield sequelize.authenticate();
+            yield sequelize.sync();
+            logger.info('Successfully connected to database');
+        }
+        catch (error) {
+            logger.error('Failed to connect to database:', error);
+            throw error;
+        }
+        return sequelize;
+    });
+}

package/lib/esm/store/model-snapshot.d.ts

@@ -0,0 +1,16 @@
+import { Model, Sequelize } from '@sequelize/core';
+export interface SnapshotModel {
+    url: string;
+    html: string;
+    lastModified?: string;
+    createdAt?: string;
+    updatedAt?: string;
+}
+export declare class Snapshot extends Model<SnapshotModel> implements SnapshotModel {
+    url: SnapshotModel['url'];
+    html: SnapshotModel['html'];
+    lastModified?: SnapshotModel['lastModified'];
+    createdAt: SnapshotModel['createdAt'];
+    updatedAt: SnapshotModel['updatedAt'];
+    static initModel(sequelize: Sequelize): void;
+}

package/lib/esm/store/model-snapshot.js

@@ -0,0 +1,25 @@
+import { DataTypes, Model } from '@sequelize/core';
+export class Snapshot extends Model {
+    static initModel(sequelize) {
+        Snapshot.init({
+            url: {
+                type: DataTypes.STRING,
+                allowNull: false,
+                primaryKey: true,
+            },
+            html: {
+                type: DataTypes.TEXT,
+                allowNull: false,
+            },
+            lastModified: {
+                type: DataTypes.STRING,
+                allowNull: true,
+            },
+        }, {
+            sequelize,
+            modelName: 'snapshot',
+            tableName: 'snap',
+            timestamps: true,
+        });
+    }
+}

package/package.json

@@ -0,0 +1,101 @@
+{
+  "name": "@arcblock/crawler-middleware",
+  "version": "1.1.1",
+  "main": "lib/cjs/index.js",
+  "module": "lib/esm/index.js",
+  "types": "lib/cjs/index.d.ts",
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "lib",
+    "*.d.ts"
+  ],
+  "exports": {
+    ".": {
+      "require": "./lib/cjs/index.js",
+      "import": "./lib/esm/index.js",
+      "types": "./lib/cjs/index.d.ts"
+    }
+  },
+  "lint-staged": {
+    "*.{mjs,js,jsx,ts,tsx}": [
+      "prettier --write",
+      "eslint"
+    ],
+    "*.{css,less,scss,json,graphql}": [
+      "prettier --write"
+    ]
+  },
+  "browserslist": {
+    "production": [
+      ">0.2%",
+      "not dead",
+      "not op_mini all"
+    ],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  },
+  "dependencies": {
+    "@abtnode/cron": "^1.16.43",
+    "@abtnode/models": "^1.16.43",
+    "@abtnode/queue": "^1.16.43",
+    "@blocklet/logger": "^1.16.43",
+    "@blocklet/puppeteer": "^22.11.3",
+    "@blocklet/sdk": "^1.16.43",
+    "@sequelize/core": "7.0.0-alpha.46",
+    "@sequelize/sqlite3": "7.0.0-alpha.46",
+    "axios": "^1.7.9",
+    "fs-extra": "^11.2.0",
+    "generic-pool": "^3.9.0",
+    "lodash": "^4.17.21",
+    "lru-cache": "^10.4.3",
+    "redis": "^4.7.0",
+    "robots-parser": "^3.0.1",
+    "sequelize": "^6.37.7",
+    "sitemap": "^7.1.2",
+    "sqlite3": "^5.1.7",
+    "ufo": "^1.5.4",
+    "@arcblock/crawler": "1.1.1"
+  },
+  "devDependencies": {
+    "@blocklet/js-sdk": "^1.16.39",
+    "@types/dotenv-flow": "^3.3.3",
+    "@types/express": "^4.17.21",
+    "@types/fs-extra": "^11.0.4",
+    "@types/lodash": "^4.17.16",
+    "@types/node": "^20.17.19",
+    "express": "^4.21.2",
+    "bumpp": "^9.11.1",
+    "nodemon": "^3.1.9",
+    "npm-run-all": "^4.1.5",
+    "puppeteer": "^24.8.2",
+    "tsx": "^4.19.3",
+    "zx": "^8.3.2"
+  },
+  "importSort": {
+    ".js, .jsx, .mjs": {
+      "parser": "babylon",
+      "style": "module"
+    },
+    ".ts, .tsx": {
+      "style": "module",
+      "parser": "typescript"
+    }
+  },
+  "simple-git-hooks": {
+    "pre-commit": "npx lint-staged"
+  },
+  "scripts": {
+    "dev": "tsc -p tsconfig.cjs.json --watch",
+    "lint": "tsc --noEmit && eslint src --ext .mjs,.js,.jsx,.ts,.tsx",
+    "lint:fix": "npm run lint -- --fix",
+    "bundle": "npm run build",
+    "build:cjs": "tsc -p tsconfig.cjs.json",
+    "build:esm": "tsc -p tsconfig.esm.json",
+    "build": "npm run build:cjs && npm run build:esm"
+  }
+}