@arcblock/crawler 1.0.0 → 1.0.2

package/dist/crawler.js

@@ -0,0 +1,314 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getPageContent = void 0;
+exports.createCrawlQueue = createCrawlQueue;
+exports.getDataDir = getDataDir;
+exports.createCrawlJob = createCrawlJob;
+exports.getJob = getJob;
+exports.formatSnapshot = formatSnapshot;
+exports.getSnapshot = getSnapshot;
+const queue_1 = __importDefault(require("@abtnode/queue"));
+const sequelize_1 = __importDefault(require("@abtnode/queue/lib/store/sequelize"));
+const core_1 = __importDefault(require("@sequelize/core"));
+const crypto_1 = require("crypto");
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const pick_1 = __importDefault(require("lodash/pick"));
+const path_1 = __importDefault(require("path"));
+const ufo_1 = require("ufo");
+const config_1 = require("./config");
+const job_1 = require("./db/job");
+const snapshot_1 = require("./db/snapshot");
+const puppeteer_1 = require("./puppeteer");
+const utils_1 = require("./utils");
+const { BaseState } = require('@abtnode/models');
+let crawlQueue;
+function createCrawlQueue() {
+    const db = new BaseState(job_1.Job);
+    crawlQueue = (0, queue_1.default)({
+        store: new sequelize_1.default(db, 'crawler'),
+        concurrency: 1,
+        onJob: (job) => __awaiter(this, void 0, void 0, function* () {
+            config_1.logger.debug('job start:', job);
+            const canCrawl = yield (0, utils_1.isAcceptCrawler)(job.url);
+            if (!canCrawl) {
+                config_1.logger.error(`failed to crawl ${job.url}, denied by robots.txt`, job);
+                const snapshot = convertJobToSnapshot({
+                    job,
+                    snapshot: {
+                        status: 'failed',
+                        error: 'Denied by robots.txt',
+                    },
+                });
+                yield snapshot_1.Snapshot.upsert(snapshot);
+                return snapshot;
+            }
+            // if index reach autoCloseBrowserCount, close browser
+            // try {
+            //   if (index >= autoCloseBrowserCount) {
+            //     await closeBrowser({ trimCache: false });
+            //   }
+            // } catch (error) {
+            //   logger.error('failed to close browser when queue index reached autoCloseBrowserCount:', error);
+            // }
+            try {
+                // get page content later
+                const result = yield (0, exports.getPageContent)(job);
+                if (!result || (!result.html && !result.screenshot)) {
+                    config_1.logger.error(`failed to crawl ${job.url}, empty content`, job);
+                    const snapshot = convertJobToSnapshot({
+                        job,
+                        snapshot: {
+                            status: 'failed',
+                            error: 'Failed to crawl content',
+                        },
+                    });
+                    yield snapshot_1.Snapshot.upsert(snapshot);
+                    return snapshot;
+                }
+                // save html and screenshot to data dir
+                const { screenshotPath, htmlPath } = yield saveSnapshotToLocal({
+                    screenshot: result.screenshot,
+                    html: result.html,
+                });
+                // const lastModified = job.lastmodMap?.get(url) || new Date().toISOString();
+                const snapshot = convertJobToSnapshot({
+                    job,
+                    snapshot: {
+                        status: 'success',
+                        screenshot: screenshotPath === null || screenshotPath === void 0 ? void 0 : screenshotPath.replace(config_1.config.dataDir, ''),
+                        html: htmlPath === null || htmlPath === void 0 ? void 0 : htmlPath.replace(config_1.config.dataDir, ''),
+                    },
+                });
+                yield snapshot_1.Snapshot.upsert(snapshot);
+                return snapshot;
+                // save to redis
+                // if (saveToRedis) {
+                //   useCache.set(url, {
+                //     html: result.html || '',
+                //     lastModified,
+                //   });
+                //   logger.info(`success to crawl ${url}`, job);
+                //   return result;
+                // }
+            }
+            catch (error) {
+                config_1.logger.error(`Failed to crawl ${job.url}`, { error, job });
+                console.error(error.stack);
+                const snapshot = convertJobToSnapshot({
+                    job,
+                    snapshot: {
+                        status: 'failed',
+                        error: 'Internal error',
+                    },
+                });
+                yield snapshot_1.Snapshot.upsert(snapshot);
+                return snapshot;
+            }
+        }),
+    });
+}
+function getDataDir() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const htmlDir = path_1.default.join(config_1.config.dataDir, 'data', 'html');
+        const screenshotDir = path_1.default.join(config_1.config.dataDir, 'data', 'screenshot');
+        yield fs_extra_1.default.ensureDir(htmlDir);
+        yield fs_extra_1.default.ensureDir(screenshotDir);
+        return { htmlDir, screenshotDir };
+    });
+}
+function saveSnapshotToLocal(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ screenshot, html }) {
+        const { htmlDir, screenshotDir } = yield getDataDir();
+        let screenshotPath = null;
+        let htmlPath = null;
+        if (screenshot) {
+            const hash = (0, utils_1.md5)(screenshot);
+            screenshotPath = path_1.default.join(screenshotDir, `${hash}.webp`);
+            config_1.logger.debug('saveSnapshotToLocal.screenshot', { screenshotPath });
+            yield fs_extra_1.default.writeFile(screenshotPath, screenshot);
+        }
+        if (html) {
+            const hash = (0, utils_1.md5)(html);
+            htmlPath = path_1.default.join(htmlDir, `${hash}.html`);
+            config_1.logger.debug('saveSnapshotToLocal.html', { htmlPath });
+            yield fs_extra_1.default.writeFile(htmlPath, html);
+        }
+        return {
+            screenshotPath,
+            htmlPath,
+        };
+    });
+}
+function formatHtml(htmlString) {
+    if (htmlString.includes('<h2>Unexpected Application Error!</h2>')) {
+        return '';
+    }
+    return htmlString;
+}
+const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, formatPageContent, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 60 * 1000, fullPage = false, }) {
+    config_1.logger.debug('getPageContent', { url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage });
+    const page = yield (0, puppeteer_1.initPage)();
+    if (width && height) {
+        yield page.setViewport({ width, height });
+    }
+    let html = null;
+    let screenshot = null;
+    try {
+        const response = yield page.goto(url, { timeout });
+        if (!response) {
+            throw new Error(`Failed to load page: response is null for ${url}`);
+        }
+        const statusCode = response.status();
+        config_1.logger.debug('getPageContent.response', { response, statusCode });
+        if (![200, 304].includes(statusCode)) {
+            throw new Error(`Request failed with status ${statusCode}, in ${url}`);
+        }
+        // await for networkidle0
+        // https://pptr.dev/api/puppeteer.page.goforward/#remarks
+        yield page.waitForNetworkIdle({
+            idleTime: 2 * 1000,
+        });
+        // get screenshot
+        if (includeScreenshot) {
+            try {
+                screenshot = yield page.screenshot({ fullPage, quality, type: 'webp' });
+            }
+            catch (err) {
+                config_1.logger.error('Failed to get screenshot:', err);
+            }
+        }
+        // get html
+        if (includeHtml) {
+            if (formatPageContent) {
+                html = yield formatPageContent({ page, url });
+            }
+            else {
+                html = yield page.content();
+            }
+        }
+    }
+    catch (error) {
+        config_1.logger.error('Failed to get page content:', error);
+        throw error;
+    }
+    finally {
+        yield page.close();
+    }
+    html = formatHtml(html || '');
+    return {
+        html,
+        screenshot,
+    };
+});
+exports.getPageContent = getPageContent;
+function createCrawlJob(params, callback) {
+    return __awaiter(this, void 0, void 0, function* () {
+        params = Object.assign(Object.assign({}, params), { url: (0, utils_1.formatUrl)(params.url) });
+        // skip duplicate job
+        const existsJob = yield getJob({
+            url: params.url,
+            includeScreenshot: params.includeScreenshot,
+            includeHtml: params.includeHtml,
+            quality: params.quality,
+            width: params.width,
+            height: params.height,
+            fullPage: params.fullPage,
+        });
+        config_1.logger.info('create crawl job', params);
+        if (existsJob) {
+            config_1.logger.warn(`Crawl job already exists for ${params.url}, skip`);
+            return existsJob.id;
+        }
+        const jobId = (0, crypto_1.randomUUID)();
+        const job = crawlQueue.push(Object.assign(Object.assign({}, params), { id: jobId }));
+        job.on('finished', ({ result }) => {
+            config_1.logger.info(`Crawl completed ${params.url}, status: ${result ? 'success' : 'failed'}`, { job: params, result });
+            callback === null || callback === void 0 ? void 0 : callback(result);
+        });
+        job.on('failed', ({ error }) => {
+            config_1.logger.error(`Failed to execute job for ${params.url}`, { error, job: params });
+            callback === null || callback === void 0 ? void 0 : callback(null);
+        });
+        return jobId;
+    });
+}
+// @ts-ignore
+function getJob(condition) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const where = Object.keys(condition)
+            .filter((key) => condition[key] !== undefined)
+            .map((key) => {
+            return core_1.default.where(core_1.default.fn('json_extract', core_1.default.col('job'), `$.${key}`), condition[key]);
+        });
+        const job = yield crawlQueue.store.db.findOne({
+            where: {
+                [core_1.default.Op.and]: where,
+            },
+        });
+        if (job) {
+            return job.job;
+        }
+        return null;
+    });
+}
+function convertJobToSnapshot({ job, snapshot }) {
+    return Object.assign({ 
+        // @ts-ignore
+        jobId: job.jobId || job.id, url: job.url, options: {
+            width: job.width,
+            height: job.height,
+            includeScreenshot: job.includeScreenshot,
+            includeHtml: job.includeHtml,
+            quality: job.quality,
+            fullPage: job.fullPage,
+        } }, snapshot);
+}
+function formatSnapshot(snapshot, columns) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let data = Object.assign({}, snapshot);
+        // format screenshot path to full url
+        if (data.screenshot) {
+            data.screenshot = (0, ufo_1.joinURL)(config_1.config.appUrl, data.screenshot);
+        }
+        // format html path to string
+        if (data.html) {
+            const html = yield fs_extra_1.default.readFile(path_1.default.join(config_1.config.dataDir, data.html));
+            data.html = html.toString();
+        }
+        if (columns === null || columns === void 0 ? void 0 : columns.length) {
+            data = (0, pick_1.default)(data, columns);
+        }
+        return data;
+    });
+}
+/**
+ * get snapshot from db or crawl queue
+ */
+function getSnapshot(jobId) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const snapshotModel = yield snapshot_1.Snapshot.findByPk(jobId);
+        if (snapshotModel) {
+            return snapshotModel.toJSON();
+        }
+        const job = yield getJob({ id: jobId });
+        if (job) {
+            return {
+                jobId,
+                status: 'pending',
+            };
+        }
+        return null;
+    });
+}

package/dist/db/index.d.ts

@@ -0,0 +1 @@
+export declare function ensureDatabase(): Promise<void>;

package/dist/db/index.js

@@ -0,0 +1,41 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ensureDatabase = ensureDatabase;
+const core_1 = require("@sequelize/core");
+const sqlite3_1 = require("@sequelize/sqlite3");
+const path_1 = __importDefault(require("path"));
+const config_1 = require("../config");
+const job_1 = require("./job");
+const snapshot_1 = require("./snapshot");
+function ensureDatabase() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const sequelize = new core_1.Sequelize({
+            dialect: sqlite3_1.SqliteDialect,
+            storage: path_1.default.join(config_1.config.dataDir, 'snap-kit.db'),
+            logging: (msg) => config_1.logger.debug(msg),
+        });
+        yield (0, snapshot_1.initSnapshotModel)(sequelize);
+        yield (0, job_1.initJobModel)(sequelize);
+        try {
+            yield sequelize.authenticate();
+            yield sequelize.sync();
+            config_1.logger.info('Successfully connected to database');
+        }
+        catch (error) {
+            config_1.logger.error('Failed to connect to database:', error);
+            throw error;
+        }
+    });
+}

package/dist/db/job.d.ts

@@ -0,0 +1,33 @@
+import { Model, Sequelize } from '@sequelize/core';
+export interface JobState {
+    id?: string;
+    jobId: string;
+    url: string;
+    includeScreenshot?: boolean;
+    includeHtml?: boolean;
+    width?: number;
+    height?: number;
+    quality?: number;
+    timeout?: number;
+    fullPage?: boolean;
+}
+export interface JobModel {
+    id: string;
+    queue: string;
+    job: JobState;
+    retryCount: number;
+    willRunAt: number;
+    delay: number;
+    cancelled: boolean;
+}
+declare class Job extends Model<JobModel> implements JobModel {
+    id: JobModel['id'];
+    queue: JobModel['queue'];
+    job: JobModel['job'];
+    retryCount: JobModel['retryCount'];
+    willRunAt: JobModel['willRunAt'];
+    delay: JobModel['delay'];
+    cancelled: JobModel['cancelled'];
+}
+export { Job };
+export declare function initJobModel(sequelize: Sequelize): typeof Job;

package/dist/db/job.js

@@ -0,0 +1,54 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Job = void 0;
+exports.initJobModel = initJobModel;
+const core_1 = require("@sequelize/core");
+class Job extends core_1.Model {
+}
+exports.Job = Job;
+function initJobModel(sequelize) {
+    Job.init({
+        id: {
+            type: core_1.DataTypes.STRING(40),
+            primaryKey: true,
+        },
+        queue: {
+            type: core_1.DataTypes.STRING(32),
+            allowNull: false,
+        },
+        job: {
+            type: core_1.DataTypes.JSON,
+            allowNull: false,
+        },
+        retryCount: {
+            type: core_1.DataTypes.INTEGER,
+        },
+        delay: {
+            type: core_1.DataTypes.INTEGER,
+        },
+        willRunAt: {
+            type: core_1.DataTypes.INTEGER,
+        },
+        cancelled: {
+            type: core_1.DataTypes.BOOLEAN,
+            defaultValue: false,
+        },
+        createdAt: {
+            type: core_1.DataTypes.DATE,
+            defaultValue: core_1.DataTypes.NOW,
+            index: true,
+        },
+        updatedAt: {
+            type: core_1.DataTypes.DATE,
+            defaultValue: core_1.DataTypes.NOW,
+            index: true,
+        },
+    }, {
+        sequelize,
+        indexes: [{ fields: ['queue'] }],
+        modelName: 'job',
+        tableName: 'jobs',
+        timestamps: true,
+    });
+    return Job;
+}

package/dist/db/snapshot.d.ts

@@ -0,0 +1,31 @@
+import { Model, Sequelize } from '@sequelize/core';
+interface SnapshotModel {
+    jobId: string;
+    url: string;
+    status: 'success' | 'failed' | 'pending';
+    html?: string | null;
+    screenshot?: string | null;
+    error?: string;
+    lastModified?: string;
+    options?: {
+        width?: number;
+        height?: number;
+        includeScreenshot?: boolean;
+        includeHtml?: boolean;
+        quality?: number;
+        fullPage?: boolean;
+    };
+}
+declare class Snapshot extends Model<SnapshotModel> implements SnapshotModel {
+    jobId: SnapshotModel['jobId'];
+    url: SnapshotModel['url'];
+    status: SnapshotModel['status'];
+    html?: SnapshotModel['html'];
+    screenshot?: SnapshotModel['screenshot'];
+    error?: SnapshotModel['error'];
+    lastModified?: SnapshotModel['lastModified'];
+    options: SnapshotModel['options'];
+}
+export { Snapshot };
+export type { SnapshotModel };
+export declare function initSnapshotModel(sequelize: Sequelize): typeof Snapshot;

package/dist/db/snapshot.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Snapshot = void 0;
+exports.initSnapshotModel = initSnapshotModel;
+const core_1 = require("@sequelize/core");
+class Snapshot extends core_1.Model {
+}
+exports.Snapshot = Snapshot;
+function initSnapshotModel(sequelize) {
+    Snapshot.init({
+        jobId: {
+            type: core_1.DataTypes.STRING,
+            primaryKey: true,
+            allowNull: false,
+        },
+        url: {
+            type: core_1.DataTypes.STRING,
+            allowNull: false,
+            index: true,
+        },
+        status: {
+            type: core_1.DataTypes.ENUM('success', 'failed'),
+            allowNull: false,
+        },
+        html: {
+            type: core_1.DataTypes.TEXT,
+            allowNull: true,
+        },
+        screenshot: {
+            type: core_1.DataTypes.STRING,
+            allowNull: true,
+        },
+        error: {
+            type: core_1.DataTypes.STRING,
+            allowNull: true,
+        },
+        lastModified: {
+            type: core_1.DataTypes.STRING,
+            allowNull: true,
+        },
+        options: {
+            type: core_1.DataTypes.JSON,
+            allowNull: true,
+        },
+    }, {
+        sequelize,
+        modelName: 'snapshot',
+        tableName: 'snap',
+        timestamps: true,
+    });
+    return Snapshot;
+}

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+import { config } from './config';
+export * from './blocklet';
+export * from './crawler';
+export * from './middleware';
+export { Snapshot } from './db/snapshot';
+export declare function initCrawler(_config: Partial<typeof config>): Promise<void>;

package/dist/index.js

@@ -0,0 +1,45 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Snapshot = void 0;
+exports.initCrawler = initCrawler;
+const config_1 = require("./config");
+const crawler_1 = require("./crawler");
+const db_1 = require("./db");
+const puppeteer_1 = require("./puppeteer");
+__exportStar(require("./blocklet"), exports);
+__exportStar(require("./crawler"), exports);
+__exportStar(require("./middleware"), exports);
+var snapshot_1 = require("./db/snapshot");
+Object.defineProperty(exports, "Snapshot", { enumerable: true, get: function () { return snapshot_1.Snapshot; } });
+function initCrawler(_config) {
+    return __awaiter(this, void 0, void 0, function* () {
+        Object.assign(config_1.config, _config);
+        config_1.logger.debug('init crawler', config_1.config);
+        yield (0, db_1.ensureDatabase)();
+        yield (0, crawler_1.createCrawlQueue)();
+        yield (0, puppeteer_1.ensureBrowser)();
+    });
+}

package/dist/middleware.d.ts

@@ -0,0 +1,4 @@
+export declare function initSEOMiddleware({ autoReturnHtml, allowCrawler, }: {
+    autoReturnHtml?: Boolean;
+    allowCrawler?: Boolean | Function;
+}): (req: any, res: any, next: Function) => Promise<any>;

package/dist/middleware.js

@@ -0,0 +1,44 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initSEOMiddleware = initSEOMiddleware;
+const cache_1 = require("./cache");
+const utils_1 = require("./utils");
+function initSEOMiddleware({ autoReturnHtml = true, allowCrawler = true, }) {
+    return (req, res, next) => __awaiter(this, void 0, void 0, function* () {
+        const isBot = (0, utils_1.isBotUserAgent)(req);
+        const isSelf = (0, utils_1.isSelfCrawler)(req);
+        if (!isBot || isSelf) {
+            return next();
+        }
+        const fullUrl = (0, utils_1.getFullUrl)(req);
+        const canCrawl = yield (0, utils_1.isAcceptCrawler)(fullUrl);
+        const allowCrawlerResult = typeof allowCrawler === 'function' ? allowCrawler(req) : allowCrawler;
+        // can not crawl, skip
+        if (!canCrawl || !allowCrawlerResult) {
+            return next();
+        }
+        const cacheData = yield cache_1.useCache.get(fullUrl);
+        // add cached html to req
+        req.cachedHtml = (cacheData === null || cacheData === void 0 ? void 0 : cacheData.content) || cacheData || null;
+        // add cached lastModified to req, ISO string to GMT string
+        req.cachedLastmod = (cacheData === null || cacheData === void 0 ? void 0 : cacheData.lastModified) ? new Date(cacheData === null || cacheData === void 0 ? void 0 : cacheData.lastModified).toUTCString() : null;
+        if (req.cachedLastmod) {
+            res.setHeader('Last-Modified', req.cachedLastmod);
+        }
+        if (autoReturnHtml && req.cachedHtml) {
+            res.send(req.cachedHtml);
+            return;
+        }
+        // missing cache
+        next();
+    });
+}

package/dist/puppeteer.d.ts

@@ -0,0 +1,16 @@
+import puppeteer, { Browser, Page } from '@blocklet/puppeteer';
+export { puppeteer };
+export declare function ensurePuppeteerrc(): Promise<{
+    cacheDirectory: string;
+    temporaryDirectory: string;
+}>;
+export declare function ensureBrowser(): Promise<void>;
+export declare function connectBrowser(): Promise<Browser | null>;
+export declare function launchBrowser(): Promise<Browser>;
+export declare const getBrowser: () => Promise<Browser>;
+export declare const closeBrowser: ({ trimCache }?: {
+    trimCache?: boolean;
+}) => Promise<void>;
+export declare function initPage({ abortResourceTypes }?: {
+    abortResourceTypes?: never[] | undefined;
+}): Promise<Page>;