@arcblock/crawler 1.0.1 → 1.0.2

package/esm/db/index.js

@@ -0,0 +1,35 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { Sequelize } from '@sequelize/core';
+import { SqliteDialect } from '@sequelize/sqlite3';
+import path from 'path';
+import { config, logger } from '../config';
+import { initJobModel } from './job';
+import { initSnapshotModel } from './snapshot';
+export function ensureDatabase() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const sequelize = new Sequelize({
+            dialect: SqliteDialect,
+            storage: path.join(config.dataDir, 'snap-kit.db'),
+            logging: (msg) => logger.debug(msg),
+        });
+        yield initSnapshotModel(sequelize);
+        yield initJobModel(sequelize);
+        try {
+            yield sequelize.authenticate();
+            yield sequelize.sync();
+            logger.info('Successfully connected to database');
+        }
+        catch (error) {
+            logger.error('Failed to connect to database:', error);
+            throw error;
+        }
+    });
+}

package/esm/db/job.d.ts

@@ -0,0 +1,33 @@
+import { Model, Sequelize } from '@sequelize/core';
+export interface JobState {
+    id?: string;
+    jobId: string;
+    url: string;
+    includeScreenshot?: boolean;
+    includeHtml?: boolean;
+    width?: number;
+    height?: number;
+    quality?: number;
+    timeout?: number;
+    fullPage?: boolean;
+}
+export interface JobModel {
+    id: string;
+    queue: string;
+    job: JobState;
+    retryCount: number;
+    willRunAt: number;
+    delay: number;
+    cancelled: boolean;
+}
+declare class Job extends Model<JobModel> implements JobModel {
+    id: JobModel['id'];
+    queue: JobModel['queue'];
+    job: JobModel['job'];
+    retryCount: JobModel['retryCount'];
+    willRunAt: JobModel['willRunAt'];
+    delay: JobModel['delay'];
+    cancelled: JobModel['cancelled'];
+}
+export { Job };
+export declare function initJobModel(sequelize: Sequelize): typeof Job;

package/esm/db/job.js

@@ -0,0 +1,50 @@
+import { DataTypes, Model } from '@sequelize/core';
+class Job extends Model {
+}
+export { Job };
+export function initJobModel(sequelize) {
+    Job.init({
+        id: {
+            type: DataTypes.STRING(40),
+            primaryKey: true,
+        },
+        queue: {
+            type: DataTypes.STRING(32),
+            allowNull: false,
+        },
+        job: {
+            type: DataTypes.JSON,
+            allowNull: false,
+        },
+        retryCount: {
+            type: DataTypes.INTEGER,
+        },
+        delay: {
+            type: DataTypes.INTEGER,
+        },
+        willRunAt: {
+            type: DataTypes.INTEGER,
+        },
+        cancelled: {
+            type: DataTypes.BOOLEAN,
+            defaultValue: false,
+        },
+        createdAt: {
+            type: DataTypes.DATE,
+            defaultValue: DataTypes.NOW,
+            index: true,
+        },
+        updatedAt: {
+            type: DataTypes.DATE,
+            defaultValue: DataTypes.NOW,
+            index: true,
+        },
+    }, {
+        sequelize,
+        indexes: [{ fields: ['queue'] }],
+        modelName: 'job',
+        tableName: 'jobs',
+        timestamps: true,
+    });
+    return Job;
+}

package/esm/db/snapshot.d.ts

@@ -0,0 +1,31 @@
+import { Model, Sequelize } from '@sequelize/core';
+interface SnapshotModel {
+    jobId: string;
+    url: string;
+    status: 'success' | 'failed' | 'pending';
+    html?: string | null;
+    screenshot?: string | null;
+    error?: string;
+    lastModified?: string;
+    options?: {
+        width?: number;
+        height?: number;
+        includeScreenshot?: boolean;
+        includeHtml?: boolean;
+        quality?: number;
+        fullPage?: boolean;
+    };
+}
+declare class Snapshot extends Model<SnapshotModel> implements SnapshotModel {
+    jobId: SnapshotModel['jobId'];
+    url: SnapshotModel['url'];
+    status: SnapshotModel['status'];
+    html?: SnapshotModel['html'];
+    screenshot?: SnapshotModel['screenshot'];
+    error?: SnapshotModel['error'];
+    lastModified?: SnapshotModel['lastModified'];
+    options: SnapshotModel['options'];
+}
+export { Snapshot };
+export type { SnapshotModel };
+export declare function initSnapshotModel(sequelize: Sequelize): typeof Snapshot;

package/esm/db/snapshot.js

@@ -0,0 +1,48 @@
+import { DataTypes, Model } from '@sequelize/core';
+class Snapshot extends Model {
+}
+export { Snapshot };
+export function initSnapshotModel(sequelize) {
+    Snapshot.init({
+        jobId: {
+            type: DataTypes.STRING,
+            primaryKey: true,
+            allowNull: false,
+        },
+        url: {
+            type: DataTypes.STRING,
+            allowNull: false,
+            index: true,
+        },
+        status: {
+            type: DataTypes.ENUM('success', 'failed'),
+            allowNull: false,
+        },
+        html: {
+            type: DataTypes.TEXT,
+            allowNull: true,
+        },
+        screenshot: {
+            type: DataTypes.STRING,
+            allowNull: true,
+        },
+        error: {
+            type: DataTypes.STRING,
+            allowNull: true,
+        },
+        lastModified: {
+            type: DataTypes.STRING,
+            allowNull: true,
+        },
+        options: {
+            type: DataTypes.JSON,
+            allowNull: true,
+        },
+    }, {
+        sequelize,
+        modelName: 'snapshot',
+        tableName: 'snap',
+        timestamps: true,
+    });
+    return Snapshot;
+}

package/esm/index.d.ts

@@ -0,0 +1,6 @@
+import { config } from './config';
+export * from './blocklet';
+export * from './crawler';
+export * from './middleware';
+export { Snapshot } from './db/snapshot';
+export declare function initCrawler(_config: Partial<typeof config>): Promise<void>;

package/esm/index.js

@@ -0,0 +1,26 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { config, logger } from './config';
+import { createCrawlQueue } from './crawler';
+import { ensureDatabase } from './db';
+import { ensureBrowser } from './puppeteer';
+export * from './blocklet';
+export * from './crawler';
+export * from './middleware';
+export { Snapshot } from './db/snapshot';
+export function initCrawler(_config) {
+    return __awaiter(this, void 0, void 0, function* () {
+        Object.assign(config, _config);
+        logger.debug('init crawler', config);
+        yield ensureDatabase();
+        yield createCrawlQueue();
+        yield ensureBrowser();
+    });
+}

package/esm/middleware.d.ts

@@ -0,0 +1,4 @@
+export declare function initSEOMiddleware({ autoReturnHtml, allowCrawler, }: {
+    autoReturnHtml?: Boolean;
+    allowCrawler?: Boolean | Function;
+}): (req: any, res: any, next: Function) => Promise<any>;

package/esm/middleware.js

@@ -0,0 +1,41 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { useCache } from './cache';
+import { getFullUrl, isAcceptCrawler, isBotUserAgent, isSelfCrawler } from './utils';
+export function initSEOMiddleware({ autoReturnHtml = true, allowCrawler = true, }) {
+    return (req, res, next) => __awaiter(this, void 0, void 0, function* () {
+        const isBot = isBotUserAgent(req);
+        const isSelf = isSelfCrawler(req);
+        if (!isBot || isSelf) {
+            return next();
+        }
+        const fullUrl = getFullUrl(req);
+        const canCrawl = yield isAcceptCrawler(fullUrl);
+        const allowCrawlerResult = typeof allowCrawler === 'function' ? allowCrawler(req) : allowCrawler;
+        // can not crawl, skip
+        if (!canCrawl || !allowCrawlerResult) {
+            return next();
+        }
+        const cacheData = yield useCache.get(fullUrl);
+        // add cached html to req
+        req.cachedHtml = (cacheData === null || cacheData === void 0 ? void 0 : cacheData.content) || cacheData || null;
+        // add cached lastModified to req, ISO string to GMT string
+        req.cachedLastmod = (cacheData === null || cacheData === void 0 ? void 0 : cacheData.lastModified) ? new Date(cacheData === null || cacheData === void 0 ? void 0 : cacheData.lastModified).toUTCString() : null;
+        if (req.cachedLastmod) {
+            res.setHeader('Last-Modified', req.cachedLastmod);
+        }
+        if (autoReturnHtml && req.cachedHtml) {
+            res.send(req.cachedHtml);
+            return;
+        }
+        // missing cache
+        next();
+    });
+}

package/esm/puppeteer.d.ts

@@ -0,0 +1,16 @@
+import puppeteer, { Browser, Page } from '@blocklet/puppeteer';
+export { puppeteer };
+export declare function ensurePuppeteerrc(): Promise<{
+    cacheDirectory: string;
+    temporaryDirectory: string;
+}>;
+export declare function ensureBrowser(): Promise<void>;
+export declare function connectBrowser(): Promise<Browser | null>;
+export declare function launchBrowser(): Promise<Browser>;
+export declare const getBrowser: () => Promise<Browser>;
+export declare const closeBrowser: ({ trimCache }?: {
+    trimCache?: boolean;
+}) => Promise<void>;
+export declare function initPage({ abortResourceTypes }?: {
+    abortResourceTypes?: never[] | undefined;
+}): Promise<Page>;

package/esm/puppeteer.js

@@ -0,0 +1,272 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+// import fs from 'fs-extra';
+// import path from 'path';
+import puppeteer from '@blocklet/puppeteer';
+import { env } from '@blocklet/sdk/lib/config';
+import fs from 'fs-extra';
+import path from 'path';
+import { clearInterval, setInterval } from 'timers';
+import { useCache } from './cache';
+import { config, logger } from './config';
+import { CRAWLER_FLAG, sleep } from './utils';
+// let puppeteerConfig: {
+//   cacheDirectory: string;
+//   temporaryDirectory: string;
+// };
+const BROWSER_WS_ENDPOINT_KEY = `browserWSEndpoint-${env.appId || 'unknown'}`;
+const BrowserStatus = {
+    Launching: 'Launching',
+    Ready: 'Ready',
+};
+let browser;
+let browserActivatedTimer;
+export { puppeteer };
+export function ensurePuppeteerrc() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const cacheDirectory = path.join(config.cacheDir, 'puppeteer', 'cache');
+        const temporaryDirectory = path.join(config.cacheDir, 'puppeteer', 'tmp');
+        const puppeteerrcPath = path.join(config.appDir, '.puppeteerrc.js');
+        // ensure directory exists
+        yield Promise.all([fs.ensureDir(cacheDirectory), fs.ensureDir(temporaryDirectory), fs.ensureFile(puppeteerrcPath)]);
+        const puppeteerConfig = {
+            cacheDirectory,
+            temporaryDirectory,
+        };
+        const fileContent = `module.exports = ${JSON.stringify(puppeteerConfig, null, 2)}`;
+        yield fs.writeFile(puppeteerrcPath, fileContent);
+        logger.debug(`Puppeteerrc file created at ${puppeteerrcPath}`, puppeteerConfig);
+        return puppeteerConfig;
+    });
+}
+export function ensureBrowser() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const puppeteerConfig = yield ensurePuppeteerrc();
+        const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium';
+        logger.info('executablePath', executablePath);
+        if (!fs.existsSync(executablePath)) {
+            logger.info('start download browser', puppeteerConfig);
+            const { downloadBrowser } = yield (() => __awaiter(this, void 0, void 0, function* () {
+                try {
+                    // @ts-ignore
+                    // eslint-disable-next-line import/extensions
+                    return yield import('@blocklet/puppeteer/internal/node/install.js');
+                }
+                catch (err) {
+                    logger.warn('Skipping browser installation because the Puppeteer build is not available. Run `npm install` again after you have re-built Puppeteer.');
+                }
+            }))();
+            if (downloadBrowser) {
+                yield downloadBrowser();
+                logger.info('Browser download completed successfully');
+            }
+        }
+        // try to launch browser
+        if (config.testOnInitialize) {
+            const browser = yield launchBrowser();
+            if (!browser) {
+                throw new Error('Failed to launch browser');
+            }
+            yield closeBrowser();
+        }
+        logger.info('Puppeteer is ready');
+    });
+}
+export function connectBrowser() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const browserWSEndpoint = yield useCache.get(BROWSER_WS_ENDPOINT_KEY);
+        if (!browserWSEndpoint) {
+            return null;
+        }
+        // retry if browser is launching
+        if (browserWSEndpoint.status === BrowserStatus.Launching) {
+            yield sleep(Math.floor(Math.random() * 1000));
+            return connectBrowser();
+        }
+        try {
+            browser = yield puppeteer.connect({
+                browserWSEndpoint: browserWSEndpoint.endpoint,
+            });
+            logger.info('Connect browser success');
+        }
+        catch (err) {
+            logger.warn('Connect browser failed, clear endpoint', err);
+            yield useCache.remove(BROWSER_WS_ENDPOINT_KEY);
+            return null;
+        }
+        return browser;
+    });
+}
+export function launchBrowser() {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield useCache.set(BROWSER_WS_ENDPOINT_KEY, {
+            endpoint: null,
+            status: BrowserStatus.Launching,
+        });
+        try {
+            // @ts-ignore
+            browser = yield puppeteer.launch({
+                headless: true,
+                args: [
+                    // docs: https://peter.sh/experiments/chromium-command-line-switches/
+                    '--no-first-run',
+                    '--hide-scrollbars',
+                    '--no-sandbox',
+                    '--no-zygote',
+                    '--disable-setuid-sandbox',
+                    '--disable-gpu',
+                    '--disable-dev-shm-usage',
+                    '--disable-site-isolation-trials',
+                    '--disable-accelerated-2d-canvas',
+                    '--disable-extensions',
+                    '--js-flags=--max_old_space_size=512', // 限制V8内存
+                    '--disable-background-networking',
+                    '--disable-default-apps',
+                    // '--disable-web-security', // 允许跨域请求
+                    '--disable-software-rasterizer',
+                    '--disable-crash-reporter',
+                    '--disable-service-workers',
+                    '--disable-notifications',
+                    '--disable-infobars',
+                    '--font-render-hinting=none',
+                ],
+            });
+            logger.info('Launch browser success');
+        }
+        catch (error) {
+            logger.error('launch browser failed: ', error);
+            // cleanup browser endpoint
+            yield useCache.remove(BROWSER_WS_ENDPOINT_KEY);
+            throw error;
+        }
+        // save browserWSEndpoint to cache
+        const endpoint = yield browser.wsEndpoint();
+        yield useCache.set(BROWSER_WS_ENDPOINT_KEY, {
+            endpoint,
+            status: BrowserStatus.Ready,
+        });
+        return browser;
+    });
+}
+function checkBrowserActivated() {
+    clearBrowserActivatedTimer();
+    let count = 0;
+    browserActivatedTimer = setInterval(() => __awaiter(this, void 0, void 0, function* () {
+        var _a;
+        if (browser) {
+            const pages = yield browser.pages().catch(() => []);
+            if (pages.length === 1 && ((_a = pages[0]) === null || _a === void 0 ? void 0 : _a.url()) === 'about:blank') {
+                count++;
+                logger.debug(`Browser inactive count: ${count}/3`);
+            }
+            else {
+                count = 0; // 重置计数器！
+            }
+            if (count >= 3) {
+                logger.info('Browser inactive for 3 minutes, closing...');
+                yield closeBrowser({
+                    trimCache: true,
+                });
+            }
+        }
+    }), 1000 * 60);
+}
+function clearBrowserActivatedTimer() {
+    if (browserActivatedTimer) {
+        clearInterval(browserActivatedTimer);
+        browserActivatedTimer = null;
+    }
+}
+export const getBrowser = () => __awaiter(void 0, void 0, void 0, function* () {
+    if (browser)
+        return browser;
+    // sleep random time (0 ~ 5s),to avoid concurrent blocklet
+    yield sleep(Math.floor(Math.random() * 1000 * 5));
+    // try to connect browser
+    const connectedBrowser = yield connectBrowser();
+    if (connectedBrowser) {
+        logger.debug('getBrowser.connectedBrowser');
+        browser = connectedBrowser;
+        return browser;
+    }
+    // try to launch browser
+    const launchedBrowser = yield launchBrowser();
+    if (launchedBrowser) {
+        logger.debug('getBrowser.launchedBrowser');
+        browser = launchedBrowser;
+        checkBrowserActivated();
+        return browser;
+    }
+    throw new Error('No browser to use, should install redis or browser');
+});
+export const closeBrowser = (...args_1) => __awaiter(void 0, [...args_1], void 0, function* ({ trimCache = true } = {}) {
+    if (!browser)
+        return;
+    // close all pages
+    try {
+        const pages = yield browser.pages();
+        yield Promise.all(pages.map((page) => page.close()));
+    }
+    catch (err) {
+        logger.error('Failed to close all pages:', err);
+    }
+    // close browser
+    try {
+        yield browser.close();
+    }
+    catch (err) {
+        logger.error('Failed to close browser:', err);
+    }
+    // clear cache
+    try {
+        if (trimCache) {
+            yield puppeteer.trimCache();
+            logger.info('Trim cache success');
+        }
+        // try to clear temporary directory
+        // if (puppeteerConfig) {
+        //   await fs.emptyDir(puppeteerConfig.temporaryDirectory);
+        // }
+        if (global.gc) {
+            global.gc();
+        }
+    }
+    catch (err) {
+        logger.error('Failed to clear browser cache:', err);
+    }
+    browser = null;
+    clearBrowserActivatedTimer();
+    yield useCache.remove(BROWSER_WS_ENDPOINT_KEY);
+    logger.info('Close browser success');
+});
+export function initPage() {
+    return __awaiter(this, arguments, void 0, function* ({ abortResourceTypes = [] } = {}) {
+        const browser = yield getBrowser();
+        const page = yield browser.newPage();
+        yield page.setViewport({ width: 1440, height: 900 });
+        // page setting
+        // add custom headers
+        yield page.setExtraHTTPHeaders({
+            [CRAWLER_FLAG]: 'true',
+        });
+        // abort resource types
+        if (abortResourceTypes.length > 0) {
+            yield page.setRequestInterception(true);
+            page.on('request', (req) => {
+                // @ts-ignore
+                if (abortResourceTypes.includes(req.resourceType())) {
+                    return req.abort();
+                }
+                return req.continue();
+            });
+        }
+        return page;
+    });
+}

package/esm/utils.d.ts

@@ -0,0 +1,15 @@
+export declare const api: import("axios").AxiosInstance;
+export declare const sleep: (ms: number) => Promise<unknown>;
+export declare const CRAWLER_FLAG = "x-crawler";
+export declare const isSelfCrawler: (req: any) => boolean;
+export declare const getDefaultRobotsUrl: (url: string) => string;
+export declare function getRobots(url: string): Promise<import("robots-parser").Robot | null>;
+export declare const getDefaultSitemapUrl: (url: string) => string;
+export declare const isAcceptCrawler: (url: string) => Promise<boolean | undefined>;
+export declare const getSitemapList: (url: string) => Promise<import("sitemap").SitemapItem[]>;
+export declare const isBotUserAgent: (req: any) => boolean;
+export declare const getComponentInfo: () => {};
+export declare const getFullUrl: (req: any) => string;
+export declare const getRelativePath: (url: string) => string;
+export declare const formatUrl: (url: string) => string;
+export declare function md5(content: string | Uint8Array): string;