@arcblock/crawler 1.0.6 → 1.1.1

package/{esm → lib/esm}/index.js

@@ -7,20 +7,29 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+import merge from 'lodash/merge';
 import { config, logger } from './config';
 import { createCrawlQueue } from './crawler';
-import { ensureDatabase } from './db';
+import { initCron } from './cron';
 import { ensureBrowser } from './puppeteer';
-export * from './blocklet';
+import { initDatabase } from './store';
 export * from './crawler';
-export * from './middleware';
-export { Snapshot } from './db/snapshot';
-export function initCrawler(_config) {
+export * from './site';
+export * from './services/snapshot';
+export * as utils from './utils';
+export function initCrawler(params) {
     return __awaiter(this, void 0, void 0, function* () {
-        Object.assign(config, _config);
-        logger.debug('init crawler', config);
-        yield ensureDatabase();
-        yield createCrawlQueue();
-        yield ensureBrowser();
+        logger.info('Init crawler', { params });
+        merge(config, params);
+        try {
+            yield initDatabase();
+            yield ensureBrowser();
+            yield createCrawlQueue();
+            yield initCron();
+        }
+        catch (err) {
+            logger.error('Init crawler error', { err });
+            throw err;
+        }
     });
 }

package/{dist → lib/esm}/puppeteer.d.ts

@@ -1,4 +1,4 @@
-import puppeteer, { Browser, Page } from '@blocklet/puppeteer';
+import puppeteer, { Browser, Page, ResourceType } from '@blocklet/puppeteer';
 export { puppeteer };
 export declare function ensurePuppeteerrc(): Promise<{
     cacheDirectory: string;
@@ -12,5 +12,5 @@ export declare const closeBrowser: ({ trimCache }?: {
     trimCache?: boolean;
 }) => Promise<void>;
 export declare function initPage({ abortResourceTypes }?: {
-    abortResourceTypes?: never[] | undefined;
+    abortResourceTypes?: ResourceType[];
 }): Promise<Page>;

package/{esm → lib/esm}/puppeteer.js

@@ -7,25 +7,20 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-// import fs from 'fs-extra';
-// import path from 'path';
 import puppeteer from '@blocklet/puppeteer';
-import { env } from '@blocklet/sdk/lib/config';
 import fs from 'fs-extra';
 import path from 'path';
 import { clearInterval, setInterval } from 'timers';
-import { useCache } from './cache';
 import { config, logger } from './config';
 import { CRAWLER_FLAG, sleep } from './utils';
-// let puppeteerConfig: {
-//   cacheDirectory: string;
-//   temporaryDirectory: string;
-// };
-const BROWSER_WS_ENDPOINT_KEY = `browserWSEndpoint-${env.appId || 'unknown'}`;
 const BrowserStatus = {
+    None: 'None',
     Launching: 'Launching',
     Ready: 'Ready',
 };
+let browserStatus = BrowserStatus.None;
+/** Chromium WebSocket endpoint that allows puppeteer browser instance to connect to the browser */
+let browserEndpoint = '';
 let browser;
 let browserActivatedTimer;
 export { puppeteer };
@@ -49,9 +44,9 @@ export function ensurePuppeteerrc() {
 export function ensureBrowser() {
     return __awaiter(this, void 0, void 0, function* () {
         const puppeteerConfig = yield ensurePuppeteerrc();
-        const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium';
-        logger.debug('Chromium executablePath', executablePath);
-        if (!fs.existsSync(executablePath)) {
+        const executablePath = config.puppeteerPath;
+        logger.debug('executablePath', executablePath);
+        if (!executablePath || !fs.existsSync(executablePath)) {
             logger.info('start download browser', puppeteerConfig);
             const { downloadBrowser } = yield (() => __awaiter(this, void 0, void 0, function* () {
                 try {
@@ -69,7 +64,7 @@ export function ensureBrowser() {
             }
         }
         // try to launch browser
-        if (config.testOnInitialize) {
+        if (config.isProd) {
             const browser = yield launchBrowser();
             if (!browser) {
                 throw new Error('Failed to launch browser');
@@ -81,24 +76,23 @@ export function ensureBrowser() {
 }
 export function connectBrowser() {
     return __awaiter(this, void 0, void 0, function* () {
-        const browserWSEndpoint = yield useCache.get(BROWSER_WS_ENDPOINT_KEY);
-        if (!browserWSEndpoint) {
+        if (!browserEndpoint) {
             return null;
         }
         // retry if browser is launching
-        if (browserWSEndpoint.status === BrowserStatus.Launching) {
+        if (browserStatus === BrowserStatus.Launching) {
             yield sleep(Math.floor(Math.random() * 1000));
             return connectBrowser();
         }
         try {
             browser = yield puppeteer.connect({
-                browserWSEndpoint: browserWSEndpoint.endpoint,
+                browserWSEndpoint: browserEndpoint,
             });
             logger.info('Connect browser success');
         }
         catch (err) {
             logger.warn('Connect browser failed, clear endpoint', err);
-            yield useCache.remove(BROWSER_WS_ENDPOINT_KEY);
+            browserEndpoint = '';
             return null;
         }
         return browser;
@@ -106,12 +100,9 @@ export function connectBrowser() {
 }
 export function launchBrowser() {
     return __awaiter(this, void 0, void 0, function* () {
-        yield useCache.set(BROWSER_WS_ENDPOINT_KEY, {
-            endpoint: null,
-            status: BrowserStatus.Launching,
-        });
+        browserEndpoint = '';
+        browserStatus = BrowserStatus.Launching;
         try {
-            // @ts-ignore
             browser = yield puppeteer.launch({
                 headless: true,
                 args: [
@@ -142,16 +133,13 @@ export function launchBrowser() {
         }
         catch (error) {
             logger.error('launch browser failed: ', error);
-            // cleanup browser endpoint
-            yield useCache.remove(BROWSER_WS_ENDPOINT_KEY);
+            browserStatus = BrowserStatus.None;
+            browserEndpoint = '';
             throw error;
         }
         // save browserWSEndpoint to cache
-        const endpoint = yield browser.wsEndpoint();
-        yield useCache.set(BROWSER_WS_ENDPOINT_KEY, {
-            endpoint,
-            status: BrowserStatus.Ready,
-        });
+        browserEndpoint = yield browser.wsEndpoint();
+        browserStatus = BrowserStatus.Ready;
         return browser;
     });
 }
@@ -194,6 +182,7 @@ export const getBrowser = () => __awaiter(void 0, void 0, void 0, function* () {
     if (connectedBrowser) {
         logger.debug('getBrowser.connectedBrowser');
         browser = connectedBrowser;
+        checkBrowserActivated();
         return browser;
     }
     // try to launch browser
@@ -243,7 +232,8 @@ export const closeBrowser = (...args_1) => __awaiter(void 0, [...args_1], void 0
     }
     browser = null;
     clearBrowserActivatedTimer();
-    yield useCache.remove(BROWSER_WS_ENDPOINT_KEY);
+    browserEndpoint = '';
+    browserStatus = BrowserStatus.None;
     logger.info('Close browser success');
 });
 export function initPage() {
@@ -260,7 +250,6 @@ export function initPage() {
         if (abortResourceTypes.length > 0) {
             yield page.setRequestInterception(true);
             page.on('request', (req) => {
-                // @ts-ignore
                 if (abortResourceTypes.includes(req.resourceType())) {
                     return req.abort();
                 }

package/lib/esm/services/snapshot.d.ts

@@ -0,0 +1,12 @@
+import { JobState } from '../store/job';
+import { SnapshotModel } from '../store/snapshot';
+export declare function convertJobToSnapshot({ job, snapshot }: {
+    job: JobState;
+    snapshot?: Partial<SnapshotModel>;
+}): SnapshotModel;
+export declare function formatSnapshot(snapshot: SnapshotModel, columns?: Array<keyof SnapshotModel>): Promise<SnapshotModel>;
+/**
+ * get snapshot from db or crawl queue
+ */
+export declare function getSnapshot(jobId: string): Promise<SnapshotModel | null>;
+export declare function getLatestSnapshot(url: string): Promise<SnapshotModel | null>;

package/lib/esm/services/snapshot.js

@@ -0,0 +1,75 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import pick from 'lodash/pick';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { joinURL } from 'ufo';
+import { config } from '../config';
+import { Job } from '../store/job';
+import { Snapshot } from '../store/snapshot';
+import { formatUrl } from '../utils';
+export function convertJobToSnapshot({ job, snapshot }) {
+    return Object.assign({ jobId: job.jobId || job.id, url: job.url, lastModified: job.lastModified || new Date().toISOString(), options: {
+            width: job.width,
+            height: job.height,
+            includeScreenshot: job.includeScreenshot,
+            includeHtml: job.includeHtml,
+            quality: job.quality,
+            fullPage: job.fullPage,
+        } }, snapshot);
+}
+export function formatSnapshot(snapshot, columns) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let data = Object.assign({}, snapshot);
+        // format screenshot path to full url
+        if (data.screenshot) {
+            data.screenshot = joinURL(config.appUrl, data.screenshot);
+        }
+        // format html path to string
+        if (data.html) {
+            const html = yield fs.readFile(path.join(config.dataDir, data.html));
+            data.html = html.toString();
+        }
+        if (columns === null || columns === void 0 ? void 0 : columns.length) {
+            data = pick(data, columns);
+        }
+        return data;
+    });
+}
+/**
+ * get snapshot from db or crawl queue
+ */
+export function getSnapshot(jobId) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const snapshot = yield Snapshot.findSnapshot({ where: { jobId } });
+        if (snapshot) {
+            return formatSnapshot(snapshot);
+        }
+        const job = yield Job.findJob({ id: jobId });
+        if (job) {
+            return {
+                jobId,
+                status: 'pending',
+            };
+        }
+        return null;
+    });
+}
+export function getLatestSnapshot(url) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const snapshot = yield Snapshot.findSnapshot({
+            where: {
+                url: formatUrl(url),
+                status: 'success',
+            },
+        });
+        return snapshot ? formatSnapshot(snapshot) : null;
+    });
+}

package/lib/esm/site.d.ts

@@ -0,0 +1,2 @@
+import { Site } from './config';
+export declare const crawlSite: ({ url, pathname, interval }: Site) => Promise<(string | undefined)[]>;

package/lib/esm/site.js

@@ -0,0 +1,69 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import uniq from 'lodash/uniq';
+import pMap from 'p-map';
+import { config, logger } from './config';
+import { crawlUrl } from './crawler';
+import { Snapshot } from './store/snapshot';
+import { formatUrl, getSitemapList } from './utils';
+const crawlBlockletRunningMap = new Map();
+function parseSitemapUrl(sitemapItem) {
+    var _a;
+    const links = ((_a = sitemapItem.links) === null || _a === void 0 ? void 0 : _a.map((item) => item.url)) || [];
+    const urls = uniq([...links, sitemapItem.url]).filter(Boolean);
+    return urls.map((url) => ({ url, sitemapItem }));
+}
+export const crawlSite = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, pathname, interval = 0 }) {
+    logger.info(`Start crawl from sitemap ${url}`, { pathname });
+    const sitemapList = yield getSitemapList(url);
+    const pathnameRegex = new RegExp(pathname);
+    const sitemapItems = sitemapList
+        .filter((item) => new URL(item.url).pathname.match(pathnameRegex))
+        .flatMap((sitemapItem) => {
+        return parseSitemapUrl(sitemapItem);
+    });
+    logger.info(`Found ${sitemapItems.length} sitemap items which match ${pathname} from ${url}`);
+    const crawlableItems = (yield Promise.all(sitemapItems.map((_a) => __awaiter(void 0, [_a], void 0, function* ({ url, sitemapItem }) {
+        const snapshot = yield Snapshot.findOne({ where: { url: formatUrl(url) } });
+        if (snapshot === null || snapshot === void 0 ? void 0 : snapshot.lastModified) {
+            const lastModified = new Date(snapshot.lastModified);
+            // skip if snapshot lastModified is greater than sitemap lastmod
+            if (sitemapItem.lastmod && lastModified >= new Date(sitemapItem.lastmod)) {
+                return null;
+            }
+            // skip if interval time has not been reached
+            if (Date.now() - lastModified.getTime() < interval * 24 * 60 * 60 * 1000) {
+                return null;
+            }
+        }
+        return { url, sitemapItem };
+    })))).filter(Boolean);
+    logger.info(`Found ${crawlableItems.length} pages to crawl from sitemap ${url}`, { pathname });
+    const key = `${url}-${pathname}`;
+    crawlBlockletRunningMap.set(key, crawlableItems);
+    try {
+        const jobIds = yield pMap(crawlableItems, ({ url, sitemapItem }) => {
+            return crawlUrl({
+                url,
+                lastModified: sitemapItem.lastmod,
+                includeScreenshot: false,
+                includeHtml: true,
+            });
+        }, { concurrency: config.siteCron.concurrency });
+        return jobIds;
+    }
+    catch (error) {
+        logger.error(`Failed to crawl from sitemap ${url} ${pathname}`, error);
+        throw new Error(error);
+    }
+    finally {
+        crawlBlockletRunningMap.delete(key);
+    }
+});

package/lib/esm/store/index.d.ts

@@ -0,0 +1,3 @@
+import { Sequelize } from '@sequelize/core';
+import { SqliteDialect } from '@sequelize/sqlite3';
+export declare function initDatabase(): Promise<Sequelize<SqliteDialect>>;

package/{esm/db → lib/esm/store}/index.js

@@ -11,18 +11,33 @@ import { Sequelize } from '@sequelize/core';
 import { SqliteDialect } from '@sequelize/sqlite3';
 import path from 'path';
 import { config, logger } from '../config';
-import { initJobModel } from './job';
-import { initSnapshotModel } from './snapshot';
-export function ensureDatabase() {
+import { Job } from './job';
+import { Snapshot } from './snapshot';
+export function initDatabase() {
     return __awaiter(this, void 0, void 0, function* () {
         const sequelize = new Sequelize({
             dialect: SqliteDialect,
             storage: path.join(config.dataDir, 'snap-kit.db'),
-            logging: (msg) => logger.debug(msg),
+            logging: (msg) => process.env.SQLITE_LOG && logger.debug(msg),
+            pool: {
+                min: 0,
+                max: 10,
+                idle: 10000,
+            },
+            retry: {
+                match: [/SQLITE_BUSY/],
+                name: 'query',
+                max: 10,
+            },
         });
-        yield initSnapshotModel(sequelize);
-        yield initJobModel(sequelize);
+        Job.initModel(sequelize);
+        Snapshot.initModel(sequelize);
         try {
+            yield Promise.all([
+                sequelize.query('pragma journal_mode = WAL;'),
+                sequelize.query('pragma synchronous = normal;'),
+                sequelize.query('pragma journal_size_limit = 67108864;'),
+            ]);
             yield sequelize.authenticate();
             yield sequelize.sync();
             logger.info('Successfully connected to database');
@@ -31,5 +46,6 @@ export function ensureDatabase() {
             logger.error('Failed to connect to database:', error);
             throw error;
         }
+        return sequelize;
     });
 }

package/{esm/db → lib/esm/store}/job.d.ts

@@ -10,6 +10,7 @@ export interface JobState {
     quality?: number;
     timeout?: number;
     fullPage?: boolean;
+    lastModified?: string;
 }
 export interface JobModel {
     id: string;
@@ -20,7 +21,7 @@ export interface JobModel {
     delay: number;
     cancelled: boolean;
 }
-declare class Job extends Model<JobModel> implements JobModel {
+export declare class Job extends Model<JobModel> implements JobModel {
     id: JobModel['id'];
     queue: JobModel['queue'];
     job: JobModel['job'];
@@ -28,6 +29,6 @@ declare class Job extends Model<JobModel> implements JobModel {
     willRunAt: JobModel['willRunAt'];
     delay: JobModel['delay'];
     cancelled: JobModel['cancelled'];
+    static initModel(sequelize: Sequelize): typeof Job;
+    static findJob(condition: Partial<JobState>): Promise<JobModel | null>;
 }
-export { Job };
-export declare function initJobModel(sequelize: Sequelize): typeof Job;

package/lib/esm/store/job.js

@@ -0,0 +1,73 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import sequelize, { DataTypes, Model } from '@sequelize/core';
+export class Job extends Model {
+    static initModel(sequelize) {
+        return Job.init({
+            id: {
+                type: DataTypes.STRING(40),
+                primaryKey: true,
+            },
+            queue: {
+                type: DataTypes.STRING(32),
+                allowNull: false,
+            },
+            job: {
+                type: DataTypes.JSON,
+                allowNull: false,
+            },
+            retryCount: {
+                type: DataTypes.INTEGER,
+            },
+            delay: {
+                type: DataTypes.INTEGER,
+            },
+            willRunAt: {
+                type: DataTypes.INTEGER,
+            },
+            cancelled: {
+                type: DataTypes.BOOLEAN,
+                defaultValue: false,
+            },
+            createdAt: {
+                type: DataTypes.DATE,
+                defaultValue: DataTypes.NOW,
+                index: true,
+            },
+            updatedAt: {
+                type: DataTypes.DATE,
+                defaultValue: DataTypes.NOW,
+                index: true,
+            },
+        }, {
+            sequelize,
+            indexes: [{ fields: ['queue'] }],
+            modelName: 'job',
+            tableName: 'jobs',
+            timestamps: true,
+        });
+    }
+    static findJob(condition) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const where = Object.keys(condition)
+                .filter((key) => condition[key] !== undefined)
+                .map((key) => {
+                return sequelize.where(sequelize.fn('json_extract', sequelize.col('job'), `$.${key}`), condition[key]);
+            });
+            const job = yield Job.findOne({
+                where: {
+                    [sequelize.Op.and]: where,
+                },
+                order: [['createdAt', 'DESC']],
+            });
+            return (job === null || job === void 0 ? void 0 : job.toJSON()) || null;
+        });
+    }
+}

package/{esm/db → lib/esm/store}/snapshot.d.ts

@@ -1,5 +1,5 @@
-import { Model, Sequelize } from '@sequelize/core';
-interface SnapshotModel {
+import { FindOptions, Model, Sequelize } from '@sequelize/core';
+export interface SnapshotModel {
     jobId: string;
     url: string;
     status: 'success' | 'failed' | 'pending';
@@ -16,7 +16,7 @@ interface SnapshotModel {
         fullPage?: boolean;
     };
 }
-declare class Snapshot extends Model<SnapshotModel> implements SnapshotModel {
+export declare class Snapshot extends Model<SnapshotModel> implements SnapshotModel {
     jobId: SnapshotModel['jobId'];
     url: SnapshotModel['url'];
     status: SnapshotModel['status'];
@@ -25,7 +25,6 @@ declare class Snapshot extends Model<SnapshotModel> implements SnapshotModel {
     error?: SnapshotModel['error'];
     lastModified?: SnapshotModel['lastModified'];
     options: SnapshotModel['options'];
+    static initModel(sequelize: Sequelize): typeof Snapshot;
+    static findSnapshot(condition: FindOptions<SnapshotModel>): Promise<SnapshotModel | null>;
 }
-export { Snapshot };
-export type { SnapshotModel };
-export declare function initSnapshotModel(sequelize: Sequelize): typeof Snapshot;

package/lib/esm/store/snapshot.js

@@ -0,0 +1,64 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { DataTypes, Model } from '@sequelize/core';
+export class Snapshot extends Model {
+    static initModel(sequelize) {
+        return Snapshot.init({
+            jobId: {
+                type: DataTypes.STRING,
+                primaryKey: true,
+                allowNull: false,
+            },
+            url: {
+                type: DataTypes.STRING,
+                allowNull: false,
+                index: true,
+            },
+            status: {
+                type: DataTypes.ENUM('success', 'failed', 'pending'),
+                allowNull: false,
+            },
+            html: {
+                type: DataTypes.TEXT,
+                allowNull: true,
+            },
+            screenshot: {
+                type: DataTypes.STRING,
+                allowNull: true,
+            },
+            error: {
+                type: DataTypes.STRING,
+                allowNull: true,
+            },
+            lastModified: {
+                type: DataTypes.STRING,
+                allowNull: true,
+            },
+            options: {
+                type: DataTypes.JSON,
+                allowNull: true,
+            },
+        }, {
+            sequelize,
+            modelName: 'snapshot',
+            tableName: 'snap',
+            timestamps: true,
+        });
+    }
+    static findSnapshot(condition) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const snapshot = yield Snapshot.findOne(Object.assign({ order: [
+                    ['lastModified', 'DESC'],
+                    ['updatedAt', 'DESC'],
+                ] }, condition));
+            return (snapshot === null || snapshot === void 0 ? void 0 : snapshot.toJSON()) || null;
+        });
+    }
+}

package/lib/esm/utils.d.ts

@@ -0,0 +1,32 @@
+import { Page } from '@blocklet/puppeteer';
+import { Request } from 'express';
+export declare const axios: import("axios").AxiosInstance;
+export declare const CRAWLER_FLAG = "x-arcblock-crawler";
+export declare const sleep: (ms: number) => Promise<unknown>;
+/**
+ * Check if the request is a arcblock crawler
+ */
+export declare const isSelfCrawler: (req: Request) => boolean;
+/**
+ * Check if the request is a static file
+ */
+export declare function isStaticFile(req: Request): boolean;
+/**
+ * Check if the request is a spider
+ */
+export declare function isSpider(req: Request): boolean;
+/**
+ * Get and parse the robots.txt by `robots-parser`
+ */
+export declare function getRobots(url: string): Promise<import("robots-parser").Robot | null>;
+/**
+ * Check if the url is allowed to crawl from robots.txt
+ */
+export declare const isAcceptCrawler: (url: string) => Promise<boolean | undefined>;
+/**
+ * Get and parse the sitemap.xml by `sitemap` package
+ */
+export declare const getSitemapList: (url: string) => Promise<import("sitemap").SitemapItem[]>;
+export declare const formatUrl: (url: string) => string;
+export declare function md5(content: string | Uint8Array): string;
+export declare function findMaxScrollHeight(page: Page): Promise<number>;