@arcblock/crawler 1.0.5 → 1.1.1

package/README.md

@@ -0,0 +1 @@
+# @arcblock/crawler

package/lib/cjs/config.d.ts

@@ -0,0 +1,22 @@
+export type Site = {
+    url: string;
+    pathname: string;
+    /** Minimum crawl interval to avoid frequent crawling by scheduled tasks, in milliseconds */
+    interval?: number;
+};
+export type Config = {
+    isProd: boolean;
+    dataDir: string;
+    appDir: string;
+    appUrl: string;
+    cacheDir: string;
+    puppeteerPath?: string;
+    siteCron: {
+        sites: Site[];
+        time: string;
+        runOnInit: boolean;
+        concurrency: number;
+    };
+};
+export declare const logger: any;
+export declare const config: Config;

package/{dist → lib/cjs}/config.js

@@ -5,13 +5,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.config = exports.logger = void 0;
 const logger_1 = __importDefault(require("@blocklet/logger"));
-exports.logger = (0, logger_1.default)('crawler', { level: process.env.LOG_LEVEL || 'info' });
+exports.logger = (0, logger_1.default)('@arcblock/crawler', { level: process.env.LOG_LEVEL || 'info' });
 exports.config = {
-    redisUrl: process.env.REDIS_URL,
+    isProd: process.env.NODE_ENV === 'production',
     dataDir: process.env.BLOCKLET_DATA_DIR,
     appDir: process.env.BLOCKLET_APP_DIR || process.cwd(),
     appUrl: process.env.BLOCKLET_APP_URL,
     puppeteerPath: process.env.PUPPETEER_EXECUTABLE_PATH,
     cacheDir: process.env.BLOCKLET_CACHE_DIR,
-    testOnInitialize: process.env.NODE_ENV === 'production',
+    // cron
+    siteCron: {
+        sites: [],
+        time: '0 0 */12 * * *',
+        runOnInit: false,
+        concurrency: 5,
+    },
 };

package/lib/cjs/crawler.d.ts

@@ -0,0 +1,26 @@
+import { JobState } from './store/job';
+import { SnapshotModel } from './store/snapshot';
+export declare function createCrawlQueue(): void;
+export declare function getDataDir(): Promise<{
+    htmlDir: string;
+    screenshotDir: string;
+}>;
+export declare const getPageContent: ({ url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, }: {
+    url: string;
+    includeScreenshot?: boolean;
+    includeHtml?: boolean;
+    width?: number;
+    height?: number;
+    quality?: number;
+    timeout?: number;
+    fullPage?: boolean;
+}) => Promise<{
+    html: string;
+    screenshot: Uint8Array<ArrayBufferLike> | null;
+}>;
+/**
+ * crawl url and return job id
+ * @param params
+ * @param callback callback when job finished
+ */
+export declare function crawlUrl(params: Omit<JobState, 'jobId'>, callback?: (snapshot: SnapshotModel | null) => void): Promise<string | undefined>;

package/{dist → lib/cjs}/crawler.js

@@ -15,22 +15,17 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.getPageContent = void 0;
 exports.createCrawlQueue = createCrawlQueue;
 exports.getDataDir = getDataDir;
-exports.createCrawlJob = createCrawlJob;
-exports.getJob = getJob;
-exports.formatSnapshot = formatSnapshot;
-exports.getSnapshot = getSnapshot;
+exports.crawlUrl = crawlUrl;
 const queue_1 = __importDefault(require("@abtnode/queue"));
 const sequelize_1 = __importDefault(require("@abtnode/queue/lib/store/sequelize"));
-const core_1 = __importDefault(require("@sequelize/core"));
 const crypto_1 = require("crypto");
 const fs_extra_1 = __importDefault(require("fs-extra"));
-const pick_1 = __importDefault(require("lodash/pick"));
 const path_1 = __importDefault(require("path"));
-const ufo_1 = require("ufo");
 const config_1 = require("./config");
-const job_1 = require("./db/job");
-const snapshot_1 = require("./db/snapshot");
 const puppeteer_1 = require("./puppeteer");
+const snapshot_1 = require("./services/snapshot");
+const job_1 = require("./store/job");
+const snapshot_2 = require("./store/snapshot");
 const utils_1 = require("./utils");
 const { BaseState } = require('@abtnode/models');
 let crawlQueue;
@@ -40,18 +35,18 @@ function createCrawlQueue() {
         store: new sequelize_1.default(db, 'crawler'),
         concurrency: 1,
         onJob: (job) => __awaiter(this, void 0, void 0, function* () {
-            config_1.logger.debug('job start:', job);
+            config_1.logger.info('Starting to execute crawl job', job);
             const canCrawl = yield (0, utils_1.isAcceptCrawler)(job.url);
             if (!canCrawl) {
                 config_1.logger.error(`failed to crawl ${job.url}, denied by robots.txt`, job);
-                const snapshot = convertJobToSnapshot({
+                const snapshot = (0, snapshot_1.convertJobToSnapshot)({
                     job,
                     snapshot: {
                         status: 'failed',
                         error: 'Denied by robots.txt',
                     },
                 });
-                yield snapshot_1.Snapshot.upsert(snapshot);
+                yield snapshot_2.Snapshot.upsert(snapshot);
                 return snapshot;
             }
             // if index reach autoCloseBrowserCount, close browser
@@ -67,14 +62,14 @@ function createCrawlQueue() {
                 const result = yield (0, exports.getPageContent)(job);
                 if (!result || (!result.html && !result.screenshot)) {
                     config_1.logger.error(`failed to crawl ${job.url}, empty content`, job);
-                    const snapshot = convertJobToSnapshot({
+                    const snapshot = (0, snapshot_1.convertJobToSnapshot)({
                         job,
                         snapshot: {
                             status: 'failed',
                             error: 'Failed to crawl content',
                         },
                     });
-                    yield snapshot_1.Snapshot.upsert(snapshot);
+                    yield snapshot_2.Snapshot.upsert(snapshot);
                     return snapshot;
                 }
                 // save html and screenshot to data dir
@@ -83,7 +78,7 @@ function createCrawlQueue() {
                     html: result.html,
                 });
                 // const lastModified = job.lastmodMap?.get(url) || new Date().toISOString();
-                const snapshot = convertJobToSnapshot({
+                const snapshot = (0, snapshot_1.convertJobToSnapshot)({
                     job,
                     snapshot: {
                         status: 'success',
@@ -91,29 +86,19 @@ function createCrawlQueue() {
                         html: htmlPath === null || htmlPath === void 0 ? void 0 : htmlPath.replace(config_1.config.dataDir, ''),
                     },
                 });
-                yield snapshot_1.Snapshot.upsert(snapshot);
+                yield snapshot_2.Snapshot.upsert(snapshot);
                 return snapshot;
-                // save to redis
-                // if (saveToRedis) {
-                //   useCache.set(url, {
-                //     html: result.html || '',
-                //     lastModified,
-                //   });
-                //   logger.info(`success to crawl ${url}`, job);
-                //   return result;
-                // }
             }
             catch (error) {
                 config_1.logger.error(`Failed to crawl ${job.url}`, { error, job });
-                console.error(error.stack);
-                const snapshot = convertJobToSnapshot({
+                const snapshot = (0, snapshot_1.convertJobToSnapshot)({
                     job,
                     snapshot: {
                         status: 'failed',
                         error: 'Internal error',
                     },
                 });
-                yield snapshot_1.Snapshot.upsert(snapshot);
+                yield snapshot_2.Snapshot.upsert(snapshot);
                 return snapshot;
             }
         }),
@@ -157,11 +142,11 @@ function formatHtml(htmlString) {
     }
     return htmlString;
 }
-const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, formatPageContent, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 60 * 1000, fullPage = false, }) {
+const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, fullPage = false, }) {
     config_1.logger.debug('getPageContent', { url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage });
     const page = yield (0, puppeteer_1.initPage)();
     if (width && height) {
-        yield page.setViewport({ width, height });
+        yield page.setViewport({ width, height, deviceScaleFactor: 2 });
     }
     let html = null;
     let screenshot = null;
@@ -176,26 +161,47 @@ const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url,
             throw new Error(`Request failed with status ${statusCode}, in ${url}`);
         }
         // await for networkidle0
-        // https://pptr.dev/api/puppeteer.page.goforward/#remarks
+        // https://pptr.dev/api/puppeteer.page.waitfornetworkidle
         yield page.waitForNetworkIdle({
-            idleTime: 2 * 1000,
+            idleTime: 1.5 * 1000,
         });
         // get screenshot
         if (includeScreenshot) {
+            // Try to find the tallest element and set the browser to the same height
+            if (fullPage) {
+                const maxScrollHeight = yield (0, utils_1.findMaxScrollHeight)(page);
+                config_1.logger.info('findMaxScrollHeight', { maxScrollHeight });
+                if (maxScrollHeight) {
+                    yield page.setViewport({ width, height: maxScrollHeight || height, deviceScaleFactor: 2 });
+                    yield page.evaluate((scrollHeight) => {
+                        window.scrollTo(0, scrollHeight || 0);
+                        document.documentElement.scrollTo(0, scrollHeight || 0);
+                    }, maxScrollHeight);
+                }
+            }
             try {
                 screenshot = yield page.screenshot({ fullPage, quality, type: 'webp' });
             }
             catch (err) {
                 config_1.logger.error('Failed to get screenshot:', err);
+                throw err;
             }
         }
         // get html
         if (includeHtml) {
-            if (formatPageContent) {
-                html = yield formatPageContent({ page, url });
+            try {
+                html = yield page.evaluate(() => {
+                    // add meta tag to record crawler
+                    const meta = document.createElement('meta');
+                    meta.name = 'arcblock-crawler';
+                    meta.content = 'true';
+                    document.head.appendChild(meta);
+                    return document.documentElement.outerHTML;
+                });
             }
-            else {
-                html = yield page.content();
+            catch (err) {
+                config_1.logger.error('Failed to get html:', err);
+                throw err;
             }
         }
     }
@@ -213,11 +219,16 @@ const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url,
     };
 });
 exports.getPageContent = getPageContent;
-function createCrawlJob(params, callback) {
+/**
+ * crawl url and return job id
+ * @param params
+ * @param callback callback when job finished
+ */
+function crawlUrl(params, callback) {
     return __awaiter(this, void 0, void 0, function* () {
         params = Object.assign(Object.assign({}, params), { url: (0, utils_1.formatUrl)(params.url) });
         // skip duplicate job
-        const existsJob = yield getJob({
+        const { job: duplicateJob } = (yield job_1.Job.findJob({
             url: params.url,
             includeScreenshot: params.includeScreenshot,
             includeHtml: params.includeHtml,
@@ -225,18 +236,18 @@ function createCrawlJob(params, callback) {
             width: params.width,
             height: params.height,
             fullPage: params.fullPage,
-        });
-        config_1.logger.info('create crawl job', params);
-        if (existsJob) {
+        })) || {};
+        if (duplicateJob) {
             config_1.logger.warn(`Crawl job already exists for ${params.url}, skip`);
-            return existsJob.id;
+            return duplicateJob.id;
         }
+        config_1.logger.info('create crawl job', params);
         const jobId = (0, crypto_1.randomUUID)();
         const job = crawlQueue.push(Object.assign(Object.assign({}, params), { id: jobId }));
-        job.on('finished', ({ result }) => {
+        job.on('finished', (_a) => __awaiter(this, [_a], void 0, function* ({ result }) {
             config_1.logger.info(`Crawl completed ${params.url}, status: ${result ? 'success' : 'failed'}`, { job: params, result });
-            callback === null || callback === void 0 ? void 0 : callback(result);
-        });
+            callback === null || callback === void 0 ? void 0 : callback(result ? yield (0, snapshot_1.formatSnapshot)(result) : null);
+        }));
         job.on('failed', ({ error }) => {
             config_1.logger.error(`Failed to execute job for ${params.url}`, { error, job: params });
             callback === null || callback === void 0 ? void 0 : callback(null);
@@ -244,71 +255,3 @@ function createCrawlJob(params, callback) {
         return jobId;
     });
 }
-// @ts-ignore
-function getJob(condition) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const where = Object.keys(condition)
-            .filter((key) => condition[key] !== undefined)
-            .map((key) => {
-            return core_1.default.where(core_1.default.fn('json_extract', core_1.default.col('job'), `$.${key}`), condition[key]);
-        });
-        const job = yield crawlQueue.store.db.findOne({
-            where: {
-                [core_1.default.Op.and]: where,
-            },
-        });
-        if (job) {
-            return job.job;
-        }
-        return null;
-    });
-}
-function convertJobToSnapshot({ job, snapshot }) {
-    return Object.assign({ 
-        // @ts-ignore
-        jobId: job.jobId || job.id, url: job.url, options: {
-            width: job.width,
-            height: job.height,
-            includeScreenshot: job.includeScreenshot,
-            includeHtml: job.includeHtml,
-            quality: job.quality,
-            fullPage: job.fullPage,
-        } }, snapshot);
-}
-function formatSnapshot(snapshot, columns) {
-    return __awaiter(this, void 0, void 0, function* () {
-        let data = Object.assign({}, snapshot);
-        // format screenshot path to full url
-        if (data.screenshot) {
-            data.screenshot = (0, ufo_1.joinURL)(config_1.config.appUrl, data.screenshot);
-        }
-        // format html path to string
-        if (data.html) {
-            const html = yield fs_extra_1.default.readFile(path_1.default.join(config_1.config.dataDir, data.html));
-            data.html = html.toString();
-        }
-        if (columns === null || columns === void 0 ? void 0 : columns.length) {
-            data = (0, pick_1.default)(data, columns);
-        }
-        return data;
-    });
-}
-/**
- * get snapshot from db or crawl queue
- */
-function getSnapshot(jobId) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const snapshotModel = yield snapshot_1.Snapshot.findByPk(jobId);
-        if (snapshotModel) {
-            return snapshotModel.toJSON();
-        }
-        const job = yield getJob({ id: jobId });
-        if (job) {
-            return {
-                jobId,
-                status: 'pending',
-            };
-        }
-        return null;
-    });
-}

package/lib/cjs/cron.d.ts

@@ -0,0 +1 @@
+export declare function initCron(): any;

package/lib/cjs/cron.js

@@ -0,0 +1,49 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initCron = initCron;
+const cron_1 = __importDefault(require("@abtnode/cron"));
+const config_1 = require("./config");
+const site_1 = require("./site");
+let cron = null;
+function initCron() {
+    if (cron)
+        return;
+    config_1.logger.info('Init cron', { config: config_1.config.siteCron });
+    cron = cron_1.default.init({
+        context: {},
+        jobs: [
+            {
+                name: 'crawl-site',
+                time: config_1.config.siteCron.time,
+                options: { runOnInit: config_1.config.siteCron.runOnInit },
+                fn: () => __awaiter(this, void 0, void 0, function* () {
+                    config_1.logger.info('Start cron to crawl site', { sites: config_1.config.siteCron.sites });
+                    for (const site of config_1.config.siteCron.sites) {
+                        try {
+                            yield (0, site_1.crawlSite)(site);
+                        }
+                        catch (err) {
+                            config_1.logger.error('Cron task error', { err, site });
+                        }
+                    }
+                }),
+            },
+        ],
+        onError: (err) => {
+            config_1.logger.error('Cron error', err);
+        },
+    });
+    return cron;
+}

package/lib/cjs/index.d.ts

@@ -0,0 +1,9 @@
+import { Config } from './config';
+export * from './crawler';
+export * from './site';
+export * from './services/snapshot';
+export * as utils from './utils';
+type DeepPartial<T> = T extends object ? {
+    [P in keyof T]?: DeepPartial<T[P]>;
+} : T;
+export declare function initCrawler(params: DeepPartial<Pick<Config, 'puppeteerPath' | 'siteCron'>>): Promise<void>;

package/lib/cjs/index.js

@@ -0,0 +1,78 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.utils = void 0;
+exports.initCrawler = initCrawler;
+const merge_1 = __importDefault(require("lodash/merge"));
+const config_1 = require("./config");
+const crawler_1 = require("./crawler");
+const cron_1 = require("./cron");
+const puppeteer_1 = require("./puppeteer");
+const store_1 = require("./store");
+__exportStar(require("./crawler"), exports);
+__exportStar(require("./site"), exports);
+__exportStar(require("./services/snapshot"), exports);
+exports.utils = __importStar(require("./utils"));
+function initCrawler(params) {
+    return __awaiter(this, void 0, void 0, function* () {
+        config_1.logger.info('Init crawler', { params });
+        (0, merge_1.default)(config_1.config, params);
+        try {
+            yield (0, store_1.initDatabase)();
+            yield (0, puppeteer_1.ensureBrowser)();
+            yield (0, crawler_1.createCrawlQueue)();
+            yield (0, cron_1.initCron)();
+        }
+        catch (err) {
+            config_1.logger.error('Init crawler error', { err });
+            throw err;
+        }
+    });
+}

package/{esm → lib/cjs}/puppeteer.d.ts

@@ -1,4 +1,4 @@
-import puppeteer, { Browser, Page } from '@blocklet/puppeteer';
+import puppeteer, { Browser, Page, ResourceType } from '@blocklet/puppeteer';
 export { puppeteer };
 export declare function ensurePuppeteerrc(): Promise<{
     cacheDirectory: string;
@@ -12,5 +12,5 @@ export declare const closeBrowser: ({ trimCache }?: {
     trimCache?: boolean;
 }) => Promise<void>;
 export declare function initPage({ abortResourceTypes }?: {
-    abortResourceTypes?: never[] | undefined;
+    abortResourceTypes?: ResourceType[];
 }): Promise<Page>;