@arcblock/crawler 1.0.4 → 1.0.6

package/dist/crawler.js

@@ -40,7 +40,7 @@ function createCrawlQueue() {
         store: new sequelize_1.default(db, 'crawler'),
         concurrency: 1,
         onJob: (job) => __awaiter(this, void 0, void 0, function* () {
-            config_1.logger.debug('job start:', job);
+            config_1.logger.info('Starting to execute crawl job', job);
             const canCrawl = yield (0, utils_1.isAcceptCrawler)(job.url);
             if (!canCrawl) {
                 config_1.logger.error(`failed to crawl ${job.url}, denied by robots.txt`, job);
@@ -157,11 +157,11 @@ function formatHtml(htmlString) {
     }
     return htmlString;
 }
-const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, formatPageContent, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 60 * 1000, fullPage = false, }) {
+const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, formatPageContent, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, fullPage = false, }) {
     config_1.logger.debug('getPageContent', { url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage });
     const page = yield (0, puppeteer_1.initPage)();
     if (width && height) {
-        yield page.setViewport({ width, height });
+        yield page.setViewport({ width, height, deviceScaleFactor: 2 });
     }
     let html = null;
     let screenshot = null;
@@ -176,12 +176,24 @@ const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url,
             throw new Error(`Request failed with status ${statusCode}, in ${url}`);
         }
         // await for networkidle0
-        // https://pptr.dev/api/puppeteer.page.goforward/#remarks
+        // https://pptr.dev/api/puppeteer.page.waitfornetworkidle
         yield page.waitForNetworkIdle({
-            idleTime: 2 * 1000,
+            idleTime: 1.5 * 1000,
         });
         // get screenshot
         if (includeScreenshot) {
+            // Try to find the tallest element and set the browser to the same height
+            if (fullPage) {
+                const maxScrollHeight = yield (0, utils_1.findMaxScrollHeight)(page);
+                config_1.logger.info('findMaxScrollHeight', { maxScrollHeight });
+                if (maxScrollHeight) {
+                    yield page.setViewport({ width, height: maxScrollHeight || height, deviceScaleFactor: 2 });
+                    yield page.evaluate((scrollHeight) => {
+                        window.scrollTo(0, scrollHeight || 0);
+                        document.documentElement.scrollTo(0, scrollHeight || 0);
+                    }, maxScrollHeight);
+                }
+            }
             try {
                 screenshot = yield page.screenshot({ fullPage, quality, type: 'webp' });
             }
@@ -215,7 +227,7 @@ const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url,
 exports.getPageContent = getPageContent;
 function createCrawlJob(params, callback) {
     return __awaiter(this, void 0, void 0, function* () {
-        params = Object.assign(Object.assign({}, params), { url: (0, utils_1.formatUrl)(params.url) });
+        params = Object.assign(Object.assign({}, params), { id: (0, crypto_1.randomUUID)(), url: (0, utils_1.formatUrl)(params.url) });
         // skip duplicate job
         const existsJob = yield getJob({
             url: params.url,
@@ -226,22 +238,21 @@ function createCrawlJob(params, callback) {
             height: params.height,
             fullPage: params.fullPage,
         });
-        config_1.logger.info('create crawl job', params);
         if (existsJob) {
             config_1.logger.warn(`Crawl job already exists for ${params.url}, skip`);
             return existsJob.id;
         }
-        const jobId = (0, crypto_1.randomUUID)();
-        const job = crawlQueue.push(Object.assign(Object.assign({}, params), { id: jobId }));
+        config_1.logger.info('create crawl job', params);
+        const job = crawlQueue.push(params);
         job.on('finished', ({ result }) => {
-            config_1.logger.info(`Crawl completed ${params.url}, status: ${result ? 'success' : 'failed'}`, { job: params, result });
+            config_1.logger.info(`Crawl completed ${params.url}`, { job: params, result });
             callback === null || callback === void 0 ? void 0 : callback(result);
         });
         job.on('failed', ({ error }) => {
             config_1.logger.error(`Failed to execute job for ${params.url}`, { error, job: params });
             callback === null || callback === void 0 ? void 0 : callback(null);
         });
-        return jobId;
+        return params.id;
     });
 }
 // @ts-ignore

package/dist/puppeteer.js

@@ -94,7 +94,7 @@ function ensureBrowser() {
     return __awaiter(this, void 0, void 0, function* () {
         const puppeteerConfig = yield ensurePuppeteerrc();
         const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium';
-        config_2.logger.info('executablePath', executablePath);
+        config_2.logger.debug('Chromium executablePath', executablePath);
         if (!fs_extra_1.default.existsSync(executablePath)) {
             config_2.logger.info('start download browser', puppeteerConfig);
             const { downloadBrowser } = yield (() => __awaiter(this, void 0, void 0, function* () {
@@ -182,7 +182,7 @@ function launchBrowser() {
                     '--font-render-hinting=none',
                 ],
             });
-            config_2.logger.info('Launch browser success');
+            config_2.logger.info('Launch browser');
         }
         catch (error) {
             config_2.logger.error('launch browser failed: ', error);
@@ -260,20 +260,20 @@ const closeBrowser = (...args_1) => __awaiter(void 0, [...args_1], void 0, funct
         yield Promise.all(pages.map((page) => page.close()));
     }
     catch (err) {
-        config_2.logger.error('Failed to close all pages:', err);
+        config_2.logger.warn('Failed to close all pages:', err);
     }
     // close browser
     try {
         yield browser.close();
     }
     catch (err) {
-        config_2.logger.error('Failed to close browser:', err);
+        config_2.logger.warn('Failed to close browser:', err);
     }
     // clear cache
     try {
         if (trimCache) {
             yield puppeteer_1.default.trimCache();
-            config_2.logger.info('Trim cache success');
+            config_2.logger.debug('Trim cache success');
         }
         // try to clear temporary directory
         // if (puppeteerConfig) {
@@ -284,7 +284,7 @@ const closeBrowser = (...args_1) => __awaiter(void 0, [...args_1], void 0, funct
         }
     }
     catch (err) {
-        config_2.logger.error('Failed to clear browser cache:', err);
+        config_2.logger.warn('Failed to clear browser cache:', err);
     }
     browser = null;
     clearBrowserActivatedTimer();

package/dist/utils.d.ts

@@ -1,3 +1,4 @@
+import { Page } from '@blocklet/puppeteer';
 export declare const api: import("axios").AxiosInstance;
 export declare const sleep: (ms: number) => Promise<unknown>;
 export declare const CRAWLER_FLAG = "x-crawler";
@@ -13,3 +14,4 @@ export declare const getFullUrl: (req: any) => string;
 export declare const getRelativePath: (url: string) => string;
 export declare const formatUrl: (url: string) => string;
 export declare function md5(content: string | Uint8Array): string;
+export declare function findMaxScrollHeight(page: Page): Promise<number>;

package/dist/utils.js

@@ -15,6 +15,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.formatUrl = exports.getRelativePath = exports.getFullUrl = exports.getComponentInfo = exports.isBotUserAgent = exports.getSitemapList = exports.isAcceptCrawler = exports.getDefaultSitemapUrl = exports.getDefaultRobotsUrl = exports.isSelfCrawler = exports.CRAWLER_FLAG = exports.sleep = exports.api = void 0;
 exports.getRobots = getRobots;
 exports.md5 = md5;
+exports.findMaxScrollHeight = findMaxScrollHeight;
 const config_1 = require("@blocklet/sdk/lib/config");
 const axios_1 = __importDefault(require("axios"));
 const flattenDeep_1 = __importDefault(require("lodash/flattenDeep"));
@@ -237,3 +238,23 @@ exports.formatUrl = formatUrl;
 function md5(content) {
     return (0, node_crypto_1.createHash)('md5').update(content).digest('hex');
 }
+function findMaxScrollHeight(page) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const maxHeightHandler = yield page.evaluateHandle(() => {
+            const elements = Array.from(document.querySelectorAll('*'));
+            let maxHeight = document.body.scrollHeight;
+            for (const el of elements) {
+                const style = window.getComputedStyle(el);
+                if (style.overflowY === 'auto' || style.overflowY === 'scroll') {
+                    if (el.scrollHeight > el.clientHeight && el.scrollHeight > maxHeight) {
+                        maxHeight = el.scrollHeight;
+                    }
+                }
+            }
+            return maxHeight;
+        });
+        const maxHeight = yield maxHeightHandler.jsonValue();
+        maxHeightHandler.dispose();
+        return maxHeight;
+    });
+}

package/esm/crawler.js

@@ -19,7 +19,7 @@ import { config, logger } from './config';
 import { Job } from './db/job';
 import { Snapshot } from './db/snapshot';
 import { initPage } from './puppeteer';
-import { formatUrl, isAcceptCrawler, md5 } from './utils';
+import { findMaxScrollHeight, formatUrl, isAcceptCrawler, md5 } from './utils';
 const { BaseState } = require('@abtnode/models');
 let crawlQueue;
 export function createCrawlQueue() {
@@ -28,7 +28,7 @@ export function createCrawlQueue() {
         store: new SequelizeStore(db, 'crawler'),
         concurrency: 1,
         onJob: (job) => __awaiter(this, void 0, void 0, function* () {
-            logger.debug('job start:', job);
+            logger.info('Starting to execute crawl job', job);
             const canCrawl = yield isAcceptCrawler(job.url);
             if (!canCrawl) {
                 logger.error(`failed to crawl ${job.url}, denied by robots.txt`, job);
@@ -145,11 +145,11 @@ function formatHtml(htmlString) {
     }
     return htmlString;
 }
-export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, formatPageContent, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 60 * 1000, fullPage = false, }) {
+export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, formatPageContent, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, fullPage = false, }) {
     logger.debug('getPageContent', { url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage });
     const page = yield initPage();
     if (width && height) {
-        yield page.setViewport({ width, height });
+        yield page.setViewport({ width, height, deviceScaleFactor: 2 });
     }
     let html = null;
     let screenshot = null;
@@ -164,12 +164,24 @@ export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function*
             throw new Error(`Request failed with status ${statusCode}, in ${url}`);
         }
         // await for networkidle0
-        // https://pptr.dev/api/puppeteer.page.goforward/#remarks
+        // https://pptr.dev/api/puppeteer.page.waitfornetworkidle
         yield page.waitForNetworkIdle({
-            idleTime: 2 * 1000,
+            idleTime: 1.5 * 1000,
         });
         // get screenshot
         if (includeScreenshot) {
+            // Try to find the tallest element and set the browser to the same height
+            if (fullPage) {
+                const maxScrollHeight = yield findMaxScrollHeight(page);
+                logger.info('findMaxScrollHeight', { maxScrollHeight });
+                if (maxScrollHeight) {
+                    yield page.setViewport({ width, height: maxScrollHeight || height, deviceScaleFactor: 2 });
+                    yield page.evaluate((scrollHeight) => {
+                        window.scrollTo(0, scrollHeight || 0);
+                        document.documentElement.scrollTo(0, scrollHeight || 0);
+                    }, maxScrollHeight);
+                }
+            }
             try {
                 screenshot = yield page.screenshot({ fullPage, quality, type: 'webp' });
             }
@@ -202,7 +214,7 @@ export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function*
 });
 export function createCrawlJob(params, callback) {
     return __awaiter(this, void 0, void 0, function* () {
-        params = Object.assign(Object.assign({}, params), { url: formatUrl(params.url) });
+        params = Object.assign(Object.assign({}, params), { id: randomUUID(), url: formatUrl(params.url) });
         // skip duplicate job
         const existsJob = yield getJob({
             url: params.url,
@@ -213,22 +225,21 @@ export function createCrawlJob(params, callback) {
             height: params.height,
             fullPage: params.fullPage,
         });
-        logger.info('create crawl job', params);
         if (existsJob) {
             logger.warn(`Crawl job already exists for ${params.url}, skip`);
             return existsJob.id;
         }
-        const jobId = randomUUID();
-        const job = crawlQueue.push(Object.assign(Object.assign({}, params), { id: jobId }));
+        logger.info('create crawl job', params);
+        const job = crawlQueue.push(params);
         job.on('finished', ({ result }) => {
-            logger.info(`Crawl completed ${params.url}, status: ${result ? 'success' : 'failed'}`, { job: params, result });
+            logger.info(`Crawl completed ${params.url}`, { job: params, result });
             callback === null || callback === void 0 ? void 0 : callback(result);
         });
         job.on('failed', ({ error }) => {
             logger.error(`Failed to execute job for ${params.url}`, { error, job: params });
             callback === null || callback === void 0 ? void 0 : callback(null);
         });
-        return jobId;
+        return params.id;
     });
 }
 // @ts-ignore

package/esm/puppeteer.js

@@ -50,7 +50,7 @@ export function ensureBrowser() {
     return __awaiter(this, void 0, void 0, function* () {
         const puppeteerConfig = yield ensurePuppeteerrc();
         const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium';
-        logger.info('executablePath', executablePath);
+        logger.debug('Chromium executablePath', executablePath);
         if (!fs.existsSync(executablePath)) {
             logger.info('start download browser', puppeteerConfig);
             const { downloadBrowser } = yield (() => __awaiter(this, void 0, void 0, function* () {
@@ -138,7 +138,7 @@ export function launchBrowser() {
                     '--font-render-hinting=none',
                 ],
             });
-            logger.info('Launch browser success');
+            logger.info('Launch browser');
         }
         catch (error) {
             logger.error('launch browser failed: ', error);
@@ -215,20 +215,20 @@ export const closeBrowser = (...args_1) => __awaiter(void 0, [...args_1], void 0
         yield Promise.all(pages.map((page) => page.close()));
     }
     catch (err) {
-        logger.error('Failed to close all pages:', err);
+        logger.warn('Failed to close all pages:', err);
     }
     // close browser
     try {
         yield browser.close();
     }
     catch (err) {
-        logger.error('Failed to close browser:', err);
+        logger.warn('Failed to close browser:', err);
     }
     // clear cache
     try {
         if (trimCache) {
             yield puppeteer.trimCache();
-            logger.info('Trim cache success');
+            logger.debug('Trim cache success');
         }
         // try to clear temporary directory
         // if (puppeteerConfig) {
@@ -239,7 +239,7 @@ export const closeBrowser = (...args_1) => __awaiter(void 0, [...args_1], void 0
         }
     }
     catch (err) {
-        logger.error('Failed to clear browser cache:', err);
+        logger.warn('Failed to clear browser cache:', err);
     }
     browser = null;
     clearBrowserActivatedTimer();

package/esm/utils.d.ts

@@ -1,3 +1,4 @@
+import { Page } from '@blocklet/puppeteer';
 export declare const api: import("axios").AxiosInstance;
 export declare const sleep: (ms: number) => Promise<unknown>;
 export declare const CRAWLER_FLAG = "x-crawler";
@@ -13,3 +14,4 @@ export declare const getFullUrl: (req: any) => string;
 export declare const getRelativePath: (url: string) => string;
 export declare const formatUrl: (url: string) => string;
 export declare function md5(content: string | Uint8Array): string;
+export declare function findMaxScrollHeight(page: Page): Promise<number>;

package/esm/utils.js

@@ -218,3 +218,23 @@ export const formatUrl = (url) => {
 export function md5(content) {
     return createHash('md5').update(content).digest('hex');
 }
+export function findMaxScrollHeight(page) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const maxHeightHandler = yield page.evaluateHandle(() => {
+            const elements = Array.from(document.querySelectorAll('*'));
+            let maxHeight = document.body.scrollHeight;
+            for (const el of elements) {
+                const style = window.getComputedStyle(el);
+                if (style.overflowY === 'auto' || style.overflowY === 'scroll') {
+                    if (el.scrollHeight > el.clientHeight && el.scrollHeight > maxHeight) {
+                        maxHeight = el.scrollHeight;
+                    }
+                }
+            }
+            return maxHeight;
+        });
+        const maxHeight = yield maxHeightHandler.jsonValue();
+        maxHeightHandler.dispose();
+        return maxHeight;
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arcblock/crawler",
-  "version": "1.0.4",
+  "version": "1.0.6",
   "main": "dist/index.js",
   "module": "esm/index.js",
   "types": "dist/index.d.ts",
@@ -113,6 +113,7 @@
     "bundle": "npm run build",
     "build:cjs": "tsc -p tsconfig.cjs.json",
     "build:esm": "tsc -p tsconfig.esm.json",
-    "build": "npm run build:cjs && npm run build:esm"
+    "build": "npm run build:cjs && npm run build:esm",
+    "fix:sqlite": "cd node_modules/sqlite3 && npm run rebuild"
   }
 }