@arcblock/crawler 1.1.3 → 1.1.5

package/lib/cjs/crawler.d.ts

@@ -5,17 +5,8 @@ export declare function getDataDir(): Promise<{
     htmlDir: string;
     screenshotDir: string;
 }>;
-export declare const getPageContent: ({ url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, }: {
-    url: string;
-    includeScreenshot?: boolean;
-    includeHtml?: boolean;
-    width?: number;
-    height?: number;
-    quality?: number;
-    timeout?: number;
-    fullPage?: boolean;
-}) => Promise<{
-    html: string;
+export declare const getPageContent: ({ url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, headers, cookies, localStorage, }: JobState) => Promise<{
+    html: string | null;
     screenshot: Uint8Array<ArrayBufferLike> | null;
     meta: {
         title?: string;
@@ -27,4 +18,4 @@ export declare const getPageContent: ({ url, includeScreenshot, includeHtml, wid
  * @param params
  * @param callback callback when job finished
  */
-export declare function crawlUrl(params: Omit<JobState, 'jobId'>, callback?: (snapshot: SnapshotModel | null) => void): Promise<string | undefined>;
+export declare function crawlUrl(params: Omit<JobState, 'jobId'>, callback?: (snapshot: SnapshotModel | null) => void): Promise<string>;

package/lib/cjs/crawler.js

@@ -59,7 +59,12 @@ function createCrawlQueue() {
             // }
             try {
                 // get page content later
-                const result = yield (0, exports.getPageContent)(job);
+                const result = yield (0, exports.getPageContent)(Object.assign({ localStorage: {
+                        // for blocklet theme
+                        blocklet_theme_prefer: 'light',
+                        // for blocklet domain warning
+                        'domain-warning-skip': Date.now().toString(),
+                    } }, job));
                 if (!result || (!result.html && !result.screenshot)) {
                     config_1.logger.error(`failed to crawl ${job.url}, empty content`, job);
                     const snapshot = (0, snapshot_1.convertJobToSnapshot)({
@@ -137,18 +142,24 @@ function saveSnapshotToLocal(_a) {
         };
     });
 }
-function formatHtml(htmlString) {
-    if (htmlString.includes('<h2>Unexpected Application Error!</h2>')) {
-        return '';
-    }
-    return htmlString;
-}
-const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, fullPage = false, }) {
-    config_1.logger.debug('getPageContent', { url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage });
+const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, fullPage = false, headers, cookies = [], localStorage, }) {
     const page = yield (0, puppeteer_1.initPage)();
     if (width && height) {
         yield page.setViewport({ width, height, deviceScaleFactor: 2 });
     }
+    if (headers) {
+        yield page.setExtraHTTPHeaders(headers);
+    }
+    if (cookies === null || cookies === void 0 ? void 0 : cookies.length) {
+        yield page.setCookie(...cookies);
+    }
+    if (localStorage) {
+        yield page.evaluateOnNewDocument((items) => {
+            Object.entries(items).forEach(([key, value]) => {
+                window.localStorage.setItem(key, value);
+            });
+        }, localStorage);
+    }
     let html = null;
     let screenshot = null;
     const meta = {};
@@ -158,7 +169,6 @@ const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url,
             throw new Error(`Failed to load page: response is null for ${url}`);
         }
         const statusCode = response.status();
-        config_1.logger.debug('getPageContent.response', { response, statusCode });
         if (![200, 304].includes(statusCode)) {
             throw new Error(`Request failed with status ${statusCode}, in ${url}`);
         }
@@ -207,6 +217,11 @@ const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url,
                     description,
                 };
             });
+            // check if the page is an error page
+            const isErrorPage = ['<h2>Unexpected Application Error!</h2>', 'Current route occurred an error'].some((errorHtml) => data.html.includes(errorHtml));
+            if (isErrorPage) {
+                throw new Error('Page is an error page');
+            }
             meta.title = data.title;
             meta.description = data.description;
             if (includeHtml) {
@@ -225,7 +240,6 @@ const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url,
     finally {
         yield page.close();
     }
-    html = formatHtml(html || '');
     return {
         html,
         screenshot,
@@ -238,24 +252,17 @@ exports.getPageContent = getPageContent;
  * @param params
  * @param callback callback when job finished
  */
+// eslint-disable-next-line require-await
 function crawlUrl(params, callback) {
     return __awaiter(this, void 0, void 0, function* () {
         params = Object.assign(Object.assign({}, params), { url: (0, utils_1.formatUrl)(params.url) });
         // skip duplicate job
-        const { job: duplicateJob } = (yield job_1.Job.findJob({
-            url: params.url,
-            includeScreenshot: params.includeScreenshot,
-            includeHtml: params.includeHtml,
-            quality: params.quality,
-            width: params.width,
-            height: params.height,
-            fullPage: params.fullPage,
-        })) || {};
-        if (duplicateJob) {
+        const existsJob = yield job_1.Job.isExists(params);
+        if (existsJob) {
             config_1.logger.info(`Crawl job already exists for ${params.url}, skip`);
-            return duplicateJob.id;
+            return existsJob.id;
         }
-        config_1.logger.info('create crawl job', params);
+        config_1.logger.info('enqueue crawl job', params);
         const jobId = (0, crypto_1.randomUUID)();
         const job = crawlQueue.push(Object.assign(Object.assign({}, params), { id: jobId }));
         job.on('finished', (_a) => __awaiter(this, [_a], void 0, function* ({ result }) {

package/lib/cjs/site.d.ts

@@ -1,2 +1,2 @@
 import { Site } from './config';
-export declare const crawlSite: ({ url, pathname, interval }: Site) => Promise<(string | null | undefined)[]>;
+export declare const crawlSite: ({ url, pathname, interval }: Site) => Promise<(string | null)[]>;

package/lib/cjs/site.js

@@ -42,6 +42,7 @@ const crawlSite = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, path
     });
     config_1.logger.info(`Found ${sitemapItems.length} sitemap items which match ${pathname} from ${url}`);
     let processCount = 0;
+    let crawlCount = 0;
     crawlBlockletRunningMap.set(key, true);
     try {
         const jobIds = yield (0, p_map_1.default)(sitemapItems, (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, sitemapItem }) {
@@ -58,7 +59,13 @@ const crawlSite = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, path
                     return null;
                 }
             }
-            config_1.logger.debug(`Sitemap process ${processCount} / ${sitemapItems.length}`);
+            config_1.logger.debug(`Sitemap process ${processCount} / ${sitemapItems.length}`, {
+                snapshotExists: !!snapshot,
+                lastModified: snapshot === null || snapshot === void 0 ? void 0 : snapshot.lastModified,
+                sitemapLastmod: sitemapItem.lastmod,
+                url,
+            });
+            crawlCount++;
             return (0, crawler_1.crawlUrl)({
                 url,
                 lastModified: sitemapItem.lastmod,
@@ -66,6 +73,12 @@ const crawlSite = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, path
                 includeHtml: true,
             });
         }), { concurrency: config_1.config.siteCron.sitemapConcurrency });
+        config_1.logger.info('Enqueued jobs from sitemap finished', {
+            url,
+            pathname,
+            processCount,
+            crawlCount,
+        });
         return jobIds;
     }
     catch (error) {

package/lib/cjs/store/job.d.ts

@@ -1,3 +1,4 @@
+import { CookieParam } from '@blocklet/puppeteer';
 import { Model, Sequelize } from '@sequelize/core';
 export interface JobState {
     id?: string;
@@ -11,6 +12,9 @@ export interface JobState {
     timeout?: number;
     fullPage?: boolean;
     lastModified?: string;
+    headers?: Record<string, string>;
+    cookies?: CookieParam[];
+    localStorage?: Record<string, string>;
 }
 export interface JobModel {
     id: string;
@@ -31,4 +35,7 @@ export declare class Job extends Model<JobModel> implements JobModel {
     cancelled: JobModel['cancelled'];
     static initModel(sequelize: Sequelize): typeof Job;
     static findJob(condition: Partial<JobState>): Promise<JobModel | null>;
+    static isExists(condition: Partial<JobState> & {
+        url: string;
+    }): Promise<JobModel | null | undefined>;
 }

package/lib/cjs/store/job.js

@@ -41,9 +41,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Job = void 0;
 const core_1 = __importStar(require("@sequelize/core"));
+const isEqual_1 = __importDefault(require("lodash/isEqual"));
 class Job extends core_1.Model {
     static initModel(sequelize) {
         return Job.init({
@@ -106,5 +110,22 @@ class Job extends core_1.Model {
             return (job === null || job === void 0 ? void 0 : job.toJSON()) || null;
         });
     }
+    static isExists(condition) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const jobs = yield Job.findAll({
+                where: core_1.default.where(core_1.default.fn('json_extract', core_1.default.col('job'), '$.url'), condition.url),
+            });
+            if (!(jobs === null || jobs === void 0 ? void 0 : jobs.length)) {
+                return null;
+            }
+            const existsJob = jobs.find((job) => {
+                const jobModel = job.get('job');
+                return Object.keys(condition).every((key) => {
+                    return (0, isEqual_1.default)(condition[key], jobModel[key]);
+                });
+            });
+            return existsJob === null || existsJob === void 0 ? void 0 : existsJob.get();
+        });
+    }
 }
 exports.Job = Job;

package/lib/cjs/store/snapshot.d.ts

@@ -18,6 +18,7 @@ export interface SnapshotModel {
         includeHtml?: boolean;
         quality?: number;
         fullPage?: boolean;
+        headers?: Record<string, string>;
     };
 }
 export declare class Snapshot extends Model<SnapshotModel> implements SnapshotModel {

package/lib/cjs/utils.js

@@ -134,6 +134,11 @@ const staticFileExtensions = [
     'xls',
     'xml',
     'zip',
+    'ts',
+    'json',
+    'md',
+    'yml',
+    'yaml',
 ];
 const sleep = (ms) => {
     return new Promise((resolve) => {
@@ -153,7 +158,7 @@ exports.isSelfCrawler = isSelfCrawler;
  * Check if the request is a static file
  */
 function isStaticFile(req) {
-    const excludeUrlPattern = new RegExp(`\\.(${staticFileExtensions.join('|')})$`, 'i');
+    const excludeUrlPattern = new RegExp(`\\.(${staticFileExtensions.join('|')})([\\?#]|$)`, 'i');
     return excludeUrlPattern.test(req.path);
 }
 /**

package/lib/esm/crawler.d.ts

@@ -5,17 +5,8 @@ export declare function getDataDir(): Promise<{
     htmlDir: string;
     screenshotDir: string;
 }>;
-export declare const getPageContent: ({ url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, }: {
-    url: string;
-    includeScreenshot?: boolean;
-    includeHtml?: boolean;
-    width?: number;
-    height?: number;
-    quality?: number;
-    timeout?: number;
-    fullPage?: boolean;
-}) => Promise<{
-    html: string;
+export declare const getPageContent: ({ url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, headers, cookies, localStorage, }: JobState) => Promise<{
+    html: string | null;
     screenshot: Uint8Array<ArrayBufferLike> | null;
     meta: {
         title?: string;
@@ -27,4 +18,4 @@ export declare const getPageContent: ({ url, includeScreenshot, includeHtml, wid
  * @param params
  * @param callback callback when job finished
  */
-export declare function crawlUrl(params: Omit<JobState, 'jobId'>, callback?: (snapshot: SnapshotModel | null) => void): Promise<string | undefined>;
+export declare function crawlUrl(params: Omit<JobState, 'jobId'>, callback?: (snapshot: SnapshotModel | null) => void): Promise<string>;

package/lib/esm/crawler.js

@@ -50,7 +50,12 @@ export function createCrawlQueue() {
             // }
             try {
                 // get page content later
-                const result = yield getPageContent(job);
+                const result = yield getPageContent(Object.assign({ localStorage: {
+                        // for blocklet theme
+                        blocklet_theme_prefer: 'light',
+                        // for blocklet domain warning
+                        'domain-warning-skip': Date.now().toString(),
+                    } }, job));
                 if (!result || (!result.html && !result.screenshot)) {
                     logger.error(`failed to crawl ${job.url}, empty content`, job);
                     const snapshot = convertJobToSnapshot({
@@ -128,18 +133,24 @@ function saveSnapshotToLocal(_a) {
         };
     });
 }
-function formatHtml(htmlString) {
-    if (htmlString.includes('<h2>Unexpected Application Error!</h2>')) {
-        return '';
-    }
-    return htmlString;
-}
-export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, fullPage = false, }) {
-    logger.debug('getPageContent', { url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage });
+export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, fullPage = false, headers, cookies = [], localStorage, }) {
     const page = yield initPage();
     if (width && height) {
         yield page.setViewport({ width, height, deviceScaleFactor: 2 });
     }
+    if (headers) {
+        yield page.setExtraHTTPHeaders(headers);
+    }
+    if (cookies === null || cookies === void 0 ? void 0 : cookies.length) {
+        yield page.setCookie(...cookies);
+    }
+    if (localStorage) {
+        yield page.evaluateOnNewDocument((items) => {
+            Object.entries(items).forEach(([key, value]) => {
+                window.localStorage.setItem(key, value);
+            });
+        }, localStorage);
+    }
     let html = null;
     let screenshot = null;
     const meta = {};
@@ -149,7 +160,6 @@ export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function*
             throw new Error(`Failed to load page: response is null for ${url}`);
         }
         const statusCode = response.status();
-        logger.debug('getPageContent.response', { response, statusCode });
         if (![200, 304].includes(statusCode)) {
             throw new Error(`Request failed with status ${statusCode}, in ${url}`);
         }
@@ -198,6 +208,11 @@ export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function*
                     description,
                 };
             });
+            // check if the page is an error page
+            const isErrorPage = ['<h2>Unexpected Application Error!</h2>', 'Current route occurred an error'].some((errorHtml) => data.html.includes(errorHtml));
+            if (isErrorPage) {
+                throw new Error('Page is an error page');
+            }
             meta.title = data.title;
             meta.description = data.description;
             if (includeHtml) {
@@ -216,7 +231,6 @@ export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function*
     finally {
         yield page.close();
     }
-    html = formatHtml(html || '');
     return {
         html,
         screenshot,
@@ -228,24 +242,17 @@ export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function*
  * @param params
  * @param callback callback when job finished
  */
+// eslint-disable-next-line require-await
 export function crawlUrl(params, callback) {
     return __awaiter(this, void 0, void 0, function* () {
         params = Object.assign(Object.assign({}, params), { url: formatUrl(params.url) });
         // skip duplicate job
-        const { job: duplicateJob } = (yield Job.findJob({
-            url: params.url,
-            includeScreenshot: params.includeScreenshot,
-            includeHtml: params.includeHtml,
-            quality: params.quality,
-            width: params.width,
-            height: params.height,
-            fullPage: params.fullPage,
-        })) || {};
-        if (duplicateJob) {
+        const existsJob = yield Job.isExists(params);
+        if (existsJob) {
             logger.info(`Crawl job already exists for ${params.url}, skip`);
-            return duplicateJob.id;
+            return existsJob.id;
         }
-        logger.info('create crawl job', params);
+        logger.info('enqueue crawl job', params);
         const jobId = randomUUID();
         const job = crawlQueue.push(Object.assign(Object.assign({}, params), { id: jobId }));
         job.on('finished', (_a) => __awaiter(this, [_a], void 0, function* ({ result }) {

package/lib/esm/site.d.ts

@@ -1,2 +1,2 @@
 import { Site } from './config';
-export declare const crawlSite: ({ url, pathname, interval }: Site) => Promise<(string | null | undefined)[]>;
+export declare const crawlSite: ({ url, pathname, interval }: Site) => Promise<(string | null)[]>;

package/lib/esm/site.js

@@ -36,6 +36,7 @@ export const crawlSite = (_a) => __awaiter(void 0, [_a], void 0, function* ({ ur
     });
     logger.info(`Found ${sitemapItems.length} sitemap items which match ${pathname} from ${url}`);
     let processCount = 0;
+    let crawlCount = 0;
     crawlBlockletRunningMap.set(key, true);
     try {
         const jobIds = yield pMap(sitemapItems, (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, sitemapItem }) {
@@ -52,7 +53,13 @@ export const crawlSite = (_a) => __awaiter(void 0, [_a], void 0, function* ({ ur
                     return null;
                 }
             }
-            logger.debug(`Sitemap process ${processCount} / ${sitemapItems.length}`);
+            logger.debug(`Sitemap process ${processCount} / ${sitemapItems.length}`, {
+                snapshotExists: !!snapshot,
+                lastModified: snapshot === null || snapshot === void 0 ? void 0 : snapshot.lastModified,
+                sitemapLastmod: sitemapItem.lastmod,
+                url,
+            });
+            crawlCount++;
             return crawlUrl({
                 url,
                 lastModified: sitemapItem.lastmod,
@@ -60,6 +67,12 @@ export const crawlSite = (_a) => __awaiter(void 0, [_a], void 0, function* ({ ur
                 includeHtml: true,
             });
         }), { concurrency: config.siteCron.sitemapConcurrency });
+        logger.info('Enqueued jobs from sitemap finished', {
+            url,
+            pathname,
+            processCount,
+            crawlCount,
+        });
         return jobIds;
     }
     catch (error) {

package/lib/esm/store/job.d.ts

@@ -1,3 +1,4 @@
+import { CookieParam } from '@blocklet/puppeteer';
 import { Model, Sequelize } from '@sequelize/core';
 export interface JobState {
     id?: string;
@@ -11,6 +12,9 @@ export interface JobState {
     timeout?: number;
     fullPage?: boolean;
     lastModified?: string;
+    headers?: Record<string, string>;
+    cookies?: CookieParam[];
+    localStorage?: Record<string, string>;
 }
 export interface JobModel {
     id: string;
@@ -31,4 +35,7 @@ export declare class Job extends Model<JobModel> implements JobModel {
     cancelled: JobModel['cancelled'];
     static initModel(sequelize: Sequelize): typeof Job;
     static findJob(condition: Partial<JobState>): Promise<JobModel | null>;
+    static isExists(condition: Partial<JobState> & {
+        url: string;
+    }): Promise<JobModel | null | undefined>;
 }

package/lib/esm/store/job.js

@@ -8,6 +8,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import sequelize, { DataTypes, Model } from '@sequelize/core';
+import isEqual from 'lodash/isEqual';
 export class Job extends Model {
     static initModel(sequelize) {
         return Job.init({
@@ -70,4 +71,21 @@ export class Job extends Model {
             return (job === null || job === void 0 ? void 0 : job.toJSON()) || null;
         });
     }
+    static isExists(condition) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const jobs = yield Job.findAll({
+                where: sequelize.where(sequelize.fn('json_extract', sequelize.col('job'), '$.url'), condition.url),
+            });
+            if (!(jobs === null || jobs === void 0 ? void 0 : jobs.length)) {
+                return null;
+            }
+            const existsJob = jobs.find((job) => {
+                const jobModel = job.get('job');
+                return Object.keys(condition).every((key) => {
+                    return isEqual(condition[key], jobModel[key]);
+                });
+            });
+            return existsJob === null || existsJob === void 0 ? void 0 : existsJob.get();
+        });
+    }
 }

package/lib/esm/store/snapshot.d.ts

@@ -18,6 +18,7 @@ export interface SnapshotModel {
         includeHtml?: boolean;
         quality?: number;
         fullPage?: boolean;
+        headers?: Record<string, string>;
     };
 }
 export declare class Snapshot extends Model<SnapshotModel> implements SnapshotModel {

package/lib/esm/utils.js

@@ -123,6 +123,11 @@ const staticFileExtensions = [
     'xls',
     'xml',
     'zip',
+    'ts',
+    'json',
+    'md',
+    'yml',
+    'yaml',
 ];
 export const sleep = (ms) => {
     return new Promise((resolve) => {
@@ -140,7 +145,7 @@ export const isSelfCrawler = (req) => {
  * Check if the request is a static file
  */
 export function isStaticFile(req) {
-    const excludeUrlPattern = new RegExp(`\\.(${staticFileExtensions.join('|')})$`, 'i');
+    const excludeUrlPattern = new RegExp(`\\.(${staticFileExtensions.join('|')})([\\?#]|$)`, 'i');
     return excludeUrlPattern.test(req.path);
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arcblock/crawler",
-  "version": "1.1.3",
+  "version": "1.1.5",
   "main": "lib/cjs/index.js",
   "module": "lib/esm/index.js",
   "types": "lib/cjs/index.d.ts",
@@ -45,33 +45,32 @@
     ]
   },
   "dependencies": {
-    "@abtnode/cron": "^1.16.43",
-    "@abtnode/models": "^1.16.43",
-    "@abtnode/queue": "^1.16.43",
-    "@blocklet/logger": "^1.16.43",
+    "@abtnode/cron": "^1.16.44",
+    "@abtnode/models": "^1.16.44",
+    "@abtnode/queue": "^1.16.44",
+    "@blocklet/logger": "^1.16.44",
     "@blocklet/puppeteer": "^22.11.3",
-    "@blocklet/sdk": "^1.16.43",
+    "@blocklet/sdk": "^1.16.44",
     "@sequelize/core": "7.0.0-alpha.46",
     "@sequelize/sqlite3": "7.0.0-alpha.46",
     "axios": "^1.7.9",
     "fs-extra": "^11.2.0",
     "lodash": "^4.17.21",
     "lru-cache": "^10.4.3",
+    "p-map": "^7.0.3",
     "robots-parser": "^3.0.1",
     "sitemap": "^7.1.2",
     "sqlite3": "^5.1.7",
-    "ufo": "^1.5.4",
-    "p-map": "^7.0.3"
+    "ufo": "^1.5.4"
   },
   "devDependencies": {
-    "@blocklet/js-sdk": "^1.16.39",
     "@types/dotenv-flow": "^3.3.3",
     "@types/express": "^4.17.21",
     "@types/fs-extra": "^11.0.4",
     "@types/lodash": "^4.17.16",
     "@types/node": "^20.17.19",
-    "express": "^4.21.2",
     "bumpp": "^9.11.1",
+    "express": "^4.21.2",
     "nodemon": "^3.1.9",
     "npm-run-all": "^4.1.5",
     "puppeteer": "^24.8.2",