@arcblock/crawler 1.0.1 → 1.0.3

package/dist/blocklet.d.ts

@@ -0,0 +1,6 @@
+export declare const crawlBlocklet: () => Promise<void>;
+export declare const initCronCrawlBlocklet: ({ time, options, }?: {
+    time: string;
+    options: any;
+}) => any;
+export declare const cancelCronCrawlBlocklet: () => void;

package/dist/blocklet.js

@@ -0,0 +1,199 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cancelCronCrawlBlocklet = exports.initCronCrawlBlocklet = exports.crawlBlocklet = void 0;
+const cron_1 = __importDefault(require("@abtnode/cron"));
+const config_1 = require("@blocklet/sdk/lib/config");
+const debounce_1 = __importDefault(require("lodash/debounce"));
+const ufo_1 = require("ufo");
+const cache_1 = require("./cache");
+const config_2 = require("./config");
+const crawler_1 = require("./crawler");
+const puppeteer_1 = require("./puppeteer");
+const utils_1 = require("./utils");
+// record crawl blocklet running
+const crawlBlockletRunningMap = new Map();
+// crawl blocklet sitemap urls
+const crawlBlocklet = () => __awaiter(void 0, void 0, void 0, function* () {
+    // @ts-ignore
+    const { mountPoint, did } = (0, utils_1.getComponentInfo)();
+    if (crawlBlockletRunningMap.has(did) && crawlBlockletRunningMap.get(did)) {
+        config_2.logger.info(`Crawler blocklet ${did} is running, skip it`);
+        return;
+    }
+    // check has browser can use
+    try {
+        const browser = yield (0, puppeteer_1.getBrowser)();
+        if (!browser) {
+            throw new Error('No Browser can use');
+        }
+        config_2.logger.info('Crawler blocklet existing can use browser');
+    }
+    catch (error) {
+        config_2.logger.info(`Crawler blocklet abort by error: ${(error === null || error === void 0 ? void 0 : error.message) || (error === null || error === void 0 ? void 0 : error.reason) || error}`);
+        return;
+    }
+    const { appUrl } = config_2.config;
+    if (!appUrl) {
+        throw new Error('appUrl not found');
+    }
+    const sitemapList = yield (0, utils_1.getSitemapList)(appUrl);
+    const matchMountPoint = (0, ufo_1.joinURL)(appUrl, !mountPoint || mountPoint === '/' ? '' : mountPoint);
+    const otherMountPointList = config_1.components
+        .filter((item) => item.mountPoint && item.mountPoint !== mountPoint)
+        .map((item) => item.mountPoint);
+    // get can use loc
+    const blockletLocList = sitemapList.filter((item) => {
+        var _a;
+        if (mountPoint !== '/') {
+            return ((_a = item === null || item === void 0 ? void 0 : item.url) === null || _a === void 0 ? void 0 : _a.indexOf(matchMountPoint)) > -1;
+        }
+        // if mountPoint is /, skip other mountPoint
+        return otherMountPointList.every((mountPoint) => { var _a; return ((_a = item === null || item === void 0 ? void 0 : item.url) === null || _a === void 0 ? void 0 : _a.indexOf(mountPoint)) === -1; });
+    });
+    const canUseBlockletLocList = [];
+    const lastmodMap = new Map();
+    let skipBlockletLocTotal = 0;
+    let blockletLocTotal = 0;
+    yield Promise.all(blockletLocList.map((item) => __awaiter(void 0, void 0, void 0, function* () {
+        var _a;
+        let tempLocList = [];
+        if (item.url) {
+            tempLocList.push(item.url);
+        }
+        if (((_a = item === null || item === void 0 ? void 0 : item.links) === null || _a === void 0 ? void 0 : _a.length) > 0) {
+            tempLocList.push(...item.links.map((ytem) => ytem.url));
+        }
+        blockletLocTotal += tempLocList.length;
+        // @ts-ignore
+        tempLocList = (yield Promise.all(tempLocList.map((loc) => __awaiter(void 0, void 0, void 0, function* () {
+            try {
+                const { lastModified: cacheLastModified } = yield cache_1.useCache.get((0, utils_1.getRelativePath)(loc));
+                // sitemap item lastmod is same as cache lastModified, skip it
+                if (item.lastmod &&
+                    cacheLastModified &&
+                    new Date(cacheLastModified).getTime() === new Date(item.lastmod).getTime()) {
+                    skipBlockletLocTotal++;
+                    return false;
+                }
+                return loc;
+            }
+            catch (error) {
+                // ignore error
+            }
+            // if can not get cache, return loc
+            return loc;
+        })))).filter(Boolean);
+        tempLocList.forEach((loc) => {
+            if (item.lastmod)
+                lastmodMap.set(loc, item.lastmod);
+        });
+        canUseBlockletLocList.push(...tempLocList);
+    })));
+    const crawlerLogText = (step = '') => [
+        `Crawler sitemap.xml about ${did} ${step}: `,
+        {
+            blockletLocTotal,
+            canUseBlockletLocTotal: canUseBlockletLocList.length,
+            skipBlockletLocTotal,
+            lastmodMapTotal: lastmodMap.size,
+        },
+    ];
+    config_2.logger.info(...crawlerLogText('start'));
+    try {
+        // record crawl blocklet running
+        crawlBlockletRunningMap.set(did, true);
+        yield (0, crawler_1.createCrawlJob)({
+            // @ts-ignore
+            urls: canUseBlockletLocList,
+            saveToRedis: true,
+            lastmodMap,
+            // formatPageContent: async ({ page }: { page: any; url: string; lastmod?: string }) => {
+            //   const pageContent = await page.evaluate(() => {
+            //     const removeElements = (tagName: string) => {
+            //       const elements = document.querySelectorAll(tagName);
+            //       for (let i = elements.length - 1; i >= 0; i--) {
+            //         try {
+            //           elements[i]?.parentNode?.removeChild(elements[i] as Node);
+            //         } catch (error) {
+            //           // do noting
+            //         }
+            //       }
+            //     };
+            //     // remove script, style, link, noscript
+            //     // removeElements('script');
+            //     // removeElements('style');
+            //     // removeElements('link');
+            //     // removeElements('noscript');
+            //     // remove uploader
+            //     removeElements('[id="uploader-container"]');
+            //     removeElements('[class^="uppy-"]');
+            //     // remove point up component
+            //     removeElements('[id="point-up-component"]');
+            //     // add meta tag to record crawler
+            //     const meta = document.createElement('meta');
+            //     meta.name = 'blocklet-crawler';
+            //     meta.content = 'true';
+            //     document.head.appendChild(meta);
+            //     return document.documentElement.outerHTML;
+            //   });
+            //   return pageContent;
+            // },
+        });
+        config_2.logger.info(...crawlerLogText('success'));
+        yield (0, puppeteer_1.closeBrowser)({
+            trimCache: true,
+        });
+    }
+    catch (error) {
+        config_2.logger.info('Crawler blocklet abort by error', error);
+    }
+    finally {
+        // delete crawl blocklet running
+        crawlBlockletRunningMap.delete(did);
+    }
+});
+exports.crawlBlocklet = crawlBlocklet;
+const CRON_CRAWL_BLOCKLET_KEY = 'cron-crawl-blocklet';
+let cronCrawlBlockletJob = null;
+// init cron crawl blocklet
+const initCronCrawlBlocklet = ({ time = '0 0 */12 * * *', // every 12 hours
+options, } = {}) => {
+    if (!cronCrawlBlockletJob) {
+        cronCrawlBlockletJob = cron_1.default.init({
+            context: {},
+            jobs: [
+                {
+                    name: CRON_CRAWL_BLOCKLET_KEY,
+                    time,
+                    fn: (0, debounce_1.default)(exports.crawlBlocklet),
+                    options: Object.assign({ runOnInit: false }, options),
+                },
+            ],
+            onError: (err) => {
+                console.error('run job failed', err);
+            },
+        });
+    }
+    return cronCrawlBlockletJob;
+};
+exports.initCronCrawlBlocklet = initCronCrawlBlocklet;
+const cancelCronCrawlBlocklet = () => {
+    if (cronCrawlBlockletJob) {
+        cronCrawlBlockletJob.jobs[CRON_CRAWL_BLOCKLET_KEY].stop();
+        cronCrawlBlockletJob = null;
+        config_2.logger.info('Cron crawl blocklet stop, clear crawl queue');
+    }
+};
+exports.cancelCronCrawlBlocklet = cancelCronCrawlBlocklet;

package/dist/cache.d.ts

@@ -0,0 +1,10 @@
+export declare const cachePool: import("generic-pool").Pool<any>;
+export declare const memoryPool: import("generic-pool").Pool<Map<string, any>>;
+export declare const withCache: (cb: Function) => Promise<any>;
+export declare const formatKey: (key: string) => string;
+export declare const useCache: {
+    get: (key: string) => Promise<any>;
+    set: (key: string, value: any, options?: any) => Promise<any>;
+    remove: (key: string) => Promise<any>;
+    list: (key?: string) => Promise<any>;
+};

package/dist/cache.js

@@ -0,0 +1,119 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var _a;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.useCache = exports.formatKey = exports.withCache = exports.memoryPool = exports.cachePool = void 0;
+const generic_pool_1 = require("generic-pool");
+const redis_1 = require("redis");
+const config_1 = require("./config");
+const cacheKeyPrefix = ((_a = process.env) === null || _a === void 0 ? void 0 : _a.BLOCKLET_REAL_DID) ? `${process.env.BLOCKLET_REAL_DID}:` : '';
+const MAX_REDIS_RETRY = 3;
+const ttl = 1000 * 60 * 60 * 24 * 7;
+exports.cachePool = (0, generic_pool_1.createPool)({
+    create: () => __awaiter(void 0, void 0, void 0, function* () {
+        try {
+            const { redisUrl } = config_1.config;
+            const redisClient = (0, redis_1.createClient)({
+                url: redisUrl,
+                socket: {
+                    // @ts-ignore
+                    reconnectStrategy: (retries) => {
+                        if (retries >= MAX_REDIS_RETRY) {
+                            return new Error('Retry Time Exhausted');
+                        }
+                        return Math.min(retries * 500, 1000 * 3);
+                    },
+                },
+            });
+            redisClient.on('error', (err) => config_1.logger.warn('Redis Client Error:', err));
+            yield redisClient.connect();
+            config_1.logger.info(`Successfully connected to Redis: ${redisUrl}`);
+            return redisClient;
+        }
+        catch (error) {
+            config_1.logger.warn('Redis connection failed', error);
+            return null;
+        }
+    }),
+    destroy: (client) => __awaiter(void 0, void 0, void 0, function* () {
+        // if is redis client
+        if (client.isReady) {
+            yield client.quit();
+        }
+    }),
+}, {
+    max: 2, // 2 clients
+    min: 0,
+    // evictionRunIntervalMillis: 0,
+});
+exports.memoryPool = (0, generic_pool_1.createPool)({
+    create: () => {
+        const map = new Map();
+        // @ts-ignore
+        map.del = map.delete;
+        return Promise.resolve(map);
+    },
+    destroy: (client) => {
+        client.clear();
+        return Promise.resolve();
+    },
+}, {
+    max: 10,
+    min: 0,
+});
+const withCache = (cb) => __awaiter(void 0, void 0, void 0, function* () {
+    const pool = config_1.config.redisUrl ? exports.cachePool : exports.memoryPool;
+    const client = yield pool.acquire();
+    if (client) {
+        try {
+            return cb(client);
+        }
+        finally {
+            // release client to pool, let other use
+            yield pool.release(client);
+        }
+    }
+});
+exports.withCache = withCache;
+const formatKey = (key) => {
+    return `${cacheKeyPrefix}${key}`;
+};
+exports.formatKey = formatKey;
+exports.useCache = {
+    get: (key) => {
+        return (0, exports.withCache)((client) => __awaiter(void 0, void 0, void 0, function* () {
+            const value = yield client.get((0, exports.formatKey)(key));
+            try {
+                return JSON.parse(value);
+            }
+            catch (error) {
+                // ignore error
+            }
+            return value;
+        }));
+    },
+    set: (key, value, options) => {
+        return (0, exports.withCache)((client) => {
+            const formatValue = typeof value === 'string' ? value : JSON.stringify(value);
+            return client.set((0, exports.formatKey)(key), formatValue, Object.assign({ PX: ttl }, options));
+        });
+    },
+    remove: (key) => {
+        return (0, exports.withCache)((client) => {
+            return client.del((0, exports.formatKey)(key));
+        });
+    },
+    list: (key = '*') => {
+        return (0, exports.withCache)((client) => {
+            return client.keys((0, exports.formatKey)(key));
+        });
+    },
+};

package/dist/config.d.ts

@@ -0,0 +1,10 @@
+export declare const logger: any;
+export declare const config: {
+    redisUrl: string;
+    dataDir: string;
+    appDir: string;
+    appUrl: string;
+    puppeteerPath: string;
+    cacheDir: string;
+    testOnInitialize: boolean;
+};

package/dist/config.js

@@ -0,0 +1,17 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.config = exports.logger = void 0;
+const logger_1 = __importDefault(require("@blocklet/logger"));
+exports.logger = (0, logger_1.default)('crawler', { level: process.env.LOG_LEVEL || 'info' });
+exports.config = {
+    redisUrl: process.env.REDIS_URL,
+    dataDir: process.env.BLOCKLET_DATA_DIR,
+    appDir: process.env.BLOCKLET_APP_DIR || process.cwd(),
+    appUrl: process.env.BLOCKLET_APP_URL,
+    puppeteerPath: process.env.PUPPETEER_EXECUTABLE_PATH,
+    cacheDir: process.env.BLOCKLET_CACHE_DIR,
+    testOnInitialize: process.env.NODE_ENV === 'production',
+};

package/dist/crawler.d.ts

@@ -0,0 +1,28 @@
+import { JobState } from './db/job';
+import { SnapshotModel } from './db/snapshot';
+export declare function createCrawlQueue(): void;
+export declare function getDataDir(): Promise<{
+    htmlDir: string;
+    screenshotDir: string;
+}>;
+export declare const getPageContent: ({ url, formatPageContent, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, }: {
+    url: string;
+    formatPageContent?: Function;
+    includeScreenshot?: boolean;
+    includeHtml?: boolean;
+    width?: number;
+    height?: number;
+    quality?: number;
+    timeout?: number;
+    fullPage?: boolean;
+}) => Promise<{
+    html: string;
+    screenshot: Uint8Array<ArrayBufferLike> | null;
+}>;
+export declare function createCrawlJob(params: JobState, callback?: (snapshot: SnapshotModel | null) => void): Promise<any>;
+export declare function getJob(condition: Partial<JobState>): Promise<any>;
+export declare function formatSnapshot(snapshot: SnapshotModel, columns?: Array<keyof SnapshotModel>): Promise<SnapshotModel>;
+/**
+ * get snapshot from db or crawl queue
+ */
+export declare function getSnapshot(jobId: string): Promise<SnapshotModel | null>;