@arcblock/crawler 1.0.0 → 1.0.2

package/dist/puppeteer.js

@@ -0,0 +1,318 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.closeBrowser = exports.getBrowser = exports.puppeteer = void 0;
+exports.ensurePuppeteerrc = ensurePuppeteerrc;
+exports.ensureBrowser = ensureBrowser;
+exports.connectBrowser = connectBrowser;
+exports.launchBrowser = launchBrowser;
+exports.initPage = initPage;
+// import fs from 'fs-extra';
+// import path from 'path';
+const puppeteer_1 = __importDefault(require("@blocklet/puppeteer"));
+exports.puppeteer = puppeteer_1.default;
+const config_1 = require("@blocklet/sdk/lib/config");
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const path_1 = __importDefault(require("path"));
+const timers_1 = require("timers");
+const cache_1 = require("./cache");
+const config_2 = require("./config");
+const utils_1 = require("./utils");
+// let puppeteerConfig: {
+//   cacheDirectory: string;
+//   temporaryDirectory: string;
+// };
+const BROWSER_WS_ENDPOINT_KEY = `browserWSEndpoint-${config_1.env.appId || 'unknown'}`;
+const BrowserStatus = {
+    Launching: 'Launching',
+    Ready: 'Ready',
+};
+let browser;
+let browserActivatedTimer;
+function ensurePuppeteerrc() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const cacheDirectory = path_1.default.join(config_2.config.cacheDir, 'puppeteer', 'cache');
+        const temporaryDirectory = path_1.default.join(config_2.config.cacheDir, 'puppeteer', 'tmp');
+        const puppeteerrcPath = path_1.default.join(config_2.config.appDir, '.puppeteerrc.js');
+        // ensure directory exists
+        yield Promise.all([fs_extra_1.default.ensureDir(cacheDirectory), fs_extra_1.default.ensureDir(temporaryDirectory), fs_extra_1.default.ensureFile(puppeteerrcPath)]);
+        const puppeteerConfig = {
+            cacheDirectory,
+            temporaryDirectory,
+        };
+        const fileContent = `module.exports = ${JSON.stringify(puppeteerConfig, null, 2)}`;
+        yield fs_extra_1.default.writeFile(puppeteerrcPath, fileContent);
+        config_2.logger.debug(`Puppeteerrc file created at ${puppeteerrcPath}`, puppeteerConfig);
+        return puppeteerConfig;
+    });
+}
+function ensureBrowser() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const puppeteerConfig = yield ensurePuppeteerrc();
+        const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium';
+        config_2.logger.info('executablePath', executablePath);
+        if (!fs_extra_1.default.existsSync(executablePath)) {
+            config_2.logger.info('start download browser', puppeteerConfig);
+            const { downloadBrowser } = yield (() => __awaiter(this, void 0, void 0, function* () {
+                try {
+                    // @ts-ignore
+                    // eslint-disable-next-line import/extensions
+                    return yield Promise.resolve().then(() => __importStar(require('@blocklet/puppeteer/internal/node/install.js')));
+                }
+                catch (err) {
+                    config_2.logger.warn('Skipping browser installation because the Puppeteer build is not available. Run `npm install` again after you have re-built Puppeteer.');
+                }
+            }))();
+            if (downloadBrowser) {
+                yield downloadBrowser();
+                config_2.logger.info('Browser download completed successfully');
+            }
+        }
+        // try to launch browser
+        if (config_2.config.testOnInitialize) {
+            const browser = yield launchBrowser();
+            if (!browser) {
+                throw new Error('Failed to launch browser');
+            }
+            yield (0, exports.closeBrowser)();
+        }
+        config_2.logger.info('Puppeteer is ready');
+    });
+}
+function connectBrowser() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const browserWSEndpoint = yield cache_1.useCache.get(BROWSER_WS_ENDPOINT_KEY);
+        if (!browserWSEndpoint) {
+            return null;
+        }
+        // retry if browser is launching
+        if (browserWSEndpoint.status === BrowserStatus.Launching) {
+            yield (0, utils_1.sleep)(Math.floor(Math.random() * 1000));
+            return connectBrowser();
+        }
+        try {
+            browser = yield puppeteer_1.default.connect({
+                browserWSEndpoint: browserWSEndpoint.endpoint,
+            });
+            config_2.logger.info('Connect browser success');
+        }
+        catch (err) {
+            config_2.logger.warn('Connect browser failed, clear endpoint', err);
+            yield cache_1.useCache.remove(BROWSER_WS_ENDPOINT_KEY);
+            return null;
+        }
+        return browser;
+    });
+}
+function launchBrowser() {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield cache_1.useCache.set(BROWSER_WS_ENDPOINT_KEY, {
+            endpoint: null,
+            status: BrowserStatus.Launching,
+        });
+        try {
+            // @ts-ignore
+            browser = yield puppeteer_1.default.launch({
+                headless: true,
+                args: [
+                    // docs: https://peter.sh/experiments/chromium-command-line-switches/
+                    '--no-first-run',
+                    '--hide-scrollbars',
+                    '--no-sandbox',
+                    '--no-zygote',
+                    '--disable-setuid-sandbox',
+                    '--disable-gpu',
+                    '--disable-dev-shm-usage',
+                    '--disable-site-isolation-trials',
+                    '--disable-accelerated-2d-canvas',
+                    '--disable-extensions',
+                    '--js-flags=--max_old_space_size=512', // 限制V8内存
+                    '--disable-background-networking',
+                    '--disable-default-apps',
+                    // '--disable-web-security', // 允许跨域请求
+                    '--disable-software-rasterizer',
+                    '--disable-crash-reporter',
+                    '--disable-service-workers',
+                    '--disable-notifications',
+                    '--disable-infobars',
+                    '--font-render-hinting=none',
+                ],
+            });
+            config_2.logger.info('Launch browser success');
+        }
+        catch (error) {
+            config_2.logger.error('launch browser failed: ', error);
+            // cleanup browser endpoint
+            yield cache_1.useCache.remove(BROWSER_WS_ENDPOINT_KEY);
+            throw error;
+        }
+        // save browserWSEndpoint to cache
+        const endpoint = yield browser.wsEndpoint();
+        yield cache_1.useCache.set(BROWSER_WS_ENDPOINT_KEY, {
+            endpoint,
+            status: BrowserStatus.Ready,
+        });
+        return browser;
+    });
+}
+function checkBrowserActivated() {
+    clearBrowserActivatedTimer();
+    let count = 0;
+    browserActivatedTimer = (0, timers_1.setInterval)(() => __awaiter(this, void 0, void 0, function* () {
+        var _a;
+        if (browser) {
+            const pages = yield browser.pages().catch(() => []);
+            if (pages.length === 1 && ((_a = pages[0]) === null || _a === void 0 ? void 0 : _a.url()) === 'about:blank') {
+                count++;
+                config_2.logger.debug(`Browser inactive count: ${count}/3`);
+            }
+            else {
+                count = 0; // 重置计数器！
+            }
+            if (count >= 3) {
+                config_2.logger.info('Browser inactive for 3 minutes, closing...');
+                yield (0, exports.closeBrowser)({
+                    trimCache: true,
+                });
+            }
+        }
+    }), 1000 * 60);
+}
+function clearBrowserActivatedTimer() {
+    if (browserActivatedTimer) {
+        (0, timers_1.clearInterval)(browserActivatedTimer);
+        browserActivatedTimer = null;
+    }
+}
+const getBrowser = () => __awaiter(void 0, void 0, void 0, function* () {
+    if (browser)
+        return browser;
+    // sleep random time (0 ~ 5s),to avoid concurrent blocklet
+    yield (0, utils_1.sleep)(Math.floor(Math.random() * 1000 * 5));
+    // try to connect browser
+    const connectedBrowser = yield connectBrowser();
+    if (connectedBrowser) {
+        config_2.logger.debug('getBrowser.connectedBrowser');
+        browser = connectedBrowser;
+        return browser;
+    }
+    // try to launch browser
+    const launchedBrowser = yield launchBrowser();
+    if (launchedBrowser) {
+        config_2.logger.debug('getBrowser.launchedBrowser');
+        browser = launchedBrowser;
+        checkBrowserActivated();
+        return browser;
+    }
+    throw new Error('No browser to use, should install redis or browser');
+});
+exports.getBrowser = getBrowser;
+const closeBrowser = (...args_1) => __awaiter(void 0, [...args_1], void 0, function* ({ trimCache = true } = {}) {
+    if (!browser)
+        return;
+    // close all pages
+    try {
+        const pages = yield browser.pages();
+        yield Promise.all(pages.map((page) => page.close()));
+    }
+    catch (err) {
+        config_2.logger.error('Failed to close all pages:', err);
+    }
+    // close browser
+    try {
+        yield browser.close();
+    }
+    catch (err) {
+        config_2.logger.error('Failed to close browser:', err);
+    }
+    // clear cache
+    try {
+        if (trimCache) {
+            yield puppeteer_1.default.trimCache();
+            config_2.logger.info('Trim cache success');
+        }
+        // try to clear temporary directory
+        // if (puppeteerConfig) {
+        //   await fs.emptyDir(puppeteerConfig.temporaryDirectory);
+        // }
+        if (global.gc) {
+            global.gc();
+        }
+    }
+    catch (err) {
+        config_2.logger.error('Failed to clear browser cache:', err);
+    }
+    browser = null;
+    clearBrowserActivatedTimer();
+    yield cache_1.useCache.remove(BROWSER_WS_ENDPOINT_KEY);
+    config_2.logger.info('Close browser success');
+});
+exports.closeBrowser = closeBrowser;
+function initPage() {
+    return __awaiter(this, arguments, void 0, function* ({ abortResourceTypes = [] } = {}) {
+        const browser = yield (0, exports.getBrowser)();
+        const page = yield browser.newPage();
+        yield page.setViewport({ width: 1440, height: 900 });
+        // page setting
+        // add custom headers
+        yield page.setExtraHTTPHeaders({
+            [utils_1.CRAWLER_FLAG]: 'true',
+        });
+        // abort resource types
+        if (abortResourceTypes.length > 0) {
+            yield page.setRequestInterception(true);
+            page.on('request', (req) => {
+                // @ts-ignore
+                if (abortResourceTypes.includes(req.resourceType())) {
+                    return req.abort();
+                }
+                return req.continue();
+            });
+        }
+        return page;
+    });
+}

package/dist/utils.d.ts

@@ -0,0 +1,15 @@
+export declare const api: import("axios").AxiosInstance;
+export declare const sleep: (ms: number) => Promise<unknown>;
+export declare const CRAWLER_FLAG = "x-crawler";
+export declare const isSelfCrawler: (req: any) => boolean;
+export declare const getDefaultRobotsUrl: (url: string) => string;
+export declare function getRobots(url: string): Promise<import("robots-parser").Robot | null>;
+export declare const getDefaultSitemapUrl: (url: string) => string;
+export declare const isAcceptCrawler: (url: string) => Promise<boolean | undefined>;
+export declare const getSitemapList: (url: string) => Promise<import("sitemap").SitemapItem[]>;
+export declare const isBotUserAgent: (req: any) => boolean;
+export declare const getComponentInfo: () => {};
+export declare const getFullUrl: (req: any) => string;
+export declare const getRelativePath: (url: string) => string;
+export declare const formatUrl: (url: string) => string;
+export declare function md5(content: string | Uint8Array): string;

package/dist/utils.js

@@ -0,0 +1,239 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatUrl = exports.getRelativePath = exports.getFullUrl = exports.getComponentInfo = exports.isBotUserAgent = exports.getSitemapList = exports.isAcceptCrawler = exports.getDefaultSitemapUrl = exports.getDefaultRobotsUrl = exports.isSelfCrawler = exports.CRAWLER_FLAG = exports.sleep = exports.api = void 0;
+exports.getRobots = getRobots;
+exports.md5 = md5;
+const config_1 = require("@blocklet/sdk/lib/config");
+const axios_1 = __importDefault(require("axios"));
+const flattenDeep_1 = __importDefault(require("lodash/flattenDeep"));
+const uniq_1 = __importDefault(require("lodash/uniq"));
+const node_crypto_1 = require("node:crypto");
+const robots_parser_1 = __importDefault(require("robots-parser"));
+const sitemap_1 = require("sitemap");
+const stream_1 = require("stream");
+const ufo_1 = require("ufo");
+exports.api = axios_1.default.create({
+    timeout: 1000 * 10,
+    headers: {
+        'Content-Type': 'application/json',
+    },
+});
+const sleep = (ms) => {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+};
+exports.sleep = sleep;
+exports.CRAWLER_FLAG = 'x-crawler';
+const isSelfCrawler = (req) => {
+    const ua = req.get('user-agent') || '';
+    return req.get(exports.CRAWLER_FLAG) === 'true' || `${ua}`.toLowerCase().indexOf('headless') !== -1;
+};
+exports.isSelfCrawler = isSelfCrawler;
+/**
+ * A default set of user agent patterns for bots/crawlers that do not perform
+ * well with pages that require JavaScript.
+ */
+const botUserAgents = [
+    /bot/i,
+    /spider/i,
+    /facebookexternalhit/i,
+    /simplepie/i,
+    /yahooseeker/i,
+    /embedly/i,
+    /quora link preview/i,
+    /outbrain/i,
+    /vkshare/i,
+    /monit/i,
+    /Pingability/i,
+    /Monitoring/i,
+    /WinHttpRequest/i,
+    /Apache-HttpClient/i,
+    /getprismatic.com/i,
+    /python-requests/i,
+    /Twurly/i,
+    /yandex/i,
+    /browserproxy/i,
+    /crawler/i,
+    /Qwantify/i,
+    /Yahoo/i,
+    /pinterest/i,
+    /Tumblr/i,
+    /Tumblr Agent/i,
+    /WhatsApp/i,
+    /Google-Structured-Data-Testing-Tool/i,
+    /Google-InspectionTool/i,
+    /Googlebot/i,
+    /GPTBot/i,
+    /Applebot/i,
+    // AI bots
+    /Anthropic-ai/i,
+    /Claude-Web/i,
+    /anthropic-ai-scraper/i,
+    /Google-Extended/i,
+    /GoogleOther/i,
+    /CCBot\/\d/i,
+    /Bytespider/i,
+    /BingBot/i,
+    /Baiduspider/i,
+    /Sogou/i,
+    /Perplexity/i,
+    /Cohere-ai/i,
+    /xlts-bot/i,
+    /THAAS/i,
+    /YisouSpider/i,
+    /AlibabaGroup/i,
+    /adaptive-edge-crawler/i,
+];
+const isSpider = (ua) => botUserAgents.some((spider) => {
+    return spider.test(ua);
+});
+/**
+ * A default set of file extensions for static assets that do not need to be
+ * proxied.
+ */
+const staticFileExtensions = [
+    'ai',
+    'avi',
+    'css',
+    'dat',
+    'dmg',
+    'doc',
+    'doc',
+    'exe',
+    'flv',
+    'gif',
+    'ico',
+    'iso',
+    'jpeg',
+    'jpg',
+    'js',
+    'less',
+    'm4a',
+    'm4v',
+    'mov',
+    'mp3',
+    'mp4',
+    'mpeg',
+    'mpg',
+    'pdf',
+    'png',
+    'ppt',
+    'psd',
+    'rar',
+    'rss',
+    'svg',
+    'swf',
+    'tif',
+    'torrent',
+    'ttf',
+    'txt',
+    'wav',
+    'wmv',
+    'woff',
+    'xls',
+    'xml',
+    'zip',
+];
+const getDefaultRobotsUrl = (url) => {
+    const { origin } = new URL(url);
+    return (0, ufo_1.joinURL)(origin, 'robots.txt?nocache=1');
+};
+exports.getDefaultRobotsUrl = getDefaultRobotsUrl;
+function getRobots(url) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { origin } = new URL(url);
+        const robotsUrl = (0, ufo_1.joinURL)(origin, 'robots.txt?nocache=1');
+        const { data } = yield exports.api.get(robotsUrl).catch(() => ({
+            data: '',
+        }));
+        return data ? (0, robots_parser_1.default)(robotsUrl, data) : null;
+    });
+}
+const getDefaultSitemapUrl = (url) => {
+    const { origin } = new URL(url);
+    return (0, ufo_1.joinURL)(origin, 'sitemap.xml?nocache=1');
+};
+exports.getDefaultSitemapUrl = getDefaultSitemapUrl;
+const isAcceptCrawler = (url) => __awaiter(void 0, void 0, void 0, function* () {
+    const robots = yield getRobots(url);
+    const isAllowed = robots ? yield robots.isAllowed(url) : true;
+    return isAllowed;
+});
+exports.isAcceptCrawler = isAcceptCrawler;
+const getSitemapList = (url) => __awaiter(void 0, void 0, void 0, function* () {
+    let sitemapUrlList = [(0, exports.getDefaultSitemapUrl)(url)];
+    const robots = yield getRobots(url);
+    if (robots) {
+        const robotsTxtSitemapUrlList = (yield robots.getSitemaps()) || [];
+        if (robotsTxtSitemapUrlList.length > 0) {
+            sitemapUrlList = robotsTxtSitemapUrlList;
+        }
+    }
+    // loop site map url list
+    const sitemapList = yield Promise.all(sitemapUrlList.map((sitemapUrl) => __awaiter(void 0, void 0, void 0, function* () {
+        const newUrl = new URL(sitemapUrl);
+        newUrl.searchParams.set('nocache', '1');
+        sitemapUrl = newUrl.toString();
+        const { data: sitemapTxt } = yield exports.api.get(sitemapUrl).catch(() => ({
+            data: '',
+        }));
+        if (sitemapTxt) {
+            const stream = stream_1.Readable.from([sitemapTxt]);
+            const sitemapJson = yield (0, sitemap_1.parseSitemap)(stream);
+            return sitemapJson;
+        }
+        return [];
+    })));
+    return (0, uniq_1.default)((0, flattenDeep_1.default)(sitemapList.filter(Boolean)));
+});
+exports.getSitemapList = getSitemapList;
+const isBotUserAgent = (req) => {
+    const ua = req.get('user-agent');
+    const excludeUrlPattern = new RegExp(`\\.(${staticFileExtensions.join('|')})$`, 'i');
+    if (ua === undefined || !isSpider(ua) || excludeUrlPattern.test(req.path)) {
+        return false;
+    }
+    return true;
+};
+exports.isBotUserAgent = isBotUserAgent;
+const getComponentInfo = () => {
+    return config_1.components.find((item) => item.did === config_1.env.componentDid) || {};
+};
+exports.getComponentInfo = getComponentInfo;
+const getFullUrl = (req) => {
+    const blockletPathname = req.headers['x-path-prefix']
+        ? (0, ufo_1.joinURL)(req.headers['x-path-prefix'], req.originalUrl)
+        : req.originalUrl;
+    return (0, ufo_1.joinURL)(config_1.env.appUrl, blockletPathname);
+};
+exports.getFullUrl = getFullUrl;
+const getRelativePath = (url) => {
+    try {
+        return new URL(url).pathname;
+    }
+    catch (error) {
+        // ignore error
+    }
+    return url;
+};
+exports.getRelativePath = getRelativePath;
+const formatUrl = (url) => {
+    return url.replace(/\/$/, '').trim();
+};
+exports.formatUrl = formatUrl;
+function md5(content) {
+    return (0, node_crypto_1.createHash)('md5').update(content).digest('hex');
+}

package/esm/blocklet.d.ts

@@ -0,0 +1,6 @@
+export declare const crawlBlocklet: () => Promise<void>;
+export declare const initCronCrawlBlocklet: ({ time, options, }?: {
+    time: string;
+    options: any;
+}) => any;
+export declare const cancelCronCrawlBlocklet: () => void;