@arcblock/crawler 1.0.1 → 1.0.2

package/esm/utils.js

@@ -0,0 +1,220 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { components, env } from '@blocklet/sdk/lib/config';
+import axios from 'axios';
+import flattenDeep from 'lodash/flattenDeep';
+import uniq from 'lodash/uniq';
+import { createHash } from 'node:crypto';
+import robotsParser from 'robots-parser';
+import { parseSitemap } from 'sitemap';
+import { Readable } from 'stream';
+import { joinURL } from 'ufo';
+export const api = axios.create({
+    timeout: 1000 * 10,
+    headers: {
+        'Content-Type': 'application/json',
+    },
+});
+export const sleep = (ms) => {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+};
+export const CRAWLER_FLAG = 'x-crawler';
+export const isSelfCrawler = (req) => {
+    const ua = req.get('user-agent') || '';
+    return req.get(CRAWLER_FLAG) === 'true' || `${ua}`.toLowerCase().indexOf('headless') !== -1;
+};
+/**
+ * A default set of user agent patterns for bots/crawlers that do not perform
+ * well with pages that require JavaScript.
+ */
+const botUserAgents = [
+    /bot/i,
+    /spider/i,
+    /facebookexternalhit/i,
+    /simplepie/i,
+    /yahooseeker/i,
+    /embedly/i,
+    /quora link preview/i,
+    /outbrain/i,
+    /vkshare/i,
+    /monit/i,
+    /Pingability/i,
+    /Monitoring/i,
+    /WinHttpRequest/i,
+    /Apache-HttpClient/i,
+    /getprismatic.com/i,
+    /python-requests/i,
+    /Twurly/i,
+    /yandex/i,
+    /browserproxy/i,
+    /crawler/i,
+    /Qwantify/i,
+    /Yahoo/i,
+    /pinterest/i,
+    /Tumblr/i,
+    /Tumblr Agent/i,
+    /WhatsApp/i,
+    /Google-Structured-Data-Testing-Tool/i,
+    /Google-InspectionTool/i,
+    /Googlebot/i,
+    /GPTBot/i,
+    /Applebot/i,
+    // AI bots
+    /Anthropic-ai/i,
+    /Claude-Web/i,
+    /anthropic-ai-scraper/i,
+    /Google-Extended/i,
+    /GoogleOther/i,
+    /CCBot\/\d/i,
+    /Bytespider/i,
+    /BingBot/i,
+    /Baiduspider/i,
+    /Sogou/i,
+    /Perplexity/i,
+    /Cohere-ai/i,
+    /xlts-bot/i,
+    /THAAS/i,
+    /YisouSpider/i,
+    /AlibabaGroup/i,
+    /adaptive-edge-crawler/i,
+];
+const isSpider = (ua) => botUserAgents.some((spider) => {
+    return spider.test(ua);
+});
+/**
+ * A default set of file extensions for static assets that do not need to be
+ * proxied.
+ */
+const staticFileExtensions = [
+    'ai',
+    'avi',
+    'css',
+    'dat',
+    'dmg',
+    'doc',
+    'doc',
+    'exe',
+    'flv',
+    'gif',
+    'ico',
+    'iso',
+    'jpeg',
+    'jpg',
+    'js',
+    'less',
+    'm4a',
+    'm4v',
+    'mov',
+    'mp3',
+    'mp4',
+    'mpeg',
+    'mpg',
+    'pdf',
+    'png',
+    'ppt',
+    'psd',
+    'rar',
+    'rss',
+    'svg',
+    'swf',
+    'tif',
+    'torrent',
+    'ttf',
+    'txt',
+    'wav',
+    'wmv',
+    'woff',
+    'xls',
+    'xml',
+    'zip',
+];
+export const getDefaultRobotsUrl = (url) => {
+    const { origin } = new URL(url);
+    return joinURL(origin, 'robots.txt?nocache=1');
+};
+export function getRobots(url) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { origin } = new URL(url);
+        const robotsUrl = joinURL(origin, 'robots.txt?nocache=1');
+        const { data } = yield api.get(robotsUrl).catch(() => ({
+            data: '',
+        }));
+        return data ? robotsParser(robotsUrl, data) : null;
+    });
+}
+export const getDefaultSitemapUrl = (url) => {
+    const { origin } = new URL(url);
+    return joinURL(origin, 'sitemap.xml?nocache=1');
+};
+export const isAcceptCrawler = (url) => __awaiter(void 0, void 0, void 0, function* () {
+    const robots = yield getRobots(url);
+    const isAllowed = robots ? yield robots.isAllowed(url) : true;
+    return isAllowed;
+});
+export const getSitemapList = (url) => __awaiter(void 0, void 0, void 0, function* () {
+    let sitemapUrlList = [getDefaultSitemapUrl(url)];
+    const robots = yield getRobots(url);
+    if (robots) {
+        const robotsTxtSitemapUrlList = (yield robots.getSitemaps()) || [];
+        if (robotsTxtSitemapUrlList.length > 0) {
+            sitemapUrlList = robotsTxtSitemapUrlList;
+        }
+    }
+    // loop site map url list
+    const sitemapList = yield Promise.all(sitemapUrlList.map((sitemapUrl) => __awaiter(void 0, void 0, void 0, function* () {
+        const newUrl = new URL(sitemapUrl);
+        newUrl.searchParams.set('nocache', '1');
+        sitemapUrl = newUrl.toString();
+        const { data: sitemapTxt } = yield api.get(sitemapUrl).catch(() => ({
+            data: '',
+        }));
+        if (sitemapTxt) {
+            const stream = Readable.from([sitemapTxt]);
+            const sitemapJson = yield parseSitemap(stream);
+            return sitemapJson;
+        }
+        return [];
+    })));
+    return uniq(flattenDeep(sitemapList.filter(Boolean)));
+});
+export const isBotUserAgent = (req) => {
+    const ua = req.get('user-agent');
+    const excludeUrlPattern = new RegExp(`\\.(${staticFileExtensions.join('|')})$`, 'i');
+    if (ua === undefined || !isSpider(ua) || excludeUrlPattern.test(req.path)) {
+        return false;
+    }
+    return true;
+};
+export const getComponentInfo = () => {
+    return components.find((item) => item.did === env.componentDid) || {};
+};
+export const getFullUrl = (req) => {
+    const blockletPathname = req.headers['x-path-prefix']
+        ? joinURL(req.headers['x-path-prefix'], req.originalUrl)
+        : req.originalUrl;
+    return joinURL(env.appUrl, blockletPathname);
+};
+export const getRelativePath = (url) => {
+    try {
+        return new URL(url).pathname;
+    }
+    catch (error) {
+        // ignore error
+    }
+    return url;
+};
+export const formatUrl = (url) => {
+    return url.replace(/\/$/, '').trim();
+};
+export function md5(content) {
+    return createHash('md5').update(content).digest('hex');
+}

package/package.json

@@ -1,11 +1,15 @@
 {
   "name": "@arcblock/crawler",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "publishConfig": {
     "access": "public"
   },
+  "files": [
+    "dist",
+    "esm"
+  ],
   "lint-staged": {
     "*.{mjs,js,jsx,ts,tsx}": [
       "prettier --write",
@@ -102,9 +106,12 @@
     "pre-commit": "npx lint-staged"
   },
   "scripts": {
-    "dev": "tsc --watch",
+    "dev": "tsc -p tsconfig.cjs.json --watch",
     "lint": "tsc --noEmit && eslint src --ext .mjs,.js,.jsx,.ts,.tsx",
     "lint:fix": "npm run lint -- --fix",
-    "bundle": "tsc"
+    "bundle": "npm run build",
+    "build:cjs": "tsc -p tsconfig.cjs.json",
+    "build:esm": "tsc -p tsconfig.esm.json",
+    "build": "npm run build:cjs && npm run build:esm"
   }
 }

package/src/blocklet.ts

@@ -1,223 +0,0 @@
-import Cron from '@abtnode/cron';
-import { components } from '@blocklet/sdk/lib/config';
-import debounce from 'lodash/debounce';
-import { joinURL } from 'ufo';
-
-import { useCache } from './cache';
-import { config, logger } from './config';
-import { createCrawlJob } from './crawler';
-import { closeBrowser, getBrowser } from './puppeteer';
-import { getComponentInfo, getRelativePath, getSitemapList } from './utils';
-
-// record crawl blocklet running
-const crawlBlockletRunningMap = new Map();
-
-// crawl blocklet sitemap urls
-export const crawlBlocklet = async () => {
-  // @ts-ignore
-  const { mountPoint, did } = getComponentInfo();
-
-  if (crawlBlockletRunningMap.has(did) && crawlBlockletRunningMap.get(did)) {
-    logger.info(`Crawler blocklet ${did} is running, skip it`);
-    return;
-  }
-
-  // check has browser can use
-  try {
-    const browser = await getBrowser();
-    if (!browser) {
-      throw new Error('No Browser can use');
-    }
-    logger.info('Crawler blocklet existing can use browser');
-  } catch (error: any) {
-    logger.info(`Crawler blocklet abort by error: ${error?.message || error?.reason || error}`);
-    return;
-  }
-
-  const { appUrl } = config;
-
-  if (!appUrl) {
-    throw new Error('appUrl not found');
-  }
-
-  const sitemapList = await getSitemapList(appUrl);
-
-  const matchMountPoint = joinURL(appUrl, !mountPoint || mountPoint === '/' ? '' : mountPoint);
-  const otherMountPointList = components
-    .filter((item) => item.mountPoint && item.mountPoint !== mountPoint)
-    .map((item) => item.mountPoint);
-
-  // get can use loc
-  const blockletLocList = sitemapList.filter((item: any) => {
-    if (mountPoint !== '/') {
-      return item?.url?.indexOf(matchMountPoint) > -1;
-    }
-    // if mountPoint is /, skip other mountPoint
-    return otherMountPointList.every((mountPoint) => item?.url?.indexOf(mountPoint) === -1);
-  }) as [];
-
-  const canUseBlockletLocList = [] as string[];
-  const lastmodMap = new Map();
-  let skipBlockletLocTotal = 0;
-  let blockletLocTotal = 0;
-
-  await Promise.all(
-    blockletLocList.map(async (item: any) => {
-      let tempLocList: string[] = [];
-
-      if (item.url) {
-        tempLocList.push(item.url);
-      }
-
-      if (item?.links?.length > 0) {
-        tempLocList.push(...item.links.map((ytem: any) => ytem.url));
-      }
-
-      blockletLocTotal += tempLocList.length;
-
-      // @ts-ignore
-      tempLocList = (
-        await Promise.all(
-          tempLocList.map(async (loc) => {
-            try {
-              const { lastModified: cacheLastModified } = await useCache.get(getRelativePath(loc));
-
-              // sitemap item lastmod is same as cache lastModified, skip it
-              if (
-                item.lastmod &&
-                cacheLastModified &&
-                new Date(cacheLastModified).getTime() === new Date(item.lastmod).getTime()
-              ) {
-                skipBlockletLocTotal++;
-                return false;
-              }
-
-              return loc;
-            } catch (error) {
-              // ignore error
-            }
-
-            // if can not get cache, return loc
-            return loc;
-          }),
-        )
-      ).filter(Boolean);
-
-      tempLocList.forEach((loc) => {
-        if (item.lastmod) lastmodMap.set(loc, item.lastmod);
-      });
-
-      canUseBlockletLocList.push(...tempLocList);
-    }),
-  );
-
-  const crawlerLogText = (step = '') => [
-    `Crawler sitemap.xml about ${did} ${step}: `,
-    {
-      blockletLocTotal,
-      canUseBlockletLocTotal: canUseBlockletLocList.length,
-      skipBlockletLocTotal,
-      lastmodMapTotal: lastmodMap.size,
-    },
-  ];
-
-  logger.info(...crawlerLogText('start'));
-
-  try {
-    // record crawl blocklet running
-    crawlBlockletRunningMap.set(did, true);
-
-    await createCrawlJob({
-      // @ts-ignore
-      urls: canUseBlockletLocList,
-      saveToRedis: true,
-      lastmodMap,
-      // formatPageContent: async ({ page }: { page: any; url: string; lastmod?: string }) => {
-      //   const pageContent = await page.evaluate(() => {
-      //     const removeElements = (tagName: string) => {
-      //       const elements = document.querySelectorAll(tagName);
-      //       for (let i = elements.length - 1; i >= 0; i--) {
-      //         try {
-      //           elements[i]?.parentNode?.removeChild(elements[i] as Node);
-      //         } catch (error) {
-      //           // do noting
-      //         }
-      //       }
-      //     };
-
-      //     // remove script, style, link, noscript
-      //     // removeElements('script');
-      //     // removeElements('style');
-      //     // removeElements('link');
-      //     // removeElements('noscript');
-
-      //     // remove uploader
-      //     removeElements('[id="uploader-container"]');
-      //     removeElements('[class^="uppy-"]');
-
-      //     // remove point up component
-      //     removeElements('[id="point-up-component"]');
-
-      //     // add meta tag to record crawler
-      //     const meta = document.createElement('meta');
-      //     meta.name = 'blocklet-crawler';
-      //     meta.content = 'true';
-      //     document.head.appendChild(meta);
-
-      //     return document.documentElement.outerHTML;
-      //   });
-
-      //   return pageContent;
-      // },
-    });
-
-    logger.info(...crawlerLogText('success'));
-
-    await closeBrowser({
-      trimCache: true,
-    });
-  } catch (error) {
-    logger.info('Crawler blocklet abort by error', error);
-  } finally {
-    // delete crawl blocklet running
-    crawlBlockletRunningMap.delete(did);
-  }
-};
-
-const CRON_CRAWL_BLOCKLET_KEY = 'cron-crawl-blocklet';
-let cronCrawlBlockletJob = null as any;
-
-// init cron crawl blocklet
-export const initCronCrawlBlocklet = (
-  {
-    time = '0 0 */12 * * *', // every 12 hours
-    options,
-  } = {} as { time: string; options: any },
-) => {
-  if (!cronCrawlBlockletJob) {
-    cronCrawlBlockletJob = Cron.init({
-      context: {},
-      jobs: [
-        {
-          name: CRON_CRAWL_BLOCKLET_KEY,
-          time,
-          fn: debounce(crawlBlocklet),
-          options: { runOnInit: false, ...options },
-        },
-      ],
-      onError: (err: Error) => {
-        console.error('run job failed', err);
-      },
-    });
-  }
-
-  return cronCrawlBlockletJob;
-};
-
-export const cancelCronCrawlBlocklet = () => {
-  if (cronCrawlBlockletJob) {
-    cronCrawlBlockletJob.jobs[CRON_CRAWL_BLOCKLET_KEY].stop();
-    cronCrawlBlockletJob = null;
-    logger.info('Cron crawl blocklet stop, clear crawl queue');
-  }
-};

package/src/cache.ts

@@ -1,117 +0,0 @@
-import { createPool } from 'generic-pool';
-import { createClient } from 'redis';
-
-import { config, logger } from './config';
-
-const cacheKeyPrefix = process.env?.BLOCKLET_REAL_DID ? `${process.env.BLOCKLET_REAL_DID}:` : '';
-const MAX_REDIS_RETRY = 3;
-const ttl = 1000 * 60 * 60 * 24 * 7;
-
-export const cachePool = createPool(
-  {
-    create: async () => {
-      try {
-        const { redisUrl } = config;
-        const redisClient = createClient({
-          url: redisUrl,
-          socket: {
-            // @ts-ignore
-            reconnectStrategy: (retries) => {
-              if (retries >= MAX_REDIS_RETRY) {
-                return new Error('Retry Time Exhausted');
-              }
-              return Math.min(retries * 500, 1000 * 3);
-            },
-          },
-        });
-
-        redisClient.on('error', (err) => logger.warn('Redis Client Error:', err));
-        await redisClient.connect();
-        logger.info(`Successfully connected to Redis: ${redisUrl}`);
-
-        return redisClient;
-      } catch (error) {
-        logger.warn('Redis connection failed', error);
-        return null;
-      }
-    },
-    destroy: async (client: any) => {
-      // if is redis client
-      if (client.isReady) {
-        await client.quit();
-      }
-    },
-  },
-  {
-    max: 2, // 2 clients
-    min: 0,
-    // evictionRunIntervalMillis: 0,
-  },
-);
-
-export const memoryPool = createPool(
-  {
-    create: () => {
-      const map = new Map<string, any>();
-      // @ts-ignore
-      map.del = map.delete;
-      return Promise.resolve(map);
-    },
-    destroy: (client: Map<string, any>) => {
-      client.clear();
-      return Promise.resolve();
-    },
-  },
-  {
-    max: 10,
-    min: 0,
-  },
-);
-
-export const withCache = async (cb: Function) => {
-  const pool = config.redisUrl ? cachePool : memoryPool;
-  const client = await pool.acquire();
-
-  if (client) {
-    try {
-      return cb(client);
-    } finally {
-      // release client to pool, let other use
-      await pool.release(client);
-    }
-  }
-};
-
-export const formatKey = (key: string) => {
-  return `${cacheKeyPrefix}${key}`;
-};
-
-export const useCache = {
-  get: (key: string) => {
-    return withCache(async (client: any) => {
-      const value = await client.get(formatKey(key));
-      try {
-        return JSON.parse(value);
-      } catch (error) {
-        // ignore error
-      }
-      return value;
-    });
-  },
-  set: (key: string, value: any, options?: any) => {
-    return withCache((client: any) => {
-      const formatValue = typeof value === 'string' ? value : JSON.stringify(value);
-      return client.set(formatKey(key), formatValue, { PX: ttl, ...options });
-    });
-  },
-  remove: (key: string) => {
-    return withCache((client: any) => {
-      return client.del(formatKey(key));
-    });
-  },
-  list: (key: string = '*') => {
-    return withCache((client: any) => {
-      return client.keys(formatKey(key));
-    });
-  },
-};

package/src/config.ts

@@ -1,13 +0,0 @@
-import createLogger from '@blocklet/logger';
-
-export const logger = createLogger('crawler', { level: process.env.LOG_LEVEL || 'info' });
-
-export const config = {
-  redisUrl: process.env.REDIS_URL!,
-  dataDir: process.env.BLOCKLET_DATA_DIR!,
-  appDir: process.env.BLOCKLET_APP_DIR! || process.cwd(),
-  appUrl: process.env.BLOCKLET_APP_URL!,
-  puppeteerPath: process.env.PUPPETEER_EXECUTABLE_PATH!,
-  cacheDir: process.env.BLOCKLET_CACHE_DIR!,
-  testOnInitialize: process.env.NODE_ENV === 'production',
-};