@arcblock/crawler 1.0.0

package/package.json

@@ -0,0 +1,110 @@
+{
+  "name": "@arcblock/crawler",
+  "version": "1.0.0",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "publishConfig": {
+    "access": "public"
+  },
+  "lint-staged": {
+    "*.{mjs,js,jsx,ts,tsx}": [
+      "prettier --write",
+      "eslint"
+    ],
+    "*.{css,less,scss,json,graphql}": [
+      "prettier --write"
+    ]
+  },
+  "browserslist": {
+    "production": [
+      ">0.2%",
+      "not dead",
+      "not op_mini all"
+    ],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  },
+  "dependencies": {
+    "@abtnode/cron": "^1.16.43",
+    "@abtnode/models": "^1.16.43",
+    "@abtnode/queue": "^1.16.43",
+    "@arcblock/did-auth": "^1.19.15",
+    "@arcblock/did-auth-storage-nedb": "^1.7.1",
+    "@blocklet/logger": "^1.16.43",
+    "@blocklet/puppeteer": "^22.11.3",
+    "@blocklet/sdk": "^1.16.43",
+    "@ocap/client": "^1.19.15",
+    "@ocap/mcrypto": "^1.19.15",
+    "@ocap/util": "^1.20.11",
+    "@ocap/wallet": "^1.19.15",
+    "@sequelize/core": "7.0.0-alpha.46",
+    "@sequelize/sqlite3": "7.0.0-alpha.46",
+    "axios": "^1.7.9",
+    "cookie-parser": "^1.4.7",
+    "cors": "^2.8.5",
+    "dotenv-flow": "^4.1.0",
+    "express": "^4.21.2",
+    "express-async-errors": "^3.1.1",
+    "fs-extra": "^11.2.0",
+    "generic-pool": "^3.9.0",
+    "lodash": "^4.17.21",
+    "lru-cache": "^10.4.3",
+    "p-queue": "6.6.2",
+    "p-wait-for": "^5.0.2",
+    "redis": "^4.7.0",
+    "robots-parser": "^3.0.1",
+    "sequelize": "^6.37.7",
+    "sitemap": "^7.1.2",
+    "sqlite3": "^5.1.7",
+    "ufo": "^1.5.4",
+    "url-join": "^4.0.1"
+  },
+  "devDependencies": {
+    "@blocklet/js-sdk": "^1.16.39",
+    "@types/cookie-parser": "^1.4.8",
+    "@types/cors": "^2.8.17",
+    "@types/dotenv-flow": "^3.3.3",
+    "@types/express": "^4.17.21",
+    "@types/fs-extra": "^11.0.4",
+    "@types/lodash": "^4.17.16",
+    "@types/node": "^20.17.19",
+    "@types/react": "^18.3.18",
+    "@types/react-dom": "^18.3.5",
+    "@vitejs/plugin-react": "^4.3.4",
+    "bumpp": "^9.11.1",
+    "nodemon": "^3.1.9",
+    "npm-run-all": "^4.1.5",
+    "puppeteer": "^24.8.2",
+    "react": "~18.2.0",
+    "react-dom": "~18.2.0",
+    "react-router-dom": "^6.29.0",
+    "rimraf": "^5.0.10",
+    "tsx": "^4.19.3",
+    "vite": "^5.4.14",
+    "vite-plugin-blocklet": "^0.9.32",
+    "vite-plugin-svgr": "^4.3.0",
+    "zx": "^8.3.2"
+  },
+  "importSort": {
+    ".js, .jsx, .mjs": {
+      "parser": "babylon",
+      "style": "module"
+    },
+    ".ts, .tsx": {
+      "style": "module",
+      "parser": "typescript"
+    }
+  },
+  "simple-git-hooks": {
+    "pre-commit": "npx lint-staged"
+  },
+  "scripts": {
+    "dev": "tsc --watch",
+    "lint": "tsc --noEmit && eslint src --ext .mjs,.js,.jsx,.ts,.tsx",
+    "lint:fix": "npm run lint -- --fix",
+    "bundle": "tsc"
+  }
+}

package/src/blocklet.ts

@@ -0,0 +1,223 @@
+import Cron from '@abtnode/cron';
+import { components } from '@blocklet/sdk/lib/config';
+import debounce from 'lodash/debounce';
+import { joinURL } from 'ufo';
+
+import { useCache } from './cache';
+import { config, logger } from './config';
+import { createCrawlJob } from './crawler';
+import { closeBrowser, getBrowser } from './puppeteer';
+import { getComponentInfo, getRelativePath, getSitemapList } from './utils';
+
+// record crawl blocklet running
+const crawlBlockletRunningMap = new Map();
+
+// crawl blocklet sitemap urls
+export const crawlBlocklet = async () => {
+  // @ts-ignore
+  const { mountPoint, did } = getComponentInfo();
+
+  if (crawlBlockletRunningMap.has(did) && crawlBlockletRunningMap.get(did)) {
+    logger.info(`Crawler blocklet ${did} is running, skip it`);
+    return;
+  }
+
+  // check has browser can use
+  try {
+    const browser = await getBrowser();
+    if (!browser) {
+      throw new Error('No Browser can use');
+    }
+    logger.info('Crawler blocklet existing can use browser');
+  } catch (error: any) {
+    logger.info(`Crawler blocklet abort by error: ${error?.message || error?.reason || error}`);
+    return;
+  }
+
+  const { appUrl } = config;
+
+  if (!appUrl) {
+    throw new Error('appUrl not found');
+  }
+
+  const sitemapList = await getSitemapList(appUrl);
+
+  const matchMountPoint = joinURL(appUrl, !mountPoint || mountPoint === '/' ? '' : mountPoint);
+  const otherMountPointList = components
+    .filter((item) => item.mountPoint && item.mountPoint !== mountPoint)
+    .map((item) => item.mountPoint);
+
+  // get can use loc
+  const blockletLocList = sitemapList.filter((item: any) => {
+    if (mountPoint !== '/') {
+      return item?.url?.indexOf(matchMountPoint) > -1;
+    }
+    // if mountPoint is /, skip other mountPoint
+    return otherMountPointList.every((mountPoint) => item?.url?.indexOf(mountPoint) === -1);
+  }) as [];
+
+  const canUseBlockletLocList = [] as string[];
+  const lastmodMap = new Map();
+  let skipBlockletLocTotal = 0;
+  let blockletLocTotal = 0;
+
+  await Promise.all(
+    blockletLocList.map(async (item: any) => {
+      let tempLocList: string[] = [];
+
+      if (item.url) {
+        tempLocList.push(item.url);
+      }
+
+      if (item?.links?.length > 0) {
+        tempLocList.push(...item.links.map((ytem: any) => ytem.url));
+      }
+
+      blockletLocTotal += tempLocList.length;
+
+      // @ts-ignore
+      tempLocList = (
+        await Promise.all(
+          tempLocList.map(async (loc) => {
+            try {
+              const { lastModified: cacheLastModified } = await useCache.get(getRelativePath(loc));
+
+              // sitemap item lastmod is same as cache lastModified, skip it
+              if (
+                item.lastmod &&
+                cacheLastModified &&
+                new Date(cacheLastModified).getTime() === new Date(item.lastmod).getTime()
+              ) {
+                skipBlockletLocTotal++;
+                return false;
+              }
+
+              return loc;
+            } catch (error) {
+              // ignore error
+            }
+
+            // if can not get cache, return loc
+            return loc;
+          }),
+        )
+      ).filter(Boolean);
+
+      tempLocList.forEach((loc) => {
+        if (item.lastmod) lastmodMap.set(loc, item.lastmod);
+      });
+
+      canUseBlockletLocList.push(...tempLocList);
+    }),
+  );
+
+  const crawlerLogText = (step = '') => [
+    `Crawler sitemap.xml about ${did} ${step}: `,
+    {
+      blockletLocTotal,
+      canUseBlockletLocTotal: canUseBlockletLocList.length,
+      skipBlockletLocTotal,
+      lastmodMapTotal: lastmodMap.size,
+    },
+  ];
+
+  logger.info(...crawlerLogText('start'));
+
+  try {
+    // record crawl blocklet running
+    crawlBlockletRunningMap.set(did, true);
+
+    await createCrawlJob({
+      // @ts-ignore
+      urls: canUseBlockletLocList,
+      saveToRedis: true,
+      lastmodMap,
+      // formatPageContent: async ({ page }: { page: any; url: string; lastmod?: string }) => {
+      //   const pageContent = await page.evaluate(() => {
+      //     const removeElements = (tagName: string) => {
+      //       const elements = document.querySelectorAll(tagName);
+      //       for (let i = elements.length - 1; i >= 0; i--) {
+      //         try {
+      //           elements[i]?.parentNode?.removeChild(elements[i] as Node);
+      //         } catch (error) {
+      //           // do noting
+      //         }
+      //       }
+      //     };
+
+      //     // remove script, style, link, noscript
+      //     // removeElements('script');
+      //     // removeElements('style');
+      //     // removeElements('link');
+      //     // removeElements('noscript');
+
+      //     // remove uploader
+      //     removeElements('[id="uploader-container"]');
+      //     removeElements('[class^="uppy-"]');
+
+      //     // remove point up component
+      //     removeElements('[id="point-up-component"]');
+
+      //     // add meta tag to record crawler
+      //     const meta = document.createElement('meta');
+      //     meta.name = 'blocklet-crawler';
+      //     meta.content = 'true';
+      //     document.head.appendChild(meta);
+
+      //     return document.documentElement.outerHTML;
+      //   });
+
+      //   return pageContent;
+      // },
+    });
+
+    logger.info(...crawlerLogText('success'));
+
+    await closeBrowser({
+      trimCache: true,
+    });
+  } catch (error) {
+    logger.info('Crawler blocklet abort by error', error);
+  } finally {
+    // delete crawl blocklet running
+    crawlBlockletRunningMap.delete(did);
+  }
+};
+
+const CRON_CRAWL_BLOCKLET_KEY = 'cron-crawl-blocklet';
+let cronCrawlBlockletJob = null as any;
+
+// init cron crawl blocklet
+export const initCronCrawlBlocklet = (
+  {
+    time = '0 0 */12 * * *', // every 12 hours
+    options,
+  } = {} as { time: string; options: any },
+) => {
+  if (!cronCrawlBlockletJob) {
+    cronCrawlBlockletJob = Cron.init({
+      context: {},
+      jobs: [
+        {
+          name: CRON_CRAWL_BLOCKLET_KEY,
+          time,
+          fn: debounce(crawlBlocklet),
+          options: { runOnInit: false, ...options },
+        },
+      ],
+      onError: (err: Error) => {
+        console.error('run job failed', err);
+      },
+    });
+  }
+
+  return cronCrawlBlockletJob;
+};
+
+export const cancelCronCrawlBlocklet = () => {
+  if (cronCrawlBlockletJob) {
+    cronCrawlBlockletJob.jobs[CRON_CRAWL_BLOCKLET_KEY].stop();
+    cronCrawlBlockletJob = null;
+    logger.info('Cron crawl blocklet stop, clear crawl queue');
+  }
+};

package/src/cache.ts

@@ -0,0 +1,117 @@
+import { createPool } from 'generic-pool';
+import { createClient } from 'redis';
+
+import { config, logger } from './config';
+
+const cacheKeyPrefix = process.env?.BLOCKLET_REAL_DID ? `${process.env.BLOCKLET_REAL_DID}:` : '';
+const MAX_REDIS_RETRY = 3;
+const ttl = 1000 * 60 * 60 * 24 * 7;
+
+export const cachePool = createPool(
+  {
+    create: async () => {
+      try {
+        const { redisUrl } = config;
+        const redisClient = createClient({
+          url: redisUrl,
+          socket: {
+            // @ts-ignore
+            reconnectStrategy: (retries) => {
+              if (retries >= MAX_REDIS_RETRY) {
+                return new Error('Retry Time Exhausted');
+              }
+              return Math.min(retries * 500, 1000 * 3);
+            },
+          },
+        });
+
+        redisClient.on('error', (err) => logger.warn('Redis Client Error:', err));
+        await redisClient.connect();
+        logger.info(`Successfully connected to Redis: ${redisUrl}`);
+
+        return redisClient;
+      } catch (error) {
+        logger.warn('Redis connection failed', error);
+        return null;
+      }
+    },
+    destroy: async (client: any) => {
+      // if is redis client
+      if (client.isReady) {
+        await client.quit();
+      }
+    },
+  },
+  {
+    max: 2, // 2 clients
+    min: 0,
+    // evictionRunIntervalMillis: 0,
+  },
+);
+
+export const memoryPool = createPool(
+  {
+    create: () => {
+      const map = new Map<string, any>();
+      // @ts-ignore
+      map.del = map.delete;
+      return Promise.resolve(map);
+    },
+    destroy: (client: Map<string, any>) => {
+      client.clear();
+      return Promise.resolve();
+    },
+  },
+  {
+    max: 10,
+    min: 0,
+  },
+);
+
+export const withCache = async (cb: Function) => {
+  const pool = config.redisUrl ? cachePool : memoryPool;
+  const client = await pool.acquire();
+
+  if (client) {
+    try {
+      return cb(client);
+    } finally {
+      // release client to pool, let other use
+      await pool.release(client);
+    }
+  }
+};
+
+export const formatKey = (key: string) => {
+  return `${cacheKeyPrefix}${key}`;
+};
+
+export const useCache = {
+  get: (key: string) => {
+    return withCache(async (client: any) => {
+      const value = await client.get(formatKey(key));
+      try {
+        return JSON.parse(value);
+      } catch (error) {
+        // ignore error
+      }
+      return value;
+    });
+  },
+  set: (key: string, value: any, options?: any) => {
+    return withCache((client: any) => {
+      const formatValue = typeof value === 'string' ? value : JSON.stringify(value);
+      return client.set(formatKey(key), formatValue, { PX: ttl, ...options });
+    });
+  },
+  remove: (key: string) => {
+    return withCache((client: any) => {
+      return client.del(formatKey(key));
+    });
+  },
+  list: (key: string = '*') => {
+    return withCache((client: any) => {
+      return client.keys(formatKey(key));
+    });
+  },
+};

package/src/config.ts

@@ -0,0 +1,13 @@
+import createLogger from '@blocklet/logger';
+
+export const logger = createLogger('crawler', { level: process.env.LOG_LEVEL || 'info' });
+
+export const config = {
+  redisUrl: process.env.REDIS_URL!,
+  dataDir: process.env.BLOCKLET_DATA_DIR!,
+  appDir: process.env.BLOCKLET_APP_DIR! || process.cwd(),
+  appUrl: process.env.BLOCKLET_APP_URL!,
+  puppeteerPath: process.env.PUPPETEER_EXECUTABLE_PATH!,
+  cacheDir: process.env.BLOCKLET_CACHE_DIR!,
+  testOnInitialize: process.env.NODE_ENV === 'production',
+};