@lobehub/chat 1.66.6 → 1.67.0

package/packages/web-crawler/src/crawImpl/browserless.ts

@@ -0,0 +1,62 @@
+import qs from 'query-string';
+
+import { CrawlImpl, CrawlSuccessResult } from '../type';
+import { htmlToMarkdown } from '../utils/htmlToMarkdown';
+
+const BASE_URL = process.env.BROWSERLESS_URL ?? 'https://chrome.browserless.io';
+const BROWSERLESS_TOKEN = process.env.BROWSERLESS_TOKEN;
+
+class BrowserlessInitError extends Error {
+  constructor() {
+    super('`BROWSERLESS_URL` or `BROWSERLESS_TOKEN` are required');
+    this.name = 'BrowserlessInitError';
+  }
+}
+
+export const browserless: CrawlImpl = async (url, { filterOptions }) => {
+  if (!process.env.BROWSERLESS_URL && !process.env.BROWSERLESS_TOKEN) {
+    throw new BrowserlessInitError();
+  }
+
+  const input = {
+    gotoOptions: { waitUntil: 'networkidle2' },
+    url,
+  };
+
+  try {
+    const res = await fetch(
+      qs.stringifyUrl({ query: { token: BROWSERLESS_TOKEN }, url: `${BASE_URL}/content` }),
+      {
+        body: JSON.stringify(input),
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        method: 'POST',
+      },
+    );
+    const html = await res.text();
+
+    const result = htmlToMarkdown(html, { filterOptions, url });
+
+    if (
+      !!result.content &&
+      result.title &&
+      // Just a moment... 说明被 CF 拦截了
+      result.title.trim() !== 'Just a moment...'
+    ) {
+      return {
+        content: result.content,
+        contentType: 'text',
+        description: result?.excerpt,
+        length: result.length,
+        siteName: result?.siteName,
+        title: result?.title,
+        url,
+      } satisfies CrawlSuccessResult;
+    }
+  } catch (error) {
+    console.error(error);
+  }
+
+  return;
+};

package/packages/web-crawler/src/crawImpl/index.ts

@@ -0,0 +1,11 @@
+import { browserless } from './browserless';
+import { jina } from './jina';
+import { naive } from './naive';
+
+export const crawlImpls = {
+  browserless,
+  jina,
+  naive,
+};
+
+export type CrawlImplType = keyof typeof crawlImpls;

package/packages/web-crawler/src/crawImpl/jina.ts

@@ -0,0 +1,37 @@
+import { CrawlImpl } from '../type';
+
+export const jina: CrawlImpl<{ apiKey?: string }> = async (url, params) => {
+  const token = params.apiKey ?? process.env.JINA_API_KEY;
+
+  try {
+    const res = await fetch(`https://r.jina.ai/${url}`, {
+      headers: {
+        'Accept': 'application/json',
+        'Authorization': token ? `Bearer ${token}` : '',
+        'x-send-from': 'LobeChat Community',
+      },
+    });
+
+    if (res.ok) {
+      const json = await res.json();
+      if (json.code === 200) {
+        const result = json.data;
+        return {
+          content: result.content,
+          contentType: 'text',
+          description: result?.description,
+          length: result.content.length,
+          siteName: result?.siteName,
+          title: result?.title,
+          url: url,
+        };
+      }
+
+      throw json;
+    }
+  } catch (error) {
+    console.error(error);
+  }
+
+  return;
+};

package/packages/web-crawler/src/crawImpl/naive.ts

@@ -0,0 +1,84 @@
+import { CrawlImpl, CrawlSuccessResult } from '../type';
+import { NetworkConnectionError, PageNotFoundError } from '../utils/errorType';
+import { htmlToMarkdown } from '../utils/htmlToMarkdown';
+
+const mixinHeaders = {
+  // 接受的内容类型
+  'Accept':
+    'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+  // 接受的编码方式
+  'Accept-Encoding': 'gzip, deflate, br',
+  // 接受的语言
+  'Accept-Language': 'en-US,en;q=0.9,zh;q=0.8',
+  // 缓存控制
+  'Cache-Control': 'max-age=0',
+  // 连接类型
+  'Connection': 'keep-alive',
+  // 表明请求来自哪个站点
+  'Referer': 'https://www.google.com/',
+  // 升级不安全请求
+  'Upgrade-Insecure-Requests': '1',
+  // 模拟真实浏览器的 User-Agent
+  'User-Agent':
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
+  // 防止跨站请求伪造
+  'sec-ch-ua': '"Google Chrome";v="121", "Not A(Brand";v="99", "Chromium";v="121"',
+  'sec-ch-ua-mobile': '?0',
+  'sec-ch-ua-platform': '"Windows"',
+  'sec-fetch-dest': 'document',
+  'sec-fetch-mode': 'navigate',
+  'sec-fetch-site': 'none',
+  'sec-fetch-user': '?1',
+};
+
+export const naive: CrawlImpl = async (url, { filterOptions }) => {
+  let res: Response;
+
+  try {
+    res = await fetch(url, { headers: mixinHeaders });
+  } catch (e) {
+    if ((e as Error).message === 'fetch failed') {
+      throw new NetworkConnectionError();
+    }
+    throw e;
+  }
+
+  if (res.status === 404) {
+    throw new PageNotFoundError(res.statusText);
+  }
+  const type = res.headers.get('content-type');
+
+  if (type?.includes('application/json')) {
+    const json = await res.json();
+    return {
+      content: JSON.stringify(json, null, 2),
+      contentType: 'json',
+      length: json.length,
+      url,
+    } satisfies CrawlSuccessResult;
+  }
+
+  try {
+    const html = await res.text();
+
+    const result = htmlToMarkdown(html, { filterOptions, url });
+
+    // if the content is not empty or blocked
+    // just return
+    if (!!result.content && result.title !== 'Just a moment...') {
+      return {
+        content: result.content,
+        contentType: 'text',
+        description: result?.excerpt,
+        length: result.length,
+        siteName: result?.siteName,
+        title: result?.title,
+        url,
+      } satisfies CrawlSuccessResult;
+    }
+  } catch (error) {
+    console.error(error);
+  }
+
+  return;
+};

package/packages/web-crawler/src/crawler.ts

@@ -0,0 +1,66 @@
+import { CrawlImplType, crawlImpls } from './crawImpl';
+import { CrawlUrlRule } from './type';
+import { crawUrlRules } from './urlRules';
+import { applyUrlRules } from './utils/appUrlRules';
+
+export class Crawler {
+  impls = ['naive', 'jina', 'browserless'] as const;
+
+  /**
+   * 爬取网页内容
+   * @param options 爬取选项
+   */
+  async crawl({
+    url,
+    impls,
+    filterOptions: userFilterOptions,
+  }: {
+    filterOptions?: CrawlUrlRule['filterOptions'];
+    impls?: string[];
+    url: string;
+  }) {
+    // 应用URL规则
+    const { transformedUrl, filterOptions: ruleFilterOptions } = applyUrlRules(url, crawUrlRules);
+
+    // 合并用户提供的过滤选项和规则中的过滤选项，用户选项优先
+    const mergedFilterOptions = {
+      ...ruleFilterOptions,
+      ...userFilterOptions,
+    };
+
+    let finalError: Error | undefined;
+
+    const finalImpls = impls
+      ? (impls.filter((impl) => Object.keys(crawlImpls).includes(impl)) as CrawlImplType[])
+      : this.impls;
+
+    //   按照内置的实现顺序依次尝试
+    for (const impl of finalImpls) {
+      try {
+        const res = await crawlImpls[impl](transformedUrl, { filterOptions: mergedFilterOptions });
+
+        if (res)
+          return {
+            crawler: impl,
+            data: res,
+            originalUrl: url,
+            transformedUrl: transformedUrl !== url ? transformedUrl : undefined,
+          };
+      } catch (error) {
+        console.error(error);
+        finalError = error as Error;
+      }
+    }
+
+    const errorType = finalError?.name || 'UnknownError';
+    const errorMessage = finalError?.message;
+
+    return {
+      content: `Fail to crawl the page. Error type: ${errorType}, error message: ${errorMessage}`,
+      errorMessage: errorMessage,
+      errorType,
+      originalUrl: url,
+      transformedUrl: transformedUrl !== url ? transformedUrl : undefined,
+    };
+  }
+}

package/packages/web-crawler/src/index.ts

@@ -0,0 +1,2 @@
+export { Crawler } from './crawler';
+export * from './type';

package/packages/web-crawler/src/type.ts

@@ -0,0 +1,42 @@
+export interface CrawlSuccessResult {
+  content?: string;
+  contentType: 'text' | 'json';
+  description?: string;
+  length?: number;
+  siteName?: string;
+  title?: string;
+  url: string;
+}
+
+export interface CrawlErrorResult {
+  content: string;
+  errorMessage: string;
+  url: string;
+}
+
+export interface FilterOptions {
+  // 是否启用Readability
+  enableReadability?: boolean;
+
+  pureText?: boolean;
+}
+
+type CrawlImplParams<T> = T & {
+  filterOptions: FilterOptions;
+};
+
+export type CrawlImpl<Params = object> = (
+  url: string,
+  params: CrawlImplParams<Params>,
+) => Promise<CrawlSuccessResult | undefined>;
+
+export interface CrawlUrlRule {
+  // 内容过滤配置（可选）
+  filterOptions?: FilterOptions;
+  // 是否使用正则表达式匹配（默认为glob模式）
+  isRegex?: boolean;
+  // URL匹配模式，支持glob模式或正则表达式
+  urlPattern: string;
+  // URL转换模板（可选），如果提供则进行URL转换
+  urlTransform?: string;
+}

package/packages/web-crawler/src/urlRules.ts

@@ -0,0 +1,34 @@
+import { CrawlUrlRule } from './type';
+
+export const crawUrlRules: CrawlUrlRule[] = [
+  // github 源码解析
+  {
+    filterOptions: {
+      enableReadability: false,
+    },
+    urlPattern: 'https://github.com/([^/]+)/([^/]+)/blob/([^/]+)/(.*)',
+    urlTransform: 'https://github.com/$1/$2/raw/refs/heads/$3/$4',
+  },
+  {
+    filterOptions: {
+      enableReadability: false,
+    },
+    // GitHub discussion
+    urlPattern: 'https://github.com/(.*)/discussions/(.*)',
+  },
+  {
+    // Medium 文章转换为 Scribe.rip
+    urlPattern: 'https://medium.com/(.*)',
+    urlTransform: 'https://scribe.rip/$1',
+  },
+
+  // 体育数据网站规则
+  {
+    filterOptions: {
+      // 对体育数据表格禁用 Readability 并且转换为纯文本
+      enableReadability: false,
+      pureText: true,
+    },
+    urlPattern: 'https://www.qiumiwu.com/standings/(.*)',
+  },
+];