@lobehub/chat 1.66.5 → 1.67.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.66.5",
+  "version": "1.67.0",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",
@@ -25,6 +25,9 @@
   "license": "MIT",
   "author": "LobeHub <i@lobehub.com>",
   "sideEffects": false,
+  "workspaces": [
+    "packages/*"
+  ],
   "scripts": {
     "build": "next build",
     "build:analyze": "ANALYZE=true next build",
@@ -105,7 +108,7 @@
     "@ant-design/icons": "^5.5.2",
     "@ant-design/pro-components": "^2.8.3",
     "@anthropic-ai/sdk": "^0.37.0",
-    "@auth/core": "^0.37.4",
+    "@auth/core": "^0.38.0",
     "@aws-sdk/client-bedrock-runtime": "^3.723.0",
     "@aws-sdk/client-s3": "^3.723.0",
     "@aws-sdk/s3-request-presigner": "^3.723.0",
@@ -124,6 +127,7 @@
     "@icons-pack/react-simple-icons": "9.6.0",
     "@khmyznikov/pwa-install": "0.3.9",
     "@langchain/community": "^0.3.22",
+    "@lobechat/web-crawler": "workspace:*",
     "@lobehub/charts": "^1.12.0",
     "@lobehub/chat-plugin-sdk": "^1.32.4",
     "@lobehub/chat-plugins-gateway": "^1.9.0",
@@ -156,7 +160,7 @@
     "debug": "^4.4.0",
     "dexie": "^3.2.7",
     "diff": "^7.0.0",
-    "drizzle-orm": "^0.39.0",
+    "drizzle-orm": "^0.40.0",
     "drizzle-zod": "^0.5.1",
     "fast-deep-equal": "^3.1.3",
     "file-type": "^20.0.0",
@@ -174,7 +178,7 @@
     "langfuse": "3.29.1",
     "langfuse-core": "3.29.1",
     "lodash-es": "^4.17.21",
-    "lucide-react": "^0.475.0",
+    "lucide-react": "^0.477.0",
     "mammoth": "^1.9.0",
     "mdast-util-to-markdown": "^2.1.2",
     "modern-screenshot": "^4.5.5",
@@ -210,7 +214,7 @@
     "react-lazy-load": "^4.0.1",
     "react-pdf": "^9.2.1",
     "react-rnd": "^10.4.14",
-    "react-scan": "^0.1.0",
+    "react-scan": "^0.2.0",
     "react-virtuoso": "^4.12.3",
     "react-wrap-balancer": "^1.1.1",
     "remark": "^15.0.1",
@@ -320,7 +324,7 @@
     "vitest": "~1.2.2",
     "vitest-canvas-mock": "^0.3.3"
   },
-  "packageManager": "pnpm@9.15.5",
+  "packageManager": "pnpm@9.15.6",
   "publishConfig": {
     "access": "public",
     "registry": "https://registry.npmjs.org"

package/packages/web-crawler/README.md

@@ -0,0 +1,34 @@
+# @lobechat/web-crawler
+
+LobeChat 内置的网页抓取模块，用于从网页中提取结构化内容，并转换为 Markdown 格式。
+
+## 📝 简介
+
+`@lobechat/web-crawler` 是 LobeChat 项目的内部组件，专门负责网页内容的抓取和处理。它能够智能地从各种网页中提取有意义的内容，剔除广告、导航栏等干扰元素，并将结果转换为结构良好的 Markdown 文本。
+
+## 🔍 主要功能
+
+- **网页内容抓取**：支持从各类网站获取原始 HTML 内容
+- **智能内容提取**：使用 Mozilla 的 Readability 算法识别页面中的主要内容
+- **降级处理机制**：当标准抓取失败时，自动切换到 Browserless.io 服务进行渲染抓取（需要自行配置环境变量）
+- **Markdown 转换**：将提取的 HTML 内容转换为易于 AI 处理的 Markdown 格式
+
+## 🛠️ 技术实现
+
+该模块主要依赖以下技术：
+
+- **@mozilla/readability**：提供了强大的内容提取算法
+- **happy-dom**：轻量级的服务端 DOM 实现
+- **node-html-markdown**：高效的 HTML 到 Markdown 转换工具
+
+## 🤝 共建改进
+
+由于网页结构的多样性和复杂性，内容提取可能会遇到各种挑战。如果您发现某些网站的抓取效果不佳，欢迎通过以下方式参与改进：
+
+1. 提交具体的问题网址和期望的输出结果
+2. 分享您对特定网站类型的处理经验
+3. 提出针对性的算法或配置调整建议
+
+## 📌 注意事项
+
+这是 LobeHub 的内部模块（`"private": true`），不作为独立包发布使用。它专为 LobeChat 的特定需求设计，与其他系统组件紧密集成。

package/packages/web-crawler/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@lobechat/web-crawler",
+  "version": "1.0.0",
+  "private": true,
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "dependencies": {
+    "@mozilla/readability": "^0.5.0",
+    "happy-dom": "^17.0.0",
+    "node-html-markdown": "^1.3.0",
+    "query-string": "^9.1.1"
+  }
+}

package/packages/web-crawler/src/crawImpl/browserless.ts

@@ -0,0 +1,62 @@
+import qs from 'query-string';
+
+import { CrawlImpl, CrawlSuccessResult } from '../type';
+import { htmlToMarkdown } from '../utils/htmlToMarkdown';
+
+const BASE_URL = process.env.BROWSERLESS_URL ?? 'https://chrome.browserless.io';
+const BROWSERLESS_TOKEN = process.env.BROWSERLESS_TOKEN;
+
+class BrowserlessInitError extends Error {
+  constructor() {
+    super('`BROWSERLESS_URL` or `BROWSERLESS_TOKEN` are required');
+    this.name = 'BrowserlessInitError';
+  }
+}
+
+export const browserless: CrawlImpl = async (url, { filterOptions }) => {
+  if (!process.env.BROWSERLESS_URL && !process.env.BROWSERLESS_TOKEN) {
+    throw new BrowserlessInitError();
+  }
+
+  const input = {
+    gotoOptions: { waitUntil: 'networkidle2' },
+    url,
+  };
+
+  try {
+    const res = await fetch(
+      qs.stringifyUrl({ query: { token: BROWSERLESS_TOKEN }, url: `${BASE_URL}/content` }),
+      {
+        body: JSON.stringify(input),
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        method: 'POST',
+      },
+    );
+    const html = await res.text();
+
+    const result = htmlToMarkdown(html, { filterOptions, url });
+
+    if (
+      !!result.content &&
+      result.title &&
+      // Just a moment... 说明被 CF 拦截了
+      result.title.trim() !== 'Just a moment...'
+    ) {
+      return {
+        content: result.content,
+        contentType: 'text',
+        description: result?.excerpt,
+        length: result.length,
+        siteName: result?.siteName,
+        title: result?.title,
+        url,
+      } satisfies CrawlSuccessResult;
+    }
+  } catch (error) {
+    console.error(error);
+  }
+
+  return;
+};

package/packages/web-crawler/src/crawImpl/index.ts

@@ -0,0 +1,11 @@
+import { browserless } from './browserless';
+import { jina } from './jina';
+import { naive } from './naive';
+
+export const crawlImpls = {
+  browserless,
+  jina,
+  naive,
+};
+
+export type CrawlImplType = keyof typeof crawlImpls;

package/packages/web-crawler/src/crawImpl/jina.ts

@@ -0,0 +1,37 @@
+import { CrawlImpl } from '../type';
+
+export const jina: CrawlImpl<{ apiKey?: string }> = async (url, params) => {
+  const token = params.apiKey ?? process.env.JINA_API_KEY;
+
+  try {
+    const res = await fetch(`https://r.jina.ai/${url}`, {
+      headers: {
+        'Accept': 'application/json',
+        'Authorization': token ? `Bearer ${token}` : '',
+        'x-send-from': 'LobeChat Community',
+      },
+    });
+
+    if (res.ok) {
+      const json = await res.json();
+      if (json.code === 200) {
+        const result = json.data;
+        return {
+          content: result.content,
+          contentType: 'text',
+          description: result?.description,
+          length: result.content.length,
+          siteName: result?.siteName,
+          title: result?.title,
+          url: url,
+        };
+      }
+
+      throw json;
+    }
+  } catch (error) {
+    console.error(error);
+  }
+
+  return;
+};

package/packages/web-crawler/src/crawImpl/naive.ts

@@ -0,0 +1,84 @@
+import { CrawlImpl, CrawlSuccessResult } from '../type';
+import { NetworkConnectionError, PageNotFoundError } from '../utils/errorType';
+import { htmlToMarkdown } from '../utils/htmlToMarkdown';
+
+const mixinHeaders = {
+  // 接受的内容类型
+  'Accept':
+    'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+  // 接受的编码方式
+  'Accept-Encoding': 'gzip, deflate, br',
+  // 接受的语言
+  'Accept-Language': 'en-US,en;q=0.9,zh;q=0.8',
+  // 缓存控制
+  'Cache-Control': 'max-age=0',
+  // 连接类型
+  'Connection': 'keep-alive',
+  // 表明请求来自哪个站点
+  'Referer': 'https://www.google.com/',
+  // 升级不安全请求
+  'Upgrade-Insecure-Requests': '1',
+  // 模拟真实浏览器的 User-Agent
+  'User-Agent':
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
+  // 防止跨站请求伪造
+  'sec-ch-ua': '"Google Chrome";v="121", "Not A(Brand";v="99", "Chromium";v="121"',
+  'sec-ch-ua-mobile': '?0',
+  'sec-ch-ua-platform': '"Windows"',
+  'sec-fetch-dest': 'document',
+  'sec-fetch-mode': 'navigate',
+  'sec-fetch-site': 'none',
+  'sec-fetch-user': '?1',
+};
+
+export const naive: CrawlImpl = async (url, { filterOptions }) => {
+  let res: Response;
+
+  try {
+    res = await fetch(url, { headers: mixinHeaders });
+  } catch (e) {
+    if ((e as Error).message === 'fetch failed') {
+      throw new NetworkConnectionError();
+    }
+    throw e;
+  }
+
+  if (res.status === 404) {
+    throw new PageNotFoundError(res.statusText);
+  }
+  const type = res.headers.get('content-type');
+
+  if (type?.includes('application/json')) {
+    const json = await res.json();
+    return {
+      content: JSON.stringify(json, null, 2),
+      contentType: 'json',
+      length: json.length,
+      url,
+    } satisfies CrawlSuccessResult;
+  }
+
+  try {
+    const html = await res.text();
+
+    const result = htmlToMarkdown(html, { filterOptions, url });
+
+    // if the content is not empty or blocked
+    // just return
+    if (!!result.content && result.title !== 'Just a moment...') {
+      return {
+        content: result.content,
+        contentType: 'text',
+        description: result?.excerpt,
+        length: result.length,
+        siteName: result?.siteName,
+        title: result?.title,
+        url,
+      } satisfies CrawlSuccessResult;
+    }
+  } catch (error) {
+    console.error(error);
+  }
+
+  return;
+};

package/packages/web-crawler/src/crawler.ts

@@ -0,0 +1,66 @@
+import { CrawlImplType, crawlImpls } from './crawImpl';
+import { CrawlUrlRule } from './type';
+import { crawUrlRules } from './urlRules';
+import { applyUrlRules } from './utils/appUrlRules';
+
+export class Crawler {
+  impls = ['naive', 'jina', 'browserless'] as const;
+
+  /**
+   * 爬取网页内容
+   * @param options 爬取选项
+   */
+  async crawl({
+    url,
+    impls,
+    filterOptions: userFilterOptions,
+  }: {
+    filterOptions?: CrawlUrlRule['filterOptions'];
+    impls?: string[];
+    url: string;
+  }) {
+    // 应用URL规则
+    const { transformedUrl, filterOptions: ruleFilterOptions } = applyUrlRules(url, crawUrlRules);
+
+    // 合并用户提供的过滤选项和规则中的过滤选项，用户选项优先
+    const mergedFilterOptions = {
+      ...ruleFilterOptions,
+      ...userFilterOptions,
+    };
+
+    let finalError: Error | undefined;
+
+    const finalImpls = impls
+      ? (impls.filter((impl) => Object.keys(crawlImpls).includes(impl)) as CrawlImplType[])
+      : this.impls;
+
+    //   按照内置的实现顺序依次尝试
+    for (const impl of finalImpls) {
+      try {
+        const res = await crawlImpls[impl](transformedUrl, { filterOptions: mergedFilterOptions });
+
+        if (res)
+          return {
+            crawler: impl,
+            data: res,
+            originalUrl: url,
+            transformedUrl: transformedUrl !== url ? transformedUrl : undefined,
+          };
+      } catch (error) {
+        console.error(error);
+        finalError = error as Error;
+      }
+    }
+
+    const errorType = finalError?.name || 'UnknownError';
+    const errorMessage = finalError?.message;
+
+    return {
+      content: `Fail to crawl the page. Error type: ${errorType}, error message: ${errorMessage}`,
+      errorMessage: errorMessage,
+      errorType,
+      originalUrl: url,
+      transformedUrl: transformedUrl !== url ? transformedUrl : undefined,
+    };
+  }
+}

package/packages/web-crawler/src/index.ts

@@ -0,0 +1,2 @@
+export { Crawler } from './crawler';
+export * from './type';

package/packages/web-crawler/src/type.ts

@@ -0,0 +1,42 @@
+export interface CrawlSuccessResult {
+  content?: string;
+  contentType: 'text' | 'json';
+  description?: string;
+  length?: number;
+  siteName?: string;
+  title?: string;
+  url: string;
+}
+
+export interface CrawlErrorResult {
+  content: string;
+  errorMessage: string;
+  url: string;
+}
+
+export interface FilterOptions {
+  // 是否启用Readability
+  enableReadability?: boolean;
+
+  pureText?: boolean;
+}
+
+type CrawlImplParams<T> = T & {
+  filterOptions: FilterOptions;
+};
+
+export type CrawlImpl<Params = object> = (
+  url: string,
+  params: CrawlImplParams<Params>,
+) => Promise<CrawlSuccessResult | undefined>;
+
+export interface CrawlUrlRule {
+  // 内容过滤配置（可选）
+  filterOptions?: FilterOptions;
+  // 是否使用正则表达式匹配（默认为glob模式）
+  isRegex?: boolean;
+  // URL匹配模式，支持glob模式或正则表达式
+  urlPattern: string;
+  // URL转换模板（可选），如果提供则进行URL转换
+  urlTransform?: string;
+}

package/packages/web-crawler/src/urlRules.ts

@@ -0,0 +1,34 @@
+import { CrawlUrlRule } from './type';
+
+export const crawUrlRules: CrawlUrlRule[] = [
+  // github 源码解析
+  {
+    filterOptions: {
+      enableReadability: false,
+    },
+    urlPattern: 'https://github.com/([^/]+)/([^/]+)/blob/([^/]+)/(.*)',
+    urlTransform: 'https://github.com/$1/$2/raw/refs/heads/$3/$4',
+  },
+  {
+    filterOptions: {
+      enableReadability: false,
+    },
+    // GitHub discussion
+    urlPattern: 'https://github.com/(.*)/discussions/(.*)',
+  },
+  {
+    // Medium 文章转换为 Scribe.rip
+    urlPattern: 'https://medium.com/(.*)',
+    urlTransform: 'https://scribe.rip/$1',
+  },
+
+  // 体育数据网站规则
+  {
+    filterOptions: {
+      // 对体育数据表格禁用 Readability 并且转换为纯文本
+      enableReadability: false,
+      pureText: true,
+    },
+    urlPattern: 'https://www.qiumiwu.com/standings/(.*)',
+  },
+];