@lobehub/chat 1.68.1 → 1.68.3

package/CHANGELOG.md

@@ -2,6 +2,56 @@
 
 # Changelog
 
+### [Version 1.68.3](https://github.com/lobehub/lobe-chat/compare/v1.68.2...v1.68.3)
+
+<sup>Released on **2025-03-03**</sup>
+
+#### 🐛 Bug Fixes
+
+- **misc**: Improve url rules.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's fixed
+
+- **misc**: Improve url rules, closes [#6669](https://github.com/lobehub/lobe-chat/issues/6669) ([5ee59e3](https://github.com/lobehub/lobe-chat/commit/5ee59e3))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
+### [Version 1.68.2](https://github.com/lobehub/lobe-chat/compare/v1.68.1...v1.68.2)
+
+<sup>Released on **2025-03-03**</sup>
+
+#### 💄 Styles
+
+- **misc**: Add build-in web search support for Wenxin & Hunyuan.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### Styles
+
+- **misc**: Add build-in web search support for Wenxin & Hunyuan, closes [#6617](https://github.com/lobehub/lobe-chat/issues/6617) ([dfd1f09](https://github.com/lobehub/lobe-chat/commit/dfd1f09))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 1.68.1](https://github.com/lobehub/lobe-chat/compare/v1.68.0...v1.68.1)
 
 <sup>Released on **2025-03-03**</sup>

package/changelog/v1.json

@@ -1,4 +1,22 @@
 [
+  {
+    "children": {
+      "fixes": [
+        "Improve url rules."
+      ]
+    },
+    "date": "2025-03-03",
+    "version": "1.68.3"
+  },
+  {
+    "children": {
+      "improvements": [
+        "Add build-in web search support for Wenxin & Hunyuan."
+      ]
+    },
+    "date": "2025-03-03",
+    "version": "1.68.2"
+  },
   {
     "children": {
       "fixes": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.68.1",
+  "version": "1.68.3",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",

package/packages/web-crawler/src/crawImpl/jina.ts

@@ -1,7 +1,7 @@
 import { CrawlImpl } from '../type';
 
 export const jina: CrawlImpl<{ apiKey?: string }> = async (url, params) => {
-  const token = params.apiKey ?? process.env.JINA_API_KEY;
+  const token = params.apiKey ?? process.env.JINA_READER_API_KEY ?? process.env.JINA_API_KEY;
 
   try {
     const res = await fetch(`https://r.jina.ai/${url}`, {

package/packages/web-crawler/src/urlRules.ts

@@ -37,7 +37,13 @@ export const crawUrlRules: CrawlUrlRule[] = [
     urlPattern: 'https://medium.com/(.*)',
     urlTransform: 'https://scribe.rip/$1',
   },
-
+  {
+    filterOptions: {
+      enableReadability: false,
+    },
+    impls: ['jina', 'browserless'],
+    urlPattern: 'https://(twitter.com|x.com)/(.*)',
+  },
   // 体育数据网站规则
   {
     filterOptions: {

package/src/config/aiModels/wenxin.ts

@@ -4,6 +4,7 @@ const wenxinChatModels: AIChatModelCard[] = [
   {
     abilities: {
       functionCall: true,
+      search: true,
     },
     contextWindowTokens: 8192,
     description:
@@ -16,11 +17,15 @@ const wenxinChatModels: AIChatModelCard[] = [
       input: 0.8,
       output: 2,
     },
+    settings: {
+      searchImpl: 'params',
+    },
     type: 'chat',
   },
   {
     abilities: {
       functionCall: true,
+      search: true,
     },
     contextWindowTokens: 8192,
     description:
@@ -32,11 +37,15 @@ const wenxinChatModels: AIChatModelCard[] = [
       input: 0.8,
       output: 2,
     },
+    settings: {
+      searchImpl: 'params',
+    },
     type: 'chat',
   },
   {
     abilities: {
       functionCall: true,
+      search: true,
     },
     contextWindowTokens: 128_000,
     description:
@@ -49,11 +58,15 @@ const wenxinChatModels: AIChatModelCard[] = [
       input: 0.8,
       output: 2,
     },
+    settings: {
+      searchImpl: 'params',
+    },
     type: 'chat',
   },
   {
     abilities: {
       functionCall: true,
+      search: true,
     },
     contextWindowTokens: 8192,
     description:
@@ -66,11 +79,15 @@ const wenxinChatModels: AIChatModelCard[] = [
       input: 30,
       output: 90,
     },
+    settings: {
+      searchImpl: 'params',
+    },
     type: 'chat',
   },
   {
     abilities: {
       functionCall: true,
+      search: true,
     },
     contextWindowTokens: 8192,
     description:
@@ -82,11 +99,15 @@ const wenxinChatModels: AIChatModelCard[] = [
       input: 30,
       output: 90,
     },
+    settings: {
+      searchImpl: 'params',
+    },
     type: 'chat',
   },
   {
     abilities: {
       functionCall: true,
+      search: true,
     },
     contextWindowTokens: 8192,
     description:
@@ -99,11 +120,15 @@ const wenxinChatModels: AIChatModelCard[] = [
       input: 20,
       output: 60,
     },
+    settings: {
+      searchImpl: 'params',
+    },
     type: 'chat',
   },
   {
     abilities: {
       functionCall: true,
+      search: true,
     },
     contextWindowTokens: 128_000,
     description:
@@ -116,11 +141,15 @@ const wenxinChatModels: AIChatModelCard[] = [
       input: 20,
       output: 60,
     },
+    settings: {
+      searchImpl: 'params',
+    },
     type: 'chat',
   },
   {
     abilities: {
       functionCall: true,
+      search: true,
     },
     contextWindowTokens: 8192,
     description:
@@ -132,6 +161,9 @@ const wenxinChatModels: AIChatModelCard[] = [
       input: 20,
       output: 60,
     },
+    settings: {
+      searchImpl: 'params',
+    },
     type: 'chat',
   },
   {

package/src/libs/agent-runtime/hunyuan/index.test.ts

@@ -1,7 +1,9 @@
 // @vitest-environment node
-import { ModelProvider } from '@/libs/agent-runtime';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { LobeOpenAICompatibleRuntime, ModelProvider } from '@/libs/agent-runtime';
+import { testProvider } from '@/libs/agent-runtime/providerTestUtils';
 
-import { testProvider } from '../providerTestUtils';
 import { LobeHunyuanAI } from './index';
 
 testProvider({
@@ -11,3 +13,136 @@ testProvider({
   chatDebugEnv: 'DEBUG_HUNYUAN_CHAT_COMPLETION',
   chatModel: 'hunyuan-lite',
 });
+
+// Mock the console.error to avoid polluting test output
+vi.spyOn(console, 'error').mockImplementation(() => {});
+
+let instance: LobeOpenAICompatibleRuntime;
+
+beforeEach(() => {
+  instance = new LobeHunyuanAI({ apiKey: 'test' });
+
+  // 使用 vi.spyOn 来模拟 chat.completions.create 方法
+  vi.spyOn(instance['client'].chat.completions, 'create').mockResolvedValue(
+    new ReadableStream() as any,
+  );
+});
+
+describe('LobeHunyuanAI', () => {
+  describe('chat', () => {
+    it('should with search citations', async () => {
+const data = [
+        {
+          id: "939fbdb8dbb9b4c5944cbbe687c977c2",
+          object: "chat.completion.chunk",
+          created: 1741000456,
+          model: "hunyuan-turbo",
+          system_fingerprint: "",
+          choices: [
+            {
+              index: 0,
+              delta: { role: "assistant", content: "为您" },
+              finish_reason: null
+            }
+          ],
+          note: "以上内容为AI生成，不代表开发者立场，请勿删除或修改本标记",
+          search_info: {
+            search_results: [
+              {
+                index: 1,
+                title: "公务员考试时政热点【2025年3月3日】_公务员考试网_华图教育",
+                url: "http://www.huatu.com/2025/0303/2803685.html",
+                icon: "https://hunyuan-img-1251316161.cos.ap-guangzhou.myqcloud.com/%2Fpublic/img/63ce96deffe0119827f12deaa5ffe7ef.jpg",
+                text: "华图教育官网"
+              },
+              {
+                index: 2,
+                title: "外交部新闻（2025年3月3日）",
+                url: "https://view.inews.qq.com/a/20250303A02NLC00?scene=qqsearch",
+                icon: "https://hunyuan-img-1251316161.cos.ap-guangzhou.myqcloud.com/%2Fpublic/img/00ce40298870d1accb7920d641152722.jpg",
+                text: "腾讯网"
+              }
+            ]
+          }
+        },
+        {
+          id: "939fbdb8dbb9b4c5944cbbe687c977c2",
+          object: "chat.completion.chunk",
+          created: 1741000456,
+          model: "hunyuan-turbo",
+          system_fingerprint: "",
+          choices: [
+            {
+              index: 0,
+              delta: { role: "assistant", content: "找到" },
+              finish_reason: null
+            }
+          ],
+          note: "以上内容为AI生成，不代表开发者立场，请勿删除或修改本标记",
+          search_info: {
+            search_results: [
+              {
+                index: 1,
+                title: "公务员考试时政热点【2025年3月3日】_公务员考试网_华图教育",
+                url: "http://www.huatu.com/2025/0303/2803685.html",
+                icon: "https://hunyuan-img-1251316161.cos.ap-guangzhou.myqcloud.com/%2Fpublic/img/63ce96deffe0119827f12deaa5ffe7ef.jpg",
+                text: "华图教育官网"
+              },
+              {
+                index: 2,
+                title: "外交部新闻（2025年3月3日）",
+                url: "https://view.inews.qq.com/a/20250303A02NLC00?scene=qqsearch",
+                icon: "https://hunyuan-img-1251316161.cos.ap-guangzhou.myqcloud.com/%2Fpublic/img/00ce40298870d1accb7920d641152722.jpg",
+                text: "腾讯网"
+              }
+            ]
+          }
+        }
+      ];
+
+      const mockStream = new ReadableStream({
+        start(controller) {
+          data.forEach((chunk) => {
+            controller.enqueue(chunk);
+          });
+
+          controller.close();
+        },
+      });
+
+      vi.spyOn(instance['client'].chat.completions, 'create').mockResolvedValue(mockStream as any);
+
+      const result = await instance.chat({
+        messages: [{ content: 'Hello', role: 'user' }],
+        model: 'mistralai/mistral-7b-instruct:free',
+        temperature: 0,
+      });
+
+      const decoder = new TextDecoder();
+      const reader = result.body!.getReader();
+      const stream: string[] = [];
+
+      while (true) {
+        const { value, done } = await reader.read();
+        if (done) break;
+        stream.push(decoder.decode(value));
+      }
+
+      expect(stream).toEqual(
+        [
+          'id: 939fbdb8dbb9b4c5944cbbe687c977c2',
+          'event: grounding',
+          'data: {"citations":[{"title":"公务员考试时政热点【2025年3月3日】_公务员考试网_华图教育","url":"http://www.huatu.com/2025/0303/2803685.html"},{"title":"外交部新闻（2025年3月3日）","url":"https://view.inews.qq.com/a/20250303A02NLC00?scene=qqsearch"}]}\n',
+          'id: 939fbdb8dbb9b4c5944cbbe687c977c2',
+          'event: text',
+          'data: "为您"\n',
+          'id: 939fbdb8dbb9b4c5944cbbe687c977c2',
+          'event: text',
+          'data: "找到"\n',
+        ].map((line) => `${line}\n`),
+      );
+
+      expect((await reader.read()).done).toBe(true);
+    });
+  });
+});

package/src/libs/agent-runtime/hunyuan/index.ts

@@ -15,14 +15,15 @@ export const LobeHunyuanAI = LobeOpenAICompatibleFactory({
 
       return {
         ...rest,
+        stream: true,
         ...(enabledSearch && {
-          /*
           citation: true,
+          enable_enhancement: true,
+          /*
           enable_multimedia: true,
-          search_info: true
           */
-          enable_enhancement: true,
           enable_speed_search: process.env.HUNYUAN_ENABLE_SPEED_SEARCH === '1',
+          search_info: true,
         }),
       } as any;
     },

package/src/libs/agent-runtime/qwen/index.ts

@@ -35,7 +35,7 @@ export const LobeQwenAI = LobeOpenAICompatibleFactory({
           : presence_penalty !== undefined && presence_penalty >= -2 && presence_penalty <= 2
             ? presence_penalty
             : undefined,
-        stream: !payload.tools,
+        stream: true,
         temperature:
           temperature !== undefined && temperature >= 0 && temperature < 2
             ? temperature

package/src/libs/agent-runtime/utils/streams/openai.ts

@@ -127,19 +127,35 @@ export const transformOpenAIStream = (
       }
 
       if (typeof content === 'string') {
-        // in Perplexity api, the citation is in every chunk, but we only need to return it once
-
-        if ('citations' in chunk && !!chunk.citations && !streamContext?.returnedPplxCitation) {
-          streamContext.returnedPplxCitation = true;
-
-          const citations = (chunk.citations as any[]).map((item) =>
-            typeof item === 'string' ? ({ title: item, url: item } as CitationItem) : item,
-          );
-
-          return [
-            { data: { citations }, id: chunk.id, type: 'grounding' },
-            { data: content, id: chunk.id, type: 'text' },
-          ];
+        if (!streamContext?.returnedCitation) {
+          const citations =
+            // in Perplexity api, the citation is in every chunk, but we only need to return it once
+            ('citations' in chunk && chunk.citations) ||
+            // in Hunyuan api, the citation is in every chunk
+            ('search_info' in chunk && (chunk.search_info as any)?.search_results) ||
+            // in Wenxin api, the citation is in the first and last chunk
+            ('search_results' in chunk && chunk.search_results);
+
+          if (citations) {
+            streamContext.returnedCitation = true;
+
+            return [
+              {
+                data: {
+                  citations: (citations as any[]).map(
+                    (item) =>
+                      ({
+                        title: typeof item === 'string' ? item : item.title,
+                        url: typeof item === 'string' ? item : item.url,
+                      }) as CitationItem
+                  ),
+                },
+                id: chunk.id,
+                type: 'grounding',
+              },
+              { data: content, id: chunk.id, type: 'text' },
+            ];
+          }
         }
 
         return { data: content, id: chunk.id, type: 'text' };

package/src/libs/agent-runtime/utils/streams/protocol.ts

@@ -9,9 +9,10 @@ export interface StreamContext {
   id: string;
   /**
    * As pplx citations is in every chunk, but we only need to return it once
-   * this flag is used to check if the pplx citation is returned,and then not return it again
+   * this flag is used to check if the pplx citation is returned,and then not return it again.
+   * Same as Hunyuan and Wenxin
    */
-  returnedPplxCitation?: boolean;
+  returnedCitation?: boolean;
   thinking?: {
     id: string;
     name: string;

package/src/libs/agent-runtime/wenxin/index.test.ts

@@ -0,0 +1,113 @@
+// @vitest-environment node
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { LobeOpenAICompatibleRuntime, ModelProvider } from '@/libs/agent-runtime';
+import { testProvider } from '@/libs/agent-runtime/providerTestUtils';
+
+import { LobeWenxinAI } from './index';
+
+testProvider({
+  Runtime: LobeWenxinAI,
+  provider: ModelProvider.Wenxin,
+  defaultBaseURL: 'https://qianfan.baidubce.com/v2',
+  chatDebugEnv: 'DEBUG_WENXIN_CHAT_COMPLETION',
+  chatModel: 'ernie-speed-128k',
+});
+
+// Mock the console.error to avoid polluting test output
+vi.spyOn(console, 'error').mockImplementation(() => {});
+
+let instance: LobeOpenAICompatibleRuntime;
+
+beforeEach(() => {
+  instance = new LobeWenxinAI({ apiKey: 'test' });
+
+  // 使用 vi.spyOn 来模拟 chat.completions.create 方法
+  vi.spyOn(instance['client'].chat.completions, 'create').mockResolvedValue(
+    new ReadableStream() as any,
+  );
+});
+
+describe('LobeWenxinAI', () => {
+  describe('chat', () => {
+    it('should with search citations', async () => {
+      const data = [
+        {
+          id: "as-bhrxwy5fq1",
+          object: "chat.completion.chunk",
+          created: 1741000028,
+          model: "ernie-4.0-8k-latest",
+          choices: [
+            {
+              index: 0,
+              delta: { content: "今天是**", role: "assistant" },
+              flag: 0
+            }
+          ],
+          search_results: [
+            { index: 1, url: "http://www.mnw.cn/news/shehui/", title: "社会新闻" },
+            { index: 2, url: "https://www.chinanews.com.cn/sh/2025/03-01/10376297.shtml", title: "中越边民共庆“春龙节”" },
+            { index: 3, url: "https://www.chinanews.com/china/index.shtml", title: "中国新闻网_时政" }
+          ]
+        },
+        {
+          id: "as-bhrxwy5fq1",
+          object: "chat.completion.chunk",
+          created: 1741000028,
+          model: "ernie-4.0-8k-latest",
+          choices: [
+            {
+              index: 0,
+              delta: { content: "20" },
+              flag: 0
+            }
+          ]
+        }
+      ];
+
+      const mockStream = new ReadableStream({
+        start(controller) {
+          data.forEach((chunk) => {
+            controller.enqueue(chunk);
+          });
+
+          controller.close();
+        },
+      });
+
+      vi.spyOn(instance['client'].chat.completions, 'create').mockResolvedValue(mockStream as any);
+
+      const result = await instance.chat({
+        messages: [{ content: 'Hello', role: 'user' }],
+        model: 'mistralai/mistral-7b-instruct:free',
+        temperature: 0,
+      });
+
+      const decoder = new TextDecoder();
+      const reader = result.body!.getReader();
+      const stream: string[] = [];
+
+      while (true) {
+        const { value, done } = await reader.read();
+        if (done) break;
+        stream.push(decoder.decode(value));
+      }
+
+      expect(stream).toEqual(
+        [
+          'id: as-bhrxwy5fq1',
+          'event: grounding',
+          'data: {"citations":[{"title":"社会新闻","url":"http://www.mnw.cn/news/shehui/"},{"title":"中越边民共庆“春龙节”","url":"https://www.chinanews.com.cn/sh/2025/03-01/10376297.shtml"},{"title":"中国新闻网_时政","url":"https://www.chinanews.com/china/index.shtml"}]}\n',
+          'id: as-bhrxwy5fq1',
+          'event: text',
+          'data: "今天是**"\n',
+          'id: as-bhrxwy5fq1',
+          'event: text',
+          'data: "20"\n',
+        ].map((line) => `${line}\n`),
+      );
+
+      expect((await reader.read()).done).toBe(true);
+    });
+  });
+});

package/src/libs/agent-runtime/wenxin/index.ts

@@ -3,6 +3,23 @@ import { LobeOpenAICompatibleFactory } from '../utils/openaiCompatibleFactory';
 
 export const LobeWenxinAI = LobeOpenAICompatibleFactory({
   baseURL: 'https://qianfan.baidubce.com/v2',
+  chatCompletion: {
+    handlePayload: (payload) => {
+      const { enabledSearch, ...rest } = payload;
+
+      return {
+        ...rest,
+        stream: true,
+        ...(enabledSearch && {
+          web_search: {
+            enable: true,
+            enable_citation: true,
+            enable_trace: true,
+          }
+        }),
+      } as any;
+    },
+  },
   debug: {
     chatCompletion: () => process.env.DEBUG_WENXIN_CHAT_COMPLETION === '1',
   },