@lobehub/lobehub 2.0.0-next.98 → 2.0.0-next.99

package/src/features/ChatList/Messages/Group/Tool/Inspector/ToolTitle.tsx

@@ -2,15 +2,14 @@ import { Icon } from '@lobehub/ui';
 import { createStyles } from 'antd-style';
 import isEqual from 'fast-deep-equal';
 import { ChevronRight } from 'lucide-react';
-import { memo, useMemo } from 'react';
+import { memo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { Flexbox } from 'react-layout-kit';
 
 import { pluginHelpers, useToolStore } from '@/store/tool';
 import { toolSelectors } from '@/store/tool/selectors';
 import { shinyTextStylish } from '@/styles/loading';
-import { LocalSystemManifest } from '@/tools/local-system';
-import { WebBrowsingManifest } from '@/tools/web-browsing';
+import { builtinToolIdentifiers } from '@/tools/identifiers';
 
 import BuiltinPluginTitle from './BuiltinPluginTitle';
 
@@ -49,33 +48,16 @@ const ToolTitle = memo<ToolTitleProps>(
 
     const pluginMeta = useToolStore(toolSelectors.getMetaById(identifier), isEqual);
 
-    const plugins = useMemo(
-      () => [
-        {
-          apiName: t(`search.apiName.${apiName}`, apiName),
-          id: WebBrowsingManifest.identifier,
-          title: t('search.title'),
-        },
-        {
-          apiName: t(`localSystem.apiName.${apiName}`, apiName),
-          id: LocalSystemManifest.identifier,
-          title: t('localSystem.title'),
-        },
-      ],
-      [],
-    );
-
-    const builtinPluginTitle = plugins.find((item) => item.id === identifier);
-
-    if (!!builtinPluginTitle) {
+    if (builtinToolIdentifiers.includes(identifier)) {
       return (
         <BuiltinPluginTitle
-          {...builtinPluginTitle}
+          apiName={t(`builtins.${identifier}.apiName.${apiName}`, apiName)}
           identifier={identifier}
           index={index}
           isExpanded={isExpanded}
           isLoading={isLoading}
           messageId={messageId}
+          title={t(`builtins.${identifier}.title`, identifier)}
           toolCallId={toolCallId}
         />
       );

package/src/features/ChatList/Messages/Tool/Inspector/ToolTitle.tsx

@@ -2,15 +2,14 @@ import { Icon } from '@lobehub/ui';
 import { createStyles } from 'antd-style';
 import isEqual from 'fast-deep-equal';
 import { ChevronRight } from 'lucide-react';
-import { memo, useMemo } from 'react';
+import { memo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { Flexbox } from 'react-layout-kit';
 
 import { pluginHelpers, useToolStore } from '@/store/tool';
 import { toolSelectors } from '@/store/tool/selectors';
 import { shinyTextStylish } from '@/styles/loading';
-import { LocalSystemManifest } from '@/tools/local-system';
-import { WebBrowsingManifest } from '@/tools/web-browsing';
+import { builtinToolIdentifiers } from '@/tools/identifiers';
 
 import BuiltinPluginTitle from './BuiltinPluginTitle';
 
@@ -43,33 +42,14 @@ const ToolTitle = memo<ToolTitleProps>(({ identifier, messageId, index, apiName,
 
   const pluginMeta = useToolStore(toolSelectors.getMetaById(identifier), isEqual);
 
-  const plugins = useMemo(
-    () => [
-      {
-        apiName: t(`search.apiName.${apiName}`, apiName),
-        // icon: <Icon icon={Globe} size={13} />,
-        id: WebBrowsingManifest.identifier,
-        title: t('search.title'),
-      },
-      {
-        apiName: t(`localSystem.apiName.${apiName}`, apiName),
-        // icon: <Icon icon={Laptop} size={13} />,
-        id: LocalSystemManifest.identifier,
-        title: t('localSystem.title'),
-      },
-    ],
-    [],
-  );
-
-  const builtinPluginTitle = plugins.find((item) => item.id === identifier);
-
-  if (!!builtinPluginTitle) {
+  if (builtinToolIdentifiers.includes(identifier)) {
     return (
       <BuiltinPluginTitle
-        {...builtinPluginTitle}
+        apiName={t(`builtins.${identifier}.apiName.${apiName}`, apiName)}
         identifier={identifier}
         index={index}
         messageId={messageId}
+        title={t(`builtins.${identifier}.title`, identifier)}
         toolCallId={toolCallId}
       />
     );

package/src/features/PluginsUI/Render/BuiltinType/index.tsx

@@ -37,7 +37,7 @@ const BuiltinType = memo<BuiltinTypeProps>(
     return (
       <Render
         apiName={apiName}
-        args={args}
+        args={args || {}}
         content={data}
         identifier={identifier}
         messageId={id}

package/src/helpers/toolEngineering/index.test.ts

@@ -1,7 +1,7 @@
 import { LobeChatPluginManifest } from '@lobehub/chat-plugin-sdk';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { createChatToolsEngine, createToolsEngine, getEnabledTools } from './index';
+import { createAgentToolsEngine, createToolsEngine, getEnabledTools } from './index';
 
 // Mock the store and helper dependencies
 vi.mock('@/store/tool', () => ({
@@ -137,7 +137,7 @@ describe('toolEngineering', () => {
 
   describe('createChatToolsEngine', () => {
     it('should include web browsing tool as default when no tools are provided', () => {
-      const toolsEngine = createChatToolsEngine({
+      const toolsEngine = createAgentToolsEngine({
         model: 'gpt-4',
         provider: 'openai',
       });
@@ -152,7 +152,7 @@ describe('toolEngineering', () => {
     });
 
     it('should include web browsing tool alongside user-provided tools', () => {
-      const toolsEngine = createChatToolsEngine({
+      const toolsEngine = createAgentToolsEngine({
         model: 'gpt-4',
         provider: 'openai',
       });

package/src/helpers/toolEngineering/index.ts

@@ -13,6 +13,9 @@ import { WebBrowsingManifest } from '@/tools/web-browsing';
 import { getSearchConfig } from '../getSearchConfig';
 import { isCanUseFC } from '../isCanUseFC';
 import { shouldEnableTool } from '../toolFilters';
+import { KnowledgeBaseManifest } from '@/tools/knowledge-base';
+import { getAgentStoreState } from '@/store/agent';
+import { agentSelectors } from '@/store/agent/slices/chat';
 
 /**
  * Tools engine configuration options
@@ -53,23 +56,33 @@ export const createToolsEngine = (config: ToolsEngineConfig = {}): ToolsEngine =
   });
 };
 
-export const createChatToolsEngine = (workingModel: WorkingModel) =>
+export const createAgentToolsEngine = (workingModel: WorkingModel) =>
   createToolsEngine({
-    // Add WebBrowsingManifest as default tool
-    defaultToolIds: [WebBrowsingManifest.identifier],
+    // Add default tools based on configuration
+    defaultToolIds: [
+      WebBrowsingManifest.identifier,
+      // Only add KnowledgeBase tool if knowledge is enabled
+      KnowledgeBaseManifest.identifier,
+    ],
     // Create search-aware enableChecker for this request
     enableChecker: ({ pluginId }) => {
       // Check platform-specific constraints (e.g., LocalSystem desktop-only)
       if (!shouldEnableTool(pluginId)) {
         return false;
       }
-
       // For WebBrowsingManifest, apply search logic
       if (pluginId === WebBrowsingManifest.identifier) {
         const searchConfig = getSearchConfig(workingModel.model, workingModel.provider);
         return searchConfig.useApplicationBuiltinSearchTool;
       }
 
+      // For KnowledgeBaseManifest, only enable if knowledge is enabled
+      if (pluginId === KnowledgeBaseManifest.identifier) {
+        const agentState = getAgentStoreState();
+
+        return agentSelectors.hasEnabledKnowledge(agentState);
+      }
+
       // For all other plugins, enable by default
       return true;
     },

package/src/libs/trpc/client/lambda.ts

@@ -34,9 +34,6 @@ const errorHandlingLink: TRPCLink<LambdaRouter> = () => {
           // Don't show notifications for abort errors
           if (showError && !isAbortError) {
             const { loginRequired } = await import('@/components/Error/loginRequiredNotification');
-            const { fetchErrorNotification } = await import(
-              '@/components/Error/fetchErrorNotification'
-            );
 
             switch (status) {
               case 401: {
@@ -53,9 +50,6 @@ const errorHandlingLink: TRPCLink<LambdaRouter> = () => {
 
               default: {
                 console.error(err);
-                if (fetchErrorNotification && status) {
-                  fetchErrorNotification.error({ errorMessage: err.message, status });
-                }
               }
             }
           }

package/src/locales/default/plugin.ts

@@ -1,4 +1,38 @@
 export default {
+  builtins: {
+    'lobe-knowledge-base': {
+      apiName: {
+        readKnowledge: '读取知识库内容',
+        searchKnowledgeBase: '搜索知识库',
+      },
+      title: '知识库',
+    },
+    'lobe-local-system': {
+      apiName: {
+        editLocalFile: '编辑文件',
+        getCommandOutput: '获取代码输出',
+        globLocalFiles: '匹配搜索文件',
+        grepContent: '搜索内容',
+        killCommand: '终止代码执行',
+        listLocalFiles: '查看文件列表',
+        moveLocalFiles: '移动文件',
+        readLocalFile: '读取文件内容',
+        renameLocalFile: '重命名',
+        runCommand: '执行代码',
+        searchLocalFiles: '搜索文件',
+        writeLocalFile: '写入文件',
+      },
+      title: '本地系统',
+    },
+    'lobe-web-browsing': {
+      apiName: {
+        crawlMultiPages: '读取多个页面内容',
+        crawlSinglePage: '读取页面内容',
+        search: '搜索页面',
+      },
+      title: '联网搜索',
+    },
+  },
   confirm: '确定',
   debug: {
     arguments: '调用参数',
@@ -253,23 +287,6 @@ export default {
     content: '调用插件中...',
     plugin: '插件运行中...',
   },
-  localSystem: {
-    apiName: {
-      editLocalFile: '编辑文件',
-      getCommandOutput: '获取代码输出',
-      globLocalFiles: '匹配搜索文件',
-      grepContent: '搜索内容',
-      killCommand: '终止代码执行',
-      listLocalFiles: '查看文件列表',
-      moveLocalFiles: '移动文件',
-      readLocalFile: '读取文件内容',
-      renameLocalFile: '重命名',
-      runCommand: '执行代码',
-      searchLocalFiles: '搜索文件',
-      writeLocalFile: '写入文件',
-    },
-    title: '本地系统',
-  },
   mcpInstall: {
     CHECKING_INSTALLATION: '检查安装环境...',
     COMPLETED: '安装完成',
@@ -378,11 +395,6 @@ export default {
     warning: '⚠️ 请确认您信任此插件的来源，恶意插件可能会危害您的系统安全。',
   },
   search: {
-    apiName: {
-      crawlMultiPages: '读取多个页面内容',
-      crawlSinglePage: '读取页面内容',
-      search: '搜索页面',
-    },
     config: {
       addKey: '添加秘钥',
       close: '删除',

package/src/locales/default/tool.ts

@@ -1,12 +1,12 @@
 export default {
-  codeInterpreter: {
+  'codeInterpreter': {
     error: '执行错误',
     executing: '执行中...',
     files: '文件：',
     output: '输出：',
     returnValue: '返回值：',
   },
-  dalle: {
+  'dalle': {
     autoGenerate: '自动生成',
     downloading: 'DallE3 生成的图片链接有效期仅1小时，正在缓存图片到本地...',
     generate: '生成',
@@ -14,7 +14,15 @@ export default {
     images: '图片:',
     prompt: '提示词',
   },
-  localFiles: {
+  'lobe-knowledge-base': {
+    readKnowledge: {
+      meta: {
+        chars: '字符数',
+        lines: '行数',
+      },
+    },
+  },
+  'localFiles': {
     editFile: {
       newString: '替换为',
       oldString: '查找内容',
@@ -47,7 +55,7 @@ export default {
       truncated: '已截断',
     },
   },
-  search: {
+  'search': {
     createNewSearch: '创建新的搜索记录',
     emptyResult: '没有搜索到结果，请修改关键词后重试',
     genAiMessage: '创建助手消息',
@@ -94,7 +102,7 @@ export default {
     summaryTooltip: '总结当前内容',
     viewMoreResults: '查看更多 {{results}} 个结果',
   },
-  updateArgs: {
+  'updateArgs': {
     duplicateKeyError: '字段键必须唯一',
     form: {
       add: '添加一项',

package/src/server/routers/lambda/chunk.ts

@@ -1,32 +1,50 @@
 import { DEFAULT_FILE_EMBEDDING_MODEL_ITEM } from '@lobechat/const';
-import { SemanticSearchSchema } from '@lobechat/types';
+import {
+  ChatSemanticSearchChunk,
+  FileSearchResult,
+  ProviderConfig,
+  SemanticSearchSchema,
+} from '@lobechat/types';
 import { TRPCError } from '@trpc/server';
 import { inArray } from 'drizzle-orm';
+import pMap from 'p-map';
 import { z } from 'zod';
 
 import { AsyncTaskModel } from '@/database/models/asyncTask';
 import { ChunkModel } from '@/database/models/chunk';
+import { DocumentModel } from '@/database/models/document';
 import { EmbeddingModel } from '@/database/models/embedding';
 import { FileModel } from '@/database/models/file';
 import { MessageModel } from '@/database/models/message';
+import { AiInfraRepos } from '@/database/repositories/aiInfra';
 import { knowledgeBaseFiles } from '@/database/schemas';
 import { authedProcedure, router } from '@/libs/trpc/lambda';
 import { keyVaults, serverDatabase } from '@/libs/trpc/lambda/middleware';
-import { getServerDefaultFilesConfig } from '@/server/globalConfig';
+import { getServerDefaultFilesConfig, getServerGlobalConfig } from '@/server/globalConfig';
+import { KeyVaultsGateKeeper } from '@/server/modules/KeyVaultsEncrypt';
 import { initModelRuntimeWithUserPayload } from '@/server/modules/ModelRuntime';
 import { ChunkService } from '@/server/services/chunk';
+import { DocumentService } from '@/server/services/document';
 
 const chunkProcedure = authedProcedure
   .use(serverDatabase)
   .use(keyVaults)
   .use(async (opts) => {
     const { ctx } = opts;
+    const { aiProvider } = await getServerGlobalConfig();
 
     return opts.next({
       ctx: {
+        aiInfraRepos: new AiInfraRepos(
+          ctx.serverDB,
+          ctx.userId,
+          aiProvider as Record<string, ProviderConfig>,
+        ),
         asyncTaskModel: new AsyncTaskModel(ctx.serverDB, ctx.userId),
         chunkModel: new ChunkModel(ctx.serverDB, ctx.userId),
         chunkService: new ChunkService(ctx.serverDB, ctx.userId),
+        documentModel: new DocumentModel(ctx.serverDB, ctx.userId),
+        documentService: new DocumentService(ctx.serverDB, ctx.userId),
         embeddingModel: new EmbeddingModel(ctx.serverDB, ctx.userId),
         fileModel: new FileModel(ctx.serverDB, ctx.userId),
         messageModel: new MessageModel(ctx.serverDB, ctx.userId),
@@ -34,6 +52,50 @@ const chunkProcedure = authedProcedure
     });
   });
 
+/**
+ * Group chunks by file and calculate relevance scores
+ */
+const groupAndRankFiles = (chunks: ChatSemanticSearchChunk[], topK: number): FileSearchResult[] => {
+  const fileMap = new Map<string, FileSearchResult>();
+
+  // Group chunks by file
+  for (const chunk of chunks) {
+    const fileId = chunk.fileId || 'unknown';
+    const fileName = chunk.fileName || `File ${fileId}`;
+
+    if (!fileMap.has(fileId)) {
+      fileMap.set(fileId, {
+        fileId,
+        fileName,
+        relevanceScore: 0,
+        topChunks: [],
+      });
+    }
+
+    const fileResult = fileMap.get(fileId)!;
+    fileResult.topChunks.push({
+      id: chunk.id,
+      similarity: chunk.similarity,
+      text: chunk.text || '',
+    });
+  }
+
+  // Calculate relevance score for each file (average of top 3 chunks)
+  for (const fileResult of fileMap.values()) {
+    fileResult.topChunks.sort((a, b) => b.similarity - a.similarity);
+    const top3 = fileResult.topChunks.slice(0, 3);
+    fileResult.relevanceScore =
+      top3.reduce((sum, chunk) => sum + chunk.similarity, 0) / top3.length;
+    // Keep only top chunks per file
+    fileResult.topChunks = fileResult.topChunks.slice(0, 3);
+  }
+
+  // Sort files by relevance score and return top K
+  return Array.from(fileMap.values())
+    .sort((a, b) => b.relevanceScore - a.relevanceScore)
+    .slice(0, topK);
+};
+
 export const chunkRouter = router({
   createEmbeddingChunksTask: chunkProcedure
     .input(
@@ -78,6 +140,66 @@ export const chunkRouter = router({
       };
     }),
 
+  getFileContents: chunkProcedure
+    .input(
+      z.object({
+        fileIds: z.array(z.string()),
+      }),
+    )
+    .mutation(async ({ ctx, input }) => {
+      return await pMap(
+        input.fileIds,
+        async (fileId) => {
+          // 1. Find file information
+          const file = await ctx.fileModel.findById(fileId);
+          if (!file) {
+            return {
+              content: '',
+              error: 'File not found',
+              fileId,
+              filename: `Unknown file ${fileId}`,
+            };
+          }
+
+          // 2. Find existing parsed document
+          let document = await ctx.documentModel.findByFileId(fileId);
+
+          // 3. If not exists, parse the file
+          if (!document) {
+            try {
+              document = await ctx.documentService.parseFile(fileId);
+            } catch (error) {
+              return {
+                content: '',
+                error: `Failed to parse file: ${(error as Error).message}`,
+                fileId,
+                filename: file.name,
+              };
+            }
+          }
+
+          // 4. Calculate file statistics
+          const content = document.content || '';
+          const lines = content.split('\n');
+          const totalLineCount = lines.length;
+          const totalCharCount = content.length;
+          const preview = lines.slice(0, 5).join('\n');
+
+          // 5. Return content with details
+          return {
+            content,
+            fileId,
+            filename: file.name,
+            metadata: document.metadata,
+            preview,
+            totalCharCount,
+            totalLineCount,
+          };
+        },
+        { concurrency: 3 },
+      );
+    }),
+
   retryParseFileTask: chunkProcedure
     .input(
       z.object({
@@ -117,7 +239,6 @@ export const chunkRouter = router({
         input: input.query,
         model,
       });
-      console.timeEnd('embedding');
 
       return ctx.chunkModel.semanticSearch({
         embedding: embeddings![0],
@@ -130,47 +251,30 @@ export const chunkRouter = router({
     .input(SemanticSearchSchema)
     .mutation(async ({ ctx, input }) => {
       try {
-        const item = await ctx.messageModel.findMessageQueriesById(input.messageId);
         const { model, provider } =
           getServerDefaultFilesConfig().embeddingModel || DEFAULT_FILE_EMBEDDING_MODEL_ITEM;
         let embedding: number[];
-        let ragQueryId: string;
-
-        // if there is no message rag or it's embeddings, then we need to create one
-        if (!item || !item.embeddings) {
-          // TODO: need to support customize
-          const agentRuntime = await initModelRuntimeWithUserPayload(provider, ctx.jwtPayload);
-
-          // slice content to make sure in the context window limit
-          const query =
-            input.rewriteQuery.length > 8000
-              ? input.rewriteQuery.slice(0, 8000)
-              : input.rewriteQuery;
-
-          const embeddings = await agentRuntime.embeddings({
-            dimensions: 1024,
-            input: query,
-            model,
-          });
 
-          embedding = embeddings![0];
-          const embeddingsId = await ctx.embeddingModel.create({
-            embeddings: embedding,
-            model,
-          });
+        const providerDetail = await ctx.aiInfraRepos.getAiProviderDetail(
+          provider,
+          KeyVaultsGateKeeper.getUserKeyVaults,
+        );
 
-          const result = await ctx.messageModel.createMessageQuery({
-            embeddingsId,
-            messageId: input.messageId,
-            rewriteQuery: input.rewriteQuery,
-            userQuery: input.userQuery,
-          });
+        const modelRuntime = initModelRuntimeWithUserPayload(
+          provider,
+          providerDetail.keyVaults || {},
+        );
 
-          ragQueryId = result.id;
-        } else {
-          embedding = item.embeddings;
-          ragQueryId = item.id;
-        }
+        // slice content to make sure in the context window limit
+        const query = input.query.length > 8000 ? input.query.slice(0, 8000) : input.query;
+
+        const embeddings = await modelRuntime.embeddings({
+          dimensions: 1024,
+          input: query,
+          model,
+        });
+
+        embedding = embeddings![0];
 
         let finalFileIds = input.fileIds ?? [];
 
@@ -185,18 +289,41 @@ export const chunkRouter = router({
         const chunks = await ctx.chunkModel.semanticSearchForChat({
           embedding,
           fileIds: finalFileIds,
-          query: input.rewriteQuery,
+          query: input.query,
+          topK: input.topK,
         });
 
+        // Group chunks by file and calculate relevance scores
+        const fileResults = groupAndRankFiles(chunks, input.topK || 15);
+
         // TODO: need to rerank the chunks
 
-        return { chunks, queryId: ragQueryId };
+        return { chunks, fileResults };
       } catch (e) {
         console.error(e);
 
+        const error = e as any;
+        const errorType = error.errorType;
+
+        // Map business error types to appropriate HTTP status codes
+        if (errorType === 'InvalidProviderAPIKey') {
+          throw new TRPCError({
+            code: 'METHOD_NOT_SUPPORTED',
+            message: error.message || 'Invalid API key for embedding provider',
+          });
+        }
+
+        if (errorType === 'ProviderBizError') {
+          throw new TRPCError({
+            code: 'BAD_REQUEST',
+            message: error.message || 'Provider service error',
+          });
+        }
+
+        // For unknown errors, still return 500 but with proper message
         throw new TRPCError({
           code: 'INTERNAL_SERVER_ERROR',
-          message: (e as any).errorType || JSON.stringify(e),
+          message: error.message || errorType || 'Failed to perform semantic search',
         });
       }
     }),

package/src/services/chat/chat.test.ts

@@ -913,7 +913,7 @@ describe('ChatService', () => {
             enabledToolIds: [WebBrowsingManifest.identifier],
           }),
         };
-        vi.spyOn(toolEngineeringModule, 'createChatToolsEngine').mockReturnValue(
+        vi.spyOn(toolEngineeringModule, 'createAgentToolsEngine').mockReturnValue(
           mockToolsEngine as any,
         );
 
@@ -964,7 +964,7 @@ describe('ChatService', () => {
             enabledToolIds: [WebBrowsingManifest.identifier],
           }),
         };
-        vi.spyOn(toolEngineeringModule, 'createChatToolsEngine').mockReturnValue(
+        vi.spyOn(toolEngineeringModule, 'createAgentToolsEngine').mockReturnValue(
           mockToolsEngine as any,
         );
 
@@ -1009,7 +1009,7 @@ describe('ChatService', () => {
             enabledToolIds: [WebBrowsingManifest.identifier],
           }),
         };
-        vi.spyOn(toolEngineeringModule, 'createChatToolsEngine').mockReturnValue(
+        vi.spyOn(toolEngineeringModule, 'createAgentToolsEngine').mockReturnValue(
           mockToolsEngine as any,
         );
 

package/src/services/chat/index.ts

@@ -14,7 +14,7 @@ import { enableAuth } from '@/const/auth';
 import { DEFAULT_AGENT_CONFIG } from '@/const/settings';
 import { isDesktop } from '@/const/version';
 import { getSearchConfig } from '@/helpers/getSearchConfig';
-import { createChatToolsEngine, createToolsEngine } from '@/helpers/toolEngineering';
+import { createAgentToolsEngine, createToolsEngine } from '@/helpers/toolEngineering';
 import { getAgentStoreState } from '@/store/agent';
 import { agentChatConfigSelectors, agentSelectors } from '@/store/agent/selectors';
 import { aiModelSelectors, aiProviderSelectors, getAiInfraStoreState } from '@/store/aiInfra';
@@ -90,7 +90,7 @@ class ChatService {
 
     const pluginIds = [...(enabledPlugins || [])];
 
-    const toolsEngine = createChatToolsEngine({
+    const toolsEngine = createAgentToolsEngine({
       model: payload.model,
       provider: payload.provider!,
     });