@lobehub/lobehub 2.0.0-next.97 → 2.0.0-next.99

package/src/helpers/toolEngineering/index.ts

@@ -13,6 +13,9 @@ import { WebBrowsingManifest } from '@/tools/web-browsing';
 import { getSearchConfig } from '../getSearchConfig';
 import { isCanUseFC } from '../isCanUseFC';
 import { shouldEnableTool } from '../toolFilters';
+import { KnowledgeBaseManifest } from '@/tools/knowledge-base';
+import { getAgentStoreState } from '@/store/agent';
+import { agentSelectors } from '@/store/agent/slices/chat';
 
 /**
  * Tools engine configuration options
@@ -53,23 +56,33 @@ export const createToolsEngine = (config: ToolsEngineConfig = {}): ToolsEngine =
   });
 };
 
-export const createChatToolsEngine = (workingModel: WorkingModel) =>
+export const createAgentToolsEngine = (workingModel: WorkingModel) =>
   createToolsEngine({
-    // Add WebBrowsingManifest as default tool
-    defaultToolIds: [WebBrowsingManifest.identifier],
+    // Add default tools based on configuration
+    defaultToolIds: [
+      WebBrowsingManifest.identifier,
+      // Only add KnowledgeBase tool if knowledge is enabled
+      KnowledgeBaseManifest.identifier,
+    ],
     // Create search-aware enableChecker for this request
     enableChecker: ({ pluginId }) => {
       // Check platform-specific constraints (e.g., LocalSystem desktop-only)
       if (!shouldEnableTool(pluginId)) {
         return false;
       }
-
       // For WebBrowsingManifest, apply search logic
       if (pluginId === WebBrowsingManifest.identifier) {
         const searchConfig = getSearchConfig(workingModel.model, workingModel.provider);
         return searchConfig.useApplicationBuiltinSearchTool;
       }
 
+      // For KnowledgeBaseManifest, only enable if knowledge is enabled
+      if (pluginId === KnowledgeBaseManifest.identifier) {
+        const agentState = getAgentStoreState();
+
+        return agentSelectors.hasEnabledKnowledge(agentState);
+      }
+
       // For all other plugins, enable by default
       return true;
     },

package/src/libs/trpc/client/lambda.ts

@@ -34,9 +34,6 @@ const errorHandlingLink: TRPCLink<LambdaRouter> = () => {
           // Don't show notifications for abort errors
           if (showError && !isAbortError) {
             const { loginRequired } = await import('@/components/Error/loginRequiredNotification');
-            const { fetchErrorNotification } = await import(
-              '@/components/Error/fetchErrorNotification'
-            );
 
             switch (status) {
               case 401: {
@@ -53,9 +50,6 @@ const errorHandlingLink: TRPCLink<LambdaRouter> = () => {
 
               default: {
                 console.error(err);
-                if (fetchErrorNotification && status) {
-                  fetchErrorNotification.error({ errorMessage: err.message, status });
-                }
               }
             }
           }

package/src/locales/default/file.ts

@@ -55,11 +55,11 @@ export default {
   },
   documentList: {
     copyContent: '复制全文',
-    documentCount: '共 {{count}} 个文档',
     duplicate: '创建副本',
-    empty: '暂无文档，点击上方按钮创建你的第一篇文档',
-    noResults: '未找到匹配的文档',
-    selectNote: '选择一个文档开始编辑',
+    empty: '暂无文稿，点击上方按钮创建你的第一篇文稿',
+    noResults: '未找到匹配的文稿',
+    pageCount: '共 {{count}} 个文稿',
+    selectNote: '选择一个文稿开始编辑',
     untitled: '无标题',
   },
   empty: '暂无已上传文件/文件夹',
@@ -70,7 +70,6 @@ export default {
       uploadFile: '上传文件',
       uploadFolder: '上传文件夹',
     },
-    newDocumentButton: '新建文档',
     newNoteDialog: {
       cancel: '取消',
       editTitle: '编辑文档',
@@ -83,14 +82,15 @@ export default {
       title: '新建文档',
       updateSuccess: '文档更新成功',
     },
+    newPageButton: '新建文稿',
     uploadButton: '上传',
   },
   home: {
     getStarted: '开始使用',
     greeting: '开始',
     quickActions: '快捷操作',
-    recentDocuments: '最近文档',
     recentFiles: '最近文件',
+    recentPages: '最近文稿',
     subtitle: '欢迎使用知识库，从这里开始管理你的文档和文档',
     uploadEntries: {
       files: {
@@ -102,8 +102,8 @@ export default {
       knowledgeBase: {
         title: '新建知识库',
       },
-      newDocument: {
-        title: '新建文档',
+      newPage: {
+        title: '新建文稿',
       },
     },
   },
@@ -117,8 +117,8 @@ export default {
     title: '知识库',
   },
   menu: {
-    allDocuments: '全部文档',
     allFiles: '全部文件',
+    allPages: '全部文稿',
   },
   networkError: '获取知识库失败，请检测网络连接后重试',
   notSupportGuide: {
@@ -143,8 +143,8 @@ export default {
     downloadFile: '下载文件',
     unsupportedFileAndContact: '此文件格式暂不支持在线预览，如有预览诉求，欢迎<1>反馈给我们</1>',
   },
-  searchDocumentPlaceholder: '搜索文档',
   searchFilePlaceholder: '搜索文件',
+  searchPagePlaceholder: '搜索文稿',
   tab: {
     all: '全部',
     audios: '语音',
@@ -157,9 +157,7 @@ export default {
     websites: '网页',
   },
   title: '知识库',
-  toggleLeftPanel: {
-    title: '显示/隐藏左侧面板',
-  },
+  toggleLeftPanel: '显示/隐藏左侧面板',
   uploadDock: {
     body: {
       collapse: '收起',

package/src/locales/default/plugin.ts

@@ -1,4 +1,38 @@
 export default {
+  builtins: {
+    'lobe-knowledge-base': {
+      apiName: {
+        readKnowledge: '读取知识库内容',
+        searchKnowledgeBase: '搜索知识库',
+      },
+      title: '知识库',
+    },
+    'lobe-local-system': {
+      apiName: {
+        editLocalFile: '编辑文件',
+        getCommandOutput: '获取代码输出',
+        globLocalFiles: '匹配搜索文件',
+        grepContent: '搜索内容',
+        killCommand: '终止代码执行',
+        listLocalFiles: '查看文件列表',
+        moveLocalFiles: '移动文件',
+        readLocalFile: '读取文件内容',
+        renameLocalFile: '重命名',
+        runCommand: '执行代码',
+        searchLocalFiles: '搜索文件',
+        writeLocalFile: '写入文件',
+      },
+      title: '本地系统',
+    },
+    'lobe-web-browsing': {
+      apiName: {
+        crawlMultiPages: '读取多个页面内容',
+        crawlSinglePage: '读取页面内容',
+        search: '搜索页面',
+      },
+      title: '联网搜索',
+    },
+  },
   confirm: '确定',
   debug: {
     arguments: '调用参数',
@@ -253,23 +287,6 @@ export default {
     content: '调用插件中...',
     plugin: '插件运行中...',
   },
-  localSystem: {
-    apiName: {
-      editLocalFile: '编辑文件',
-      getCommandOutput: '获取代码输出',
-      globLocalFiles: '匹配搜索文件',
-      grepContent: '搜索内容',
-      killCommand: '终止代码执行',
-      listLocalFiles: '查看文件列表',
-      moveLocalFiles: '移动文件',
-      readLocalFile: '读取文件内容',
-      renameLocalFile: '重命名',
-      runCommand: '执行代码',
-      searchLocalFiles: '搜索文件',
-      writeLocalFile: '写入文件',
-    },
-    title: '本地系统',
-  },
   mcpInstall: {
     CHECKING_INSTALLATION: '检查安装环境...',
     COMPLETED: '安装完成',
@@ -378,11 +395,6 @@ export default {
     warning: '⚠️ 请确认您信任此插件的来源，恶意插件可能会危害您的系统安全。',
   },
   search: {
-    apiName: {
-      crawlMultiPages: '读取多个页面内容',
-      crawlSinglePage: '读取页面内容',
-      search: '搜索页面',
-    },
     config: {
       addKey: '添加秘钥',
       close: '删除',

package/src/locales/default/tool.ts

@@ -1,12 +1,12 @@
 export default {
-  codeInterpreter: {
+  'codeInterpreter': {
     error: '执行错误',
     executing: '执行中...',
     files: '文件：',
     output: '输出：',
     returnValue: '返回值：',
   },
-  dalle: {
+  'dalle': {
     autoGenerate: '自动生成',
     downloading: 'DallE3 生成的图片链接有效期仅1小时，正在缓存图片到本地...',
     generate: '生成',
@@ -14,7 +14,15 @@ export default {
     images: '图片:',
     prompt: '提示词',
   },
-  localFiles: {
+  'lobe-knowledge-base': {
+    readKnowledge: {
+      meta: {
+        chars: '字符数',
+        lines: '行数',
+      },
+    },
+  },
+  'localFiles': {
     editFile: {
       newString: '替换为',
       oldString: '查找内容',
@@ -47,7 +55,7 @@ export default {
       truncated: '已截断',
     },
   },
-  search: {
+  'search': {
     createNewSearch: '创建新的搜索记录',
     emptyResult: '没有搜索到结果，请修改关键词后重试',
     genAiMessage: '创建助手消息',
@@ -94,7 +102,7 @@ export default {
     summaryTooltip: '总结当前内容',
     viewMoreResults: '查看更多 {{results}} 个结果',
   },
-  updateArgs: {
+  'updateArgs': {
     duplicateKeyError: '字段键必须唯一',
     form: {
       add: '添加一项',

package/src/server/routers/lambda/chunk.ts

@@ -1,32 +1,50 @@
 import { DEFAULT_FILE_EMBEDDING_MODEL_ITEM } from '@lobechat/const';
-import { SemanticSearchSchema } from '@lobechat/types';
+import {
+  ChatSemanticSearchChunk,
+  FileSearchResult,
+  ProviderConfig,
+  SemanticSearchSchema,
+} from '@lobechat/types';
 import { TRPCError } from '@trpc/server';
 import { inArray } from 'drizzle-orm';
+import pMap from 'p-map';
 import { z } from 'zod';
 
 import { AsyncTaskModel } from '@/database/models/asyncTask';
 import { ChunkModel } from '@/database/models/chunk';
+import { DocumentModel } from '@/database/models/document';
 import { EmbeddingModel } from '@/database/models/embedding';
 import { FileModel } from '@/database/models/file';
 import { MessageModel } from '@/database/models/message';
+import { AiInfraRepos } from '@/database/repositories/aiInfra';
 import { knowledgeBaseFiles } from '@/database/schemas';
 import { authedProcedure, router } from '@/libs/trpc/lambda';
 import { keyVaults, serverDatabase } from '@/libs/trpc/lambda/middleware';
-import { getServerDefaultFilesConfig } from '@/server/globalConfig';
+import { getServerDefaultFilesConfig, getServerGlobalConfig } from '@/server/globalConfig';
+import { KeyVaultsGateKeeper } from '@/server/modules/KeyVaultsEncrypt';
 import { initModelRuntimeWithUserPayload } from '@/server/modules/ModelRuntime';
 import { ChunkService } from '@/server/services/chunk';
+import { DocumentService } from '@/server/services/document';
 
 const chunkProcedure = authedProcedure
   .use(serverDatabase)
   .use(keyVaults)
   .use(async (opts) => {
     const { ctx } = opts;
+    const { aiProvider } = await getServerGlobalConfig();
 
     return opts.next({
       ctx: {
+        aiInfraRepos: new AiInfraRepos(
+          ctx.serverDB,
+          ctx.userId,
+          aiProvider as Record<string, ProviderConfig>,
+        ),
         asyncTaskModel: new AsyncTaskModel(ctx.serverDB, ctx.userId),
         chunkModel: new ChunkModel(ctx.serverDB, ctx.userId),
         chunkService: new ChunkService(ctx.serverDB, ctx.userId),
+        documentModel: new DocumentModel(ctx.serverDB, ctx.userId),
+        documentService: new DocumentService(ctx.serverDB, ctx.userId),
         embeddingModel: new EmbeddingModel(ctx.serverDB, ctx.userId),
         fileModel: new FileModel(ctx.serverDB, ctx.userId),
         messageModel: new MessageModel(ctx.serverDB, ctx.userId),
@@ -34,6 +52,50 @@ const chunkProcedure = authedProcedure
     });
   });
 
+/**
+ * Group chunks by file and calculate relevance scores
+ */
+const groupAndRankFiles = (chunks: ChatSemanticSearchChunk[], topK: number): FileSearchResult[] => {
+  const fileMap = new Map<string, FileSearchResult>();
+
+  // Group chunks by file
+  for (const chunk of chunks) {
+    const fileId = chunk.fileId || 'unknown';
+    const fileName = chunk.fileName || `File ${fileId}`;
+
+    if (!fileMap.has(fileId)) {
+      fileMap.set(fileId, {
+        fileId,
+        fileName,
+        relevanceScore: 0,
+        topChunks: [],
+      });
+    }
+
+    const fileResult = fileMap.get(fileId)!;
+    fileResult.topChunks.push({
+      id: chunk.id,
+      similarity: chunk.similarity,
+      text: chunk.text || '',
+    });
+  }
+
+  // Calculate relevance score for each file (average of top 3 chunks)
+  for (const fileResult of fileMap.values()) {
+    fileResult.topChunks.sort((a, b) => b.similarity - a.similarity);
+    const top3 = fileResult.topChunks.slice(0, 3);
+    fileResult.relevanceScore =
+      top3.reduce((sum, chunk) => sum + chunk.similarity, 0) / top3.length;
+    // Keep only top chunks per file
+    fileResult.topChunks = fileResult.topChunks.slice(0, 3);
+  }
+
+  // Sort files by relevance score and return top K
+  return Array.from(fileMap.values())
+    .sort((a, b) => b.relevanceScore - a.relevanceScore)
+    .slice(0, topK);
+};
+
 export const chunkRouter = router({
   createEmbeddingChunksTask: chunkProcedure
     .input(
@@ -78,6 +140,66 @@ export const chunkRouter = router({
       };
     }),
 
+  getFileContents: chunkProcedure
+    .input(
+      z.object({
+        fileIds: z.array(z.string()),
+      }),
+    )
+    .mutation(async ({ ctx, input }) => {
+      return await pMap(
+        input.fileIds,
+        async (fileId) => {
+          // 1. Find file information
+          const file = await ctx.fileModel.findById(fileId);
+          if (!file) {
+            return {
+              content: '',
+              error: 'File not found',
+              fileId,
+              filename: `Unknown file ${fileId}`,
+            };
+          }
+
+          // 2. Find existing parsed document
+          let document = await ctx.documentModel.findByFileId(fileId);
+
+          // 3. If not exists, parse the file
+          if (!document) {
+            try {
+              document = await ctx.documentService.parseFile(fileId);
+            } catch (error) {
+              return {
+                content: '',
+                error: `Failed to parse file: ${(error as Error).message}`,
+                fileId,
+                filename: file.name,
+              };
+            }
+          }
+
+          // 4. Calculate file statistics
+          const content = document.content || '';
+          const lines = content.split('\n');
+          const totalLineCount = lines.length;
+          const totalCharCount = content.length;
+          const preview = lines.slice(0, 5).join('\n');
+
+          // 5. Return content with details
+          return {
+            content,
+            fileId,
+            filename: file.name,
+            metadata: document.metadata,
+            preview,
+            totalCharCount,
+            totalLineCount,
+          };
+        },
+        { concurrency: 3 },
+      );
+    }),
+
   retryParseFileTask: chunkProcedure
     .input(
       z.object({
@@ -117,7 +239,6 @@ export const chunkRouter = router({
         input: input.query,
         model,
       });
-      console.timeEnd('embedding');
 
       return ctx.chunkModel.semanticSearch({
         embedding: embeddings![0],
@@ -130,47 +251,30 @@ export const chunkRouter = router({
     .input(SemanticSearchSchema)
     .mutation(async ({ ctx, input }) => {
       try {
-        const item = await ctx.messageModel.findMessageQueriesById(input.messageId);
         const { model, provider } =
           getServerDefaultFilesConfig().embeddingModel || DEFAULT_FILE_EMBEDDING_MODEL_ITEM;
         let embedding: number[];
-        let ragQueryId: string;
-
-        // if there is no message rag or it's embeddings, then we need to create one
-        if (!item || !item.embeddings) {
-          // TODO: need to support customize
-          const agentRuntime = await initModelRuntimeWithUserPayload(provider, ctx.jwtPayload);
-
-          // slice content to make sure in the context window limit
-          const query =
-            input.rewriteQuery.length > 8000
-              ? input.rewriteQuery.slice(0, 8000)
-              : input.rewriteQuery;
-
-          const embeddings = await agentRuntime.embeddings({
-            dimensions: 1024,
-            input: query,
-            model,
-          });
 
-          embedding = embeddings![0];
-          const embeddingsId = await ctx.embeddingModel.create({
-            embeddings: embedding,
-            model,
-          });
+        const providerDetail = await ctx.aiInfraRepos.getAiProviderDetail(
+          provider,
+          KeyVaultsGateKeeper.getUserKeyVaults,
+        );
 
-          const result = await ctx.messageModel.createMessageQuery({
-            embeddingsId,
-            messageId: input.messageId,
-            rewriteQuery: input.rewriteQuery,
-            userQuery: input.userQuery,
-          });
+        const modelRuntime = initModelRuntimeWithUserPayload(
+          provider,
+          providerDetail.keyVaults || {},
+        );
 
-          ragQueryId = result.id;
-        } else {
-          embedding = item.embeddings;
-          ragQueryId = item.id;
-        }
+        // slice content to make sure in the context window limit
+        const query = input.query.length > 8000 ? input.query.slice(0, 8000) : input.query;
+
+        const embeddings = await modelRuntime.embeddings({
+          dimensions: 1024,
+          input: query,
+          model,
+        });
+
+        embedding = embeddings![0];
 
         let finalFileIds = input.fileIds ?? [];
 
@@ -185,18 +289,41 @@ export const chunkRouter = router({
         const chunks = await ctx.chunkModel.semanticSearchForChat({
           embedding,
           fileIds: finalFileIds,
-          query: input.rewriteQuery,
+          query: input.query,
+          topK: input.topK,
         });
 
+        // Group chunks by file and calculate relevance scores
+        const fileResults = groupAndRankFiles(chunks, input.topK || 15);
+
         // TODO: need to rerank the chunks
 
-        return { chunks, queryId: ragQueryId };
+        return { chunks, fileResults };
       } catch (e) {
         console.error(e);
 
+        const error = e as any;
+        const errorType = error.errorType;
+
+        // Map business error types to appropriate HTTP status codes
+        if (errorType === 'InvalidProviderAPIKey') {
+          throw new TRPCError({
+            code: 'METHOD_NOT_SUPPORTED',
+            message: error.message || 'Invalid API key for embedding provider',
+          });
+        }
+
+        if (errorType === 'ProviderBizError') {
+          throw new TRPCError({
+            code: 'BAD_REQUEST',
+            message: error.message || 'Provider service error',
+          });
+        }
+
+        // For unknown errors, still return 500 but with proper message
         throw new TRPCError({
           code: 'INTERNAL_SERVER_ERROR',
-          message: (e as any).errorType || JSON.stringify(e),
+          message: error.message || errorType || 'Failed to perform semantic search',
         });
       }
     }),

package/src/services/chat/chat.test.ts

@@ -913,7 +913,7 @@ describe('ChatService', () => {
             enabledToolIds: [WebBrowsingManifest.identifier],
           }),
         };
-        vi.spyOn(toolEngineeringModule, 'createChatToolsEngine').mockReturnValue(
+        vi.spyOn(toolEngineeringModule, 'createAgentToolsEngine').mockReturnValue(
           mockToolsEngine as any,
         );
 
@@ -964,7 +964,7 @@ describe('ChatService', () => {
             enabledToolIds: [WebBrowsingManifest.identifier],
           }),
         };
-        vi.spyOn(toolEngineeringModule, 'createChatToolsEngine').mockReturnValue(
+        vi.spyOn(toolEngineeringModule, 'createAgentToolsEngine').mockReturnValue(
           mockToolsEngine as any,
         );
 
@@ -1009,7 +1009,7 @@ describe('ChatService', () => {
             enabledToolIds: [WebBrowsingManifest.identifier],
           }),
         };
-        vi.spyOn(toolEngineeringModule, 'createChatToolsEngine').mockReturnValue(
+        vi.spyOn(toolEngineeringModule, 'createAgentToolsEngine').mockReturnValue(
           mockToolsEngine as any,
         );
 

package/src/services/chat/index.ts

@@ -14,7 +14,7 @@ import { enableAuth } from '@/const/auth';
 import { DEFAULT_AGENT_CONFIG } from '@/const/settings';
 import { isDesktop } from '@/const/version';
 import { getSearchConfig } from '@/helpers/getSearchConfig';
-import { createChatToolsEngine, createToolsEngine } from '@/helpers/toolEngineering';
+import { createAgentToolsEngine, createToolsEngine } from '@/helpers/toolEngineering';
 import { getAgentStoreState } from '@/store/agent';
 import { agentChatConfigSelectors, agentSelectors } from '@/store/agent/selectors';
 import { aiModelSelectors, aiProviderSelectors, getAiInfraStoreState } from '@/store/aiInfra';
@@ -90,7 +90,7 @@ class ChatService {
 
     const pluginIds = [...(enabledPlugins || [])];
 
-    const toolsEngine = createChatToolsEngine({
+    const toolsEngine = createAgentToolsEngine({
       model: payload.model,
       provider: payload.provider!,
     });

package/src/services/rag.ts

@@ -22,8 +22,12 @@ class RAGService {
     return lambdaClient.chunk.semanticSearch.mutate({ fileIds, query });
   };
 
-  semanticSearchForChat = async (params: SemanticSearchSchemaType) => {
-    return lambdaClient.chunk.semanticSearchForChat.mutate(params);
+  semanticSearchForChat = async (params: SemanticSearchSchemaType, signal?: AbortSignal) => {
+    return lambdaClient.chunk.semanticSearchForChat.mutate(params, { signal });
+  };
+
+  getFileContents = async (fileIds: string[], signal?: AbortSignal) => {
+    return lambdaClient.chunk.getFileContents.mutate({ fileIds }, { signal });
   };
 
   deleteMessageRagQuery = async (id: string) => {

package/src/store/chat/agents/__tests__/createAgentExecutors/helpers/testExecutor.ts

@@ -1,7 +1,4 @@
-import type {
-  AgentInstruction,
-  AgentState,
-} from '@lobechat/agent-runtime';
+import type { AgentInstruction, AgentState } from '@lobechat/agent-runtime';
 
 import { createAgentExecutors } from '@/store/chat/agents/createAgentExecutors';
 import type { OperationType } from '@/store/chat/slices/operation/types';

package/src/store/chat/slices/aiChat/actions/__tests__/conversationLifecycle.test.ts

@@ -15,6 +15,17 @@ import {
 // Keep zustand mock as it's needed globally
 vi.mock('zustand/traditional');
 
+// Mock lambdaClient to prevent network requests
+vi.mock('@/libs/trpc/client', () => ({
+  lambdaClient: {
+    session: {
+      updateSession: {
+        mutate: vi.fn().mockResolvedValue(undefined),
+      },
+    },
+  },
+}));
+
 beforeEach(() => {
   resetTestEnvironment();
   setupMockSelectors();