@lobehub/chat 1.32.0 → 1.32.2

package/CHANGELOG.md

@@ -2,6 +2,56 @@
 
 # Changelog
 
+### [Version 1.32.2](https://github.com/lobehub/lobe-chat/compare/v1.32.1...v1.32.2)
+
+<sup>Released on **2024-11-19**</sup>
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
+### [Version 1.32.1](https://github.com/lobehub/lobe-chat/compare/v1.32.0...v1.32.1)
+
+<sup>Released on **2024-11-19**</sup>
+
+#### 🐛 Bug Fixes
+
+- **misc**: Keyword search for chat history & sessions.
+
+#### 💄 Styles
+
+- **misc**: Support o1 models using streaming.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's fixed
+
+- **misc**: Keyword search for chat history & sessions, closes [#4725](https://github.com/lobehub/lobe-chat/issues/4725) ([415d772](https://github.com/lobehub/lobe-chat/commit/415d772))
+
+#### Styles
+
+- **misc**: Support o1 models using streaming, closes [#4732](https://github.com/lobehub/lobe-chat/issues/4732) ([7e9e71a](https://github.com/lobehub/lobe-chat/commit/7e9e71a))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ## [Version 1.32.0](https://github.com/lobehub/lobe-chat/compare/v1.31.11...v1.32.0)
 
 <sup>Released on **2024-11-19**</sup>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.32.0",
+  "version": "1.32.2",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",

package/src/database/server/models/__tests__/session.test.ts

@@ -231,8 +231,18 @@ describe('SessionModel', () => {
 
     it('should return sessions with matching title', async () => {
       await serverDB.insert(sessions).values([
-        { id: '1', userId, title: 'Hello World', description: 'Some description' },
-        { id: '2', userId, title: 'Another Session', description: 'Another description' },
+        { id: '1', userId },
+        { id: '2', userId },
+      ]);
+
+      await serverDB.insert(agents).values([
+        { id: 'agent-1', userId, model: 'gpt-3.5-turbo', title: 'Hello, Agent 1' },
+        { id: 'agent-2', userId, model: 'gpt-4', title: 'Agent 2' },
+      ]);
+
+      await serverDB.insert(agentsToSessions).values([
+        { agentId: 'agent-1', sessionId: '1' },
+        { agentId: 'agent-2', sessionId: '2' },
       ]);
 
       const result = await sessionModel.queryByKeyword('hello');
@@ -241,9 +251,21 @@ describe('SessionModel', () => {
     });
 
     it('should return sessions with matching description', async () => {
+      // The sessions has no title and desc,
+      // see: https://github.com/lobehub/lobe-chat/pull/4725
       await serverDB.insert(sessions).values([
-        { id: '1', userId, title: 'Session 1', description: 'Description with keyword' },
-        { id: '2', userId, title: 'Session 2', description: 'Another description' },
+        { id: '1', userId },
+        { id: '2', userId },
+      ]);
+
+      await serverDB.insert(agents).values([
+        { id: 'agent-1', userId, model: 'gpt-3.5-turbo', title: 'Agent 1', description: 'Description with Keyword' },
+        { id: 'agent-2', userId, model: 'gpt-4', title: 'Agent 2' },
+      ]);
+
+      await serverDB.insert(agentsToSessions).values([
+        { agentId: 'agent-1', sessionId: '1' },
+        { agentId: 'agent-2', sessionId: '2' },
       ]);
 
       const result = await sessionModel.queryByKeyword('keyword');
@@ -253,11 +275,23 @@ describe('SessionModel', () => {
 
     it('should return sessions with matching title or description', async () => {
       await serverDB.insert(sessions).values([
+        { id: '1', userId },
+        { id: '2', userId },
+        { id: '3', userId },
+      ]);
+
+      await serverDB.insert(agents).values([
         { id: '1', userId, title: 'Title with keyword', description: 'Some description' },
         { id: '2', userId, title: 'Another Session', description: 'Description with keyword' },
         { id: '3', userId, title: 'Third Session', description: 'Third description' },
       ]);
 
+      await serverDB.insert(agentsToSessions).values([
+        { agentId: '1', sessionId: '1' },
+        { agentId: '2', sessionId: '2' },
+        { agentId: '3', sessionId: '3' },
+      ]);
+
       const result = await sessionModel.queryByKeyword('keyword');
       expect(result).toHaveLength(2);
       expect(result.map((s) => s.id)).toEqual(['1', '2']);

package/src/database/server/models/file.ts

@@ -1,5 +1,6 @@
 import { asc, count, eq, ilike, inArray, notExists, or, sum } from 'drizzle-orm';
 import { and, desc, like } from 'drizzle-orm/expressions';
+import type { PgTransaction } from 'drizzle-orm/pg-core';
 
 import { serverDBEnv } from '@/config/db';
 import { serverDB } from '@/database/server/core/db';
@@ -9,6 +10,9 @@ import {
   FileItem,
   NewFile,
   NewGlobalFile,
+  chunks,
+  embeddings,
+  fileChunks,
   files,
   globalFiles,
   knowledgeBaseFiles,
@@ -68,6 +72,10 @@ export class FileModel {
     const fileHash = file.fileHash!;
 
     return await serverDB.transaction(async (trx) => {
+      // 1. 删除相关的 chunks
+      await this.deleteFileChunks(trx as any, [id]);
+
+      // 2. 删除文件记录
       await trx.delete(files).where(and(eq(files.id, id), eq(files.userId, this.userId)));
 
       const result = await trx
@@ -107,6 +115,9 @@ export class FileModel {
     const hashList = fileList.map((file) => file.fileHash!);
 
     return await serverDB.transaction(async (trx) => {
+      // 1. 删除相关的 chunks
+      await this.deleteFileChunks(trx as any, ids);
+
       // delete the files
       await trx.delete(files).where(and(inArray(files.id, ids), eq(files.userId, this.userId)));
 
@@ -289,4 +300,30 @@ export class FileModel {
       ),
     });
   }
+
+  // 抽象出通用的删除 chunks 方法
+  private async deleteFileChunks(trx: PgTransaction<any>, fileIds: string[]) {
+    const BATCH_SIZE = 1000; // 每批处理的数量
+
+    // 1. 获取所有关联的 chunk IDs
+    const relatedChunks = await trx
+      .select({ chunkId: fileChunks.chunkId })
+      .from(fileChunks)
+      .where(inArray(fileChunks.fileId, fileIds));
+
+    const chunkIds = relatedChunks.map((c) => c.chunkId).filter(Boolean) as string[];
+
+    if (chunkIds.length === 0) return;
+
+    // 2. 分批处理删除
+    for (let i = 0; i < chunkIds.length; i += BATCH_SIZE) {
+      const batchChunkIds = chunkIds.slice(i, i + BATCH_SIZE);
+
+      await trx.delete(embeddings).where(inArray(embeddings.chunkId, batchChunkIds));
+
+      await trx.delete(chunks).where(inArray(chunks.id, batchChunkIds));
+    }
+
+    return chunkIds;
+  }
 }

package/src/database/server/models/session.ts

@@ -61,7 +61,7 @@ export class SessionModel {
 
     const keywordLowerCase = keyword.toLowerCase();
 
-    const data = await this.findSessions({ keyword: keywordLowerCase });
+    const data = await this.findSessionsByKeywords({ keyword: keywordLowerCase });
 
     return data.map((item) => this.mapSessionItem(item as any));
   }
@@ -281,15 +281,15 @@ export class SessionModel {
         pinned !== undefined ? eq(sessions.pinned, pinned) : eq(sessions.userId, this.userId),
         keyword
           ? or(
-              like(
-                sql`lower(${sessions.title})` as unknown as Column,
-                `%${keyword.toLowerCase()}%`,
-              ),
-              like(
-                sql`lower(${sessions.description})` as unknown as Column,
-                `%${keyword.toLowerCase()}%`,
-              ),
-            )
+            like(
+              sql`lower(${sessions.title})` as unknown as Column,
+              `%${keyword.toLowerCase()}%`,
+            ),
+            like(
+              sql`lower(${sessions.description})` as unknown as Column,
+              `%${keyword.toLowerCase()}%`,
+            ),
+          )
           : eq(sessions.userId, this.userId),
         group ? eq(sessions.groupId, group) : isNull(sessions.groupId),
       ),
@@ -297,4 +297,37 @@ export class SessionModel {
       with: { agentsToSessions: { columns: {}, with: { agent: true } }, group: true },
     });
   }
+
+  async findSessionsByKeywords(params: {
+    current?: number;
+    keyword: string;
+    pageSize?: number;
+  }) {
+    const { keyword, pageSize = 9999, current = 0 } = params;
+    const offset = current * pageSize;
+    const results = await serverDB.query.agents.findMany({
+      limit: pageSize,
+      offset,
+      orderBy: [desc(agents.updatedAt)],
+      where: and(
+        eq(agents.userId, this.userId),
+        or(
+          like(
+            sql`lower(${agents.title})` as unknown as Column,
+            `%${keyword.toLowerCase()}%`,
+          ),
+          like(
+            sql`lower(${agents.description})` as unknown as Column,
+            `%${keyword.toLowerCase()}%`,
+          ),
+        )
+      ),
+      with: { agentsToSessions: { columns: {}, with: { session: true } } },
+    });
+    try {
+      // @ts-expect-error
+      return results.map((item) => item.agentsToSessions[0].session);
+    } catch {}
+    return []
+  }
 }

package/src/database/server/models/topic.ts

@@ -85,7 +85,12 @@ export class TopicModel {
             serverDB
               .select()
               .from(messages)
-              .where(and(eq(messages.topicId, topics.id), or(matchKeyword(messages.content)))),
+              .where(
+                and(
+                  eq(messages.topicId, topics.id),
+                  matchKeyword(messages.content)
+                )
+              ),
           ),
         ),
       ),

package/src/libs/agent-runtime/github/index.ts

@@ -10,7 +10,7 @@ export const LobeGithubAI = LobeOpenAICompatibleFactory({
       const { model } = payload;
 
       if (o1Models.has(model)) {
-        return pruneO1Payload(payload) as any;
+        return { ...pruneO1Payload(payload), stream: false } as any;
       }
 
       return { ...payload, stream: payload.stream ?? true };

package/src/libs/agent-runtime/openai/index.ts

@@ -17,7 +17,6 @@ export const pruneO1Payload = (payload: ChatStreamPayload) => ({
     role: message.role === 'system' ? 'user' : message.role,
   })),
   presence_penalty: 0,
-  stream: false,
   temperature: 1,
   top_p: 1,
 });

package/src/server/routers/lambda/file.ts

@@ -154,8 +154,6 @@ export const fileRouter = router({
   removeFile: fileProcedure.input(z.object({ id: z.string() })).mutation(async ({ input, ctx }) => {
     const file = await ctx.fileModel.delete(input.id);
 
-    // delete the orphan chunks
-    await ctx.chunkModel.deleteOrphanChunks();
     if (!file) return;
 
     // delele the file from remove from S3 if it is not used by other files
@@ -187,9 +185,6 @@ export const fileRouter = router({
     .mutation(async ({ input, ctx }) => {
       const needToRemoveFileList = await ctx.fileModel.deleteMany(input.ids);
 
-      // delete the orphan chunks
-      await ctx.chunkModel.deleteOrphanChunks();
-
       if (!needToRemoveFileList || needToRemoveFileList.length === 0) return;
 
       // remove from S3

package/src/store/chat/slices/topic/selectors.ts

@@ -42,7 +42,7 @@ const currentActiveTopicSummary = (s: ChatStoreState): ChatTopicSummary | undefi
 const isCreatingTopic = (s: ChatStoreState) => s.creatingTopic;
 
 const groupedTopicsSelector = (s: ChatStoreState): GroupedTopic[] => {
-  const topics = currentTopics(s);
+  const topics = displayTopics(s);
 
   if (!topics) return [];
   const favTopics = currentFavTopics(s);