@lobehub/chat 1.12.10 → 1.12.12

package/.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
           --health-cmd pg_isready --health-interval 10s --health-timeout 5s  --health-retries 5
         ports:
           - 5432:5432
-          
+
     steps:
       - uses: actions/checkout@v4
 
@@ -40,7 +40,8 @@ jobs:
           DATABASE_DRIVER: node
           NEXT_PUBLIC_SERVICE_MODE: server
           KEY_VAULTS_SECRET: LA7n9k3JdEcbSgml2sxfw+4TV1AzaaFU5+R176aQz4s=
-          NEXT_PUBLIC_S3_DOMAIN: https://example.com
+          S3_PUBLIC_DOMAIN: https://example.com
+          APP_URL: https://home.com
 
       - name: Test App Coverage
         run: bun run test-app:coverage

package/.github/workflows/test.yml

@@ -39,7 +39,8 @@ jobs:
           DATABASE_DRIVER: node
           NEXT_PUBLIC_SERVICE_MODE: server
           KEY_VAULTS_SECRET: LA7n9k3JdEcbSgml2sxfw+4TV1AzaaFU5+R176aQz4s=
-          NEXT_PUBLIC_S3_DOMAIN: https://example.com
+          S3_PUBLIC_DOMAIN: https://example.com
+          APP_URL: https://home.com
 
       - name: Upload Server coverage to Codecov
         uses: codecov/codecov-action@v4

package/CHANGELOG.md

@@ -2,6 +2,48 @@
 
 # Changelog
 
+### [Version 1.12.12](https://github.com/lobehub/lobe-chat/compare/v1.12.11...v1.12.12)
+
+<sup>Released on **2024-08-24**</sup>
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
+### [Version 1.12.11](https://github.com/lobehub/lobe-chat/compare/v1.12.10...v1.12.11)
+
+<sup>Released on **2024-08-23**</sup>
+
+#### 🐛 Bug Fixes
+
+- **misc**: Remove orphan chunks if there is no related file.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's fixed
+
+- **misc**: Remove orphan chunks if there is no related file, closes [#3578](https://github.com/lobehub/lobe-chat/issues/3578) ([36bcaf3](https://github.com/lobehub/lobe-chat/commit/36bcaf3))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 1.12.10](https://github.com/lobehub/lobe-chat/compare/v1.12.9...v1.12.10)
 
 <sup>Released on **2024-08-23**</sup>

package/Dockerfile.database

@@ -24,6 +24,7 @@ FROM base AS builder
 ARG USE_CN_MIRROR
 
 ENV NEXT_PUBLIC_SERVICE_MODE="server" \
+    APP_URL="http://app.com" \
     DATABASE_DRIVER="node" \
     DATABASE_URL="postgres://postgres:password@localhost:5432/postgres" \
     KEY_VAULTS_SECRET="use-for-build"

package/README.md

@@ -269,11 +269,11 @@ Our marketplace is not just a showcase platform but also a collaborative space.
 | Recent Submits                                                                                                                                                                                | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
 | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | [Variable Name Conversion Expert](https://chat-preview.lobehub.com/market?agent=variable-name-conversion)<br/><sup>By **[zengyishou](https://github.com/zengyishou)** on **2024-08-21**</sup> | In software development, naming variables is a common yet relatively time-consuming task. This assistant can automatically convert Chinese variable names into English variable names that conform to camelCase, PascalCase, snake_case, kebab-case, and constant naming conventions based on specific naming rules. This not only improves code readability but also alleviates the frustration of variable naming.<br/>`software-development` `variable-naming` `chinese-to-english` `code-standards` `automatic-conversion` |
+| [Prompt Engineering Expert](https://chat-preview.lobehub.com/market?agent=ai-prompts-assistant)<br/><sup>By **[cyicz123](https://github.com/cyicz123)** on **2024-08-12**</sup>               | Specializing in prompt optimization and design<br/>`prompt-engineering` `ai-interaction` `writing` `optimization` `consulting`                                                                                                                                                                                                                                                                                                                                                                                                 |
 | [Commit Message Generator](https://chat-preview.lobehub.com/market?agent=commit-assistant)<br/><sup>By **[cyicz123](https://github.com/cyicz123)** on **2024-08-12**</sup>                    | Expert at generating precise Git commit messages<br/>`programming` `git` `commit-message` `code-review`                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | [RO-SCIRAW Prompt Word Expert](https://chat-preview.lobehub.com/market?agent=rosciraw)<br/><sup>By **[kirklin](https://github.com/kirklin)** on **2024-08-06**</sup>                          | The RO-SCIRAW framework, created by Kirk Lin, is a methodology for prompt words that provides a new paradigm for building highly precise and efficient prompt words. Please enter the information for the persona you want to create.<br/>`prompt-word-framework`                                                                                                                                                                                                                                                              |
-| [Social Media Sage](https://chat-preview.lobehub.com/market?agent=social-media-sage)<br/><sup>By **[thedivergentai](https://github.com/thedivergentai)** on **2024-08-06**</sup>              | Social Media Marketing expert crafting winning strategies for brands and empowering businesses to thrive online<br/>`social-media-marketing` `branding` `growth-strategies`                                                                                                                                                                                                                                                                                                                                                    |
 
-> 📊 Total agents: [<kbd>**314**</kbd> ](https://github.com/lobehub/lobe-chat-agents)
+> 📊 Total agents: [<kbd>**315**</kbd> ](https://github.com/lobehub/lobe-chat-agents)
 
  <!-- AGENT LIST -->
 

package/README.zh-CN.md

@@ -257,11 +257,11 @@ LobeChat 的插件生态系统是其核心功能的重要扩展，它极大地
 | 最近新增                                                                                                                                                                         | 助手说明                                                                                                                                                                                                                                                                              |
 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | [开发变量名转换专家](https://chat-preview.lobehub.com/market?agent=variable-name-conversion)<br/><sup>By **[zengyishou](https://github.com/zengyishou)** on **2024-08-21**</sup> | 在软件开发过程中，命名变量是一项常见却相对耗时的任务。本助手能够根据特定的命名规则自动将中文变量名转换为符合小驼峰、大驼峰、下划线、横线以及常量命名规范的英文变量名。这不仅提高了代码的可读性，还解决了变量命名的苦恼。<br/>`软件开发` `变量命名` `中文转英文` `代码规范` `自动转换` |
+| [提示工程专家](https://chat-preview.lobehub.com/market?agent=ai-prompts-assistant)<br/><sup>By **[cyicz123](https://github.com/cyicz123)** on **2024-08-12**</sup>               | 专精 Prompt 优化与设计<br/>`提示工程` `ai交互` `写作` `优化` `咨询`                                                                                                                                                                                                                   |
 | [提交信息生成器](https://chat-preview.lobehub.com/market?agent=commit-assistant)<br/><sup>By **[cyicz123](https://github.com/cyicz123)** on **2024-08-12**</sup>                 | 擅长生成精准的 Git 提交信息<br/>`编程` `git` `提交信息` `代码审查`                                                                                                                                                                                                                    |
 | [RO-SCIRAW 提示词专家](https://chat-preview.lobehub.com/market?agent=rosciraw)<br/><sup>By **[kirklin](https://github.com/kirklin)** on **2024-08-06**</sup>                     | RO-SCIRAW 框架是由 Kirk Lin 开创的提示词方法论，为构建高度精确和高效的提示词提供了一个全新的范式。请输入你要创建的分身信息。<br/>`提示词框架`                                                                                                                                         |
-| [社交媒体专家](https://chat-preview.lobehub.com/market?agent=social-media-sage)<br/><sup>By **[thedivergentai](https://github.com/thedivergentai)** on **2024-08-06**</sup>      | 社交媒体营销专家，为品牌制定成功策略，帮助企业在线蓬勃发展<br/>`社交媒体营销` `品牌塑造` `增长策略`                                                                                                                                                                                   |
 
-> 📊 Total agents: [<kbd>**314**</kbd> ](https://github.com/lobehub/lobe-chat-agents)
+> 📊 Total agents: [<kbd>**315**</kbd> ](https://github.com/lobehub/lobe-chat-agents)
 
  <!-- AGENT LIST -->
 

package/docs/self-hosting/advanced/auth.mdx

@@ -1,6 +1,9 @@
 ---
 title: LobeChat Authentication Service Configuration
-description: Learn how to configure external authentication services using Clerk or Next Auth for centralized user authorization management. Supported authentication services include Auth0, Azure ID, etc.
+description: >-
+  Learn how to configure external authentication services using Clerk or Next
+  Auth for centralized user authorization management. Supported authentication
+  services include Auth0, Azure ID, etc.
 tags:
   - Authentication Service
   - Next Auth

package/docs/self-hosting/advanced/s3.mdx

@@ -51,7 +51,8 @@ In this documentation, S3 refers to a compatible S3 storage solution, which supp
 
   <Callout type={'warning'}>
     Additionally, since this access domain is often a separate URL, it needs to be configured to allow cross-origin access to the site. Otherwise, cross-origin issues will occur in the browser.
-  </Callout>
+ 
+</Callout>
 
   ### `S3_ENABLE_PATH_STYLE`
 
@@ -65,13 +66,15 @@ In this documentation, S3 refers to a compatible S3 storage solution, which supp
   - path-style: `s3.example.net/mybucket/config.env`
   - virtual-host: `mybucket.s3.example.net/config.env`
 
-  </Callout>
+ 
+</Callout>
 
   <Callout type={'tip'}>
 
   Common S3 cloud service providers often default to the `virtual-host` mode, while self-deployed services like Minio default to using `path-style`. Therefore, if you use Minio as the S3 service, you need to set `S3_ENABLE_PATH_STYLE=1`.
 
-  </Callout>
+ 
+</Callout>
 
 </Steps>
 

package/docs/self-hosting/advanced/s3.zh-CN.mdx

@@ -73,10 +73,8 @@ S3_ENDPOINT=https://0b33a03b5c993fd2f453379dc36558e5.r2.cloudflarestorage.com
 
 常见的 S3 Cloud 服务商往往默认采用 `virtual-host` 模式，而自部署服务 minio 则默认使用的是 `path-style`。 因此如果你使用了 minio 作为 S3 服务，你需要设置 `S3_ENABLE_PATH_STYLE=1` 。
 
-
 </Callout>
 
-
 </Steps>
 
 ## S3 配置指南

package/docs/self-hosting/advanced/upstream-sync.mdx

@@ -40,6 +40,18 @@ After forking the project, due to Github's limitations, you need to manually ena
   src="https://github-production-user-asset-6210df.s3.amazonaws.com/17870709/266985177-7677b4ce-c348-4145-9f60-829d448d5be6.png"
 />
 
+If you encounter a sync failure, you need to manually click "Update Branch" once.
+
+<Image
+  alt="GitHub Action Sync Failure"
+  src="https://github.com/user-attachments/assets/9baacac6-5af4-460b-862d-682b76c18459"
+/>
+
+<Image
+  alt="Manually Sync 'Update Branch'"
+  src="https://github.com/user-attachments/assets/d524c20d-306a-45bc-971b-96920b87fab4"
+/>
+
 ## `B` Docker Deployment
 
 Upgrading the Docker deployment version is very simple, you just need to redeploy the latest LobeChat image. Here are the commands required to perform these steps:

package/docs/self-hosting/advanced/upstream-sync.zh-CN.mdx

@@ -35,6 +35,18 @@ tags:
   src="https://github-production-user-asset-6210df.s3.amazonaws.com/17870709/266985177-7677b4ce-c348-4145-9f60-829d448d5be6.png"
 />
 
+如果你遇到了同步失败的情况，你需要手动重新点一次 「Update Branch」。
+
+<Image
+  alt="GitHub Action 同步失败"
+  src="https://github.com/user-attachments/assets/9baacac6-5af4-460b-862d-682b76c18459"
+/>
+
+<Image
+  alt="手动同步 「Update Branch」"
+  src="https://github.com/user-attachments/assets/d524c20d-306a-45bc-971b-96920b87fab4"
+/>
+
 ## `B` Docker 部署
 
 Docker 部署版本的升级非常简单，只需要重新部署 LobeChat 的最新镜像即可。 以下是执行这些步骤所需的指令：

package/docs/self-hosting/server-database/vercel.mdx

@@ -109,6 +109,10 @@ KEY_VAULTS_SECRET=jgwsK28dspyVQoIf8/M3IIHl1h6LYYceSYNXeLpy6uk=
 
 Make sure to add this to the Vercel environment variables as well.
 
+### Add the `APP_URL` Environment Variable
+
+Finally, you need to add the `APP_URL` environment variable, which specifies the URL address of the LobeChat application.
+
 </Steps>
 
 ## 2. Configure Authentication Service
@@ -394,6 +398,8 @@ src={'https://github.com/lobehub/lobe-chat/assets/28616219/9cb5150d-6e1e-4c59-9a
 For easy copying, here is a summary of the environment variables required to configure the server-side database:
 
 ```shell
+APP_URL=https://your-project.com
+
 # Specify the service mode as server
 NEXT_PUBLIC_SERVICE_MODE=server
 

package/docs/self-hosting/server-database/vercel.zh-CN.mdx

@@ -102,6 +102,10 @@ KEY_VAULTS_SECRET=jgwsK28dspyVQoIf8/M3IIHl1h6LYYceSYNXeLpy6uk=
 
 同样需要将其添加到 Vercel 环境变量中。
 
+### 添加 `APP_URL` 环境变量
+
+该部分最后需要添加 APP_URL 环境变量，用于指定LobeChat 应用的 URL 地址。
+
 </Steps>
 
 ## 二、 配置身份验证服务
@@ -375,6 +379,8 @@ S3_PUBLIC_DOMAIN=https://s3-dev.your-domain.com
 为方便一键复制，在此汇总配置服务端数据库所需要的环境变量：
 
 ```shell
+APP_URL=https://your-project.com
+
 # 指定服务模式为 server
 NEXT_PUBLIC_SERVICE_MODE=server
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.12.10",
+  "version": "1.12.12",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",
@@ -260,7 +260,7 @@
     "eslint-plugin-mdx": "^2.3.4",
     "eslint-plugin-unused-imports": "4.0.1",
     "fake-indexeddb": "^6.0.0",
-    "glob": "^10.4.5",
+    "glob": "^11.0.0",
     "gray-matter": "^4.0.3",
     "happy-dom": "^14.12.3",
     "husky": "^9.1.5",

package/src/config/app.ts

@@ -2,6 +2,8 @@
 import { createEnv } from '@t3-oss/env-nextjs';
 import { z } from 'zod';
 
+import { isServerMode } from '@/const/version';
+
 declare global {
   // eslint-disable-next-line @typescript-eslint/no-namespace
   namespace NodeJS {
@@ -10,14 +12,19 @@ declare global {
     }
   }
 }
+const isInVercel = process.env.VERCEL === '1';
 
-export const getAppConfig = () => {
-  const ACCESS_CODES = process.env.ACCESS_CODE?.split(',').filter(Boolean) || [];
-  const isInVercel = process.env.VERCEL === '1';
+const vercelUrl = `https://${process.env.VERCEL_URL}`;
 
-  const vercelUrl = `https://${process.env.VERCEL_URL}`;
+const APP_URL = process.env.APP_URL ? process.env.APP_URL : isInVercel ? vercelUrl : undefined;
 
-  const APP_URL = process.env.APP_URL ? process.env.APP_URL : isInVercel ? vercelUrl : undefined;
+// only throw error in server mode and server side
+if (typeof window === 'undefined' && isServerMode && !APP_URL) {
+  throw new Error('`APP_URL` is required in server mode');
+}
+
+export const getAppConfig = () => {
+  const ACCESS_CODES = process.env.ACCESS_CODE?.split(',').filter(Boolean) || [];
 
   return createEnv({
     client: {

package/src/config/db.ts

@@ -11,6 +11,8 @@ export const getServerDBConfig = () => {
       DATABASE_TEST_URL: process.env.DATABASE_TEST_URL,
       DATABASE_URL: process.env.DATABASE_URL,
 
+      DISABLE_REMOVE_GLOBAL_FILE: process.env.DISABLE_REMOVE_GLOBAL_FILE === '1',
+
       KEY_VAULTS_SECRET: process.env.KEY_VAULTS_SECRET,
 
       NEXT_PUBLIC_ENABLED_SERVER_SERVICE: process.env.NEXT_PUBLIC_SERVICE_MODE === 'server',
@@ -20,6 +22,8 @@ export const getServerDBConfig = () => {
       DATABASE_TEST_URL: z.string().optional(),
       DATABASE_URL: z.string().optional(),
 
+      DISABLE_REMOVE_GLOBAL_FILE: z.boolean().optional(),
+
       KEY_VAULTS_SECRET: z.string().optional(),
     },
   });

package/src/database/server/models/__tests__/chunk.test.ts

@@ -98,7 +98,66 @@ describe('ChunkModel', () => {
     });
   });
 
-  // Add more test cases for other methods...
+  describe('deleteOrphanChunks', () => {
+    it('should delete orphaned chunks', async () => {
+      // Create orphaned chunks
+      await serverDB
+        .insert(chunks)
+        .values([
+          { text: 'Orphan Chunk 1', userId },
+          { text: 'Orphan Chunk 2', userId },
+        ])
+        .returning();
+
+      // Create a non-orphaned chunk
+      const [nonOrphanChunk] = await serverDB
+        .insert(chunks)
+        .values([{ text: 'Non-Orphan Chunk', userId }])
+        .returning();
+
+      await serverDB.insert(fileChunks).values([{ fileId: '1', chunkId: nonOrphanChunk.id }]);
+
+      // Execute the method
+      await chunkModel.deleteOrphanChunks();
+
+      // Check if orphaned chunks are deleted
+      const remainingChunks = await serverDB.query.chunks.findMany();
+      expect(remainingChunks).toHaveLength(1);
+      expect(remainingChunks[0].id).toBe(nonOrphanChunk.id);
+    });
+
+    it('should not delete any chunks when there are no orphans', async () => {
+      // Create non-orphaned chunks
+      const [chunk1, chunk2] = await serverDB
+        .insert(chunks)
+        .values([
+          { text: 'Chunk 1', userId },
+          { text: 'Chunk 2', userId },
+        ])
+        .returning();
+
+      await serverDB.insert(fileChunks).values([
+        { fileId: '1', chunkId: chunk1.id },
+        { fileId: '2', chunkId: chunk2.id },
+      ]);
+
+      // Execute the method
+      await chunkModel.deleteOrphanChunks();
+
+      // Check if all chunks are still present
+      const remainingChunks = await serverDB.query.chunks.findMany();
+      expect(remainingChunks).toHaveLength(2);
+    });
+
+    it('should not throw an error when the database is empty', async () => {
+      // Ensure the database is empty
+      await serverDB.delete(chunks);
+      await serverDB.delete(fileChunks);
+
+      // Execute the method and expect it not to throw
+      await expect(chunkModel.deleteOrphanChunks()).resolves.not.toThrow();
+    });
+  });
 
   describe('semanticSearch', () => {
     it('should perform semantic search and return results', async () => {

package/src/database/server/models/__tests__/file.test.ts

@@ -1,7 +1,8 @@
 // @vitest-environment node
-import { eq } from 'drizzle-orm';
+import { eq, inArray } from 'drizzle-orm';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
+import { getServerDBConfig, serverDBEnv } from '@/config/db';
 import { getTestDBInstance } from '@/database/server/core/dbForTest';
 import { FilesTabs, SortType } from '@/types/files';
 
@@ -22,6 +23,20 @@ vi.mock('@/database/server/core/db', async () => ({
   },
 }));
 
+let DISABLE_REMOVE_GLOBAL_FILE = false;
+
+vi.mock('@/config/db', async () => ({
+  get serverDBEnv() {
+    return {
+      get DISABLE_REMOVE_GLOBAL_FILE() {
+        return DISABLE_REMOVE_GLOBAL_FILE;
+      },
+      DATABASE_TEST_URL: process.env.DATABASE_TEST_URL,
+      DATABASE_DRIVER: 'node',
+    };
+  },
+}));
+
 const userId = 'file-model-test-user-id';
 const fileModel = new FileModel(userId);
 
@@ -73,18 +88,21 @@ describe('FileModel', () => {
     });
   });
 
-  it('should create a global file', async () => {
-    const globalFile = {
-      hashId: 'test-hash',
-      fileType: 'text/plain',
-      size: 100,
-      url: 'https://example.com/global-file.txt',
-      metadata: { key: 'value' },
-    };
+  describe('createGlobalFile', () => {
+    it('should create a global file', async () => {
+      const globalFile = {
+        hashId: 'test-hash',
+        fileType: 'text/plain',
+        size: 100,
+        url: 'https://example.com/global-file.txt',
+        metadata: { key: 'value' },
+      };
 
-    const result = await fileModel.createGlobalFile(globalFile);
-    expect(result[0]).toMatchObject(globalFile);
+      const result = await fileModel.createGlobalFile(globalFile);
+      expect(result[0]).toMatchObject(globalFile);
+    });
   });
+
   describe('checkHash', () => {
     it('should return isExist: false for non-existent hash', async () => {
       const result = await fileModel.checkHash('non-existent-hash');
@@ -113,58 +131,183 @@ describe('FileModel', () => {
     });
   });
 
-  it('should delete a file by id', async () => {
-    const { id } = await fileModel.create({
-      name: 'test-file.txt',
-      url: 'https://example.com/test-file.txt',
-      size: 100,
-      fileType: 'text/plain',
-    });
+  describe('delete', () => {
+    it('should delete a file by id', async () => {
+      await fileModel.createGlobalFile({
+        hashId: '1',
+        url: 'https://example.com/file1.txt',
+        size: 100,
+        fileType: 'text/plain',
+      });
 
-    await fileModel.delete(id);
+      const { id } = await fileModel.create({
+        name: 'test-file.txt',
+        url: 'https://example.com/test-file.txt',
+        size: 100,
+        fileType: 'text/plain',
+        fileHash: '1',
+      });
 
-    const file = await serverDB.query.files.findFirst({ where: eq(files.id, id) });
-    expect(file).toBeUndefined();
-  });
+      await fileModel.delete(id);
 
-  it('should delete multiple files', async () => {
-    const file1 = await fileModel.create({
-      name: 'file1.txt',
-      url: 'https://example.com/file1.txt',
-      size: 100,
-      fileType: 'text/plain',
-    });
-    const file2 = await fileModel.create({
-      name: 'file2.txt',
-      url: 'https://example.com/file2.txt',
-      size: 200,
-      fileType: 'text/plain',
+      const file = await serverDB.query.files.findFirst({ where: eq(files.id, id) });
+      const globalFile = await serverDB.query.globalFiles.findFirst({
+        where: eq(globalFiles.hashId, '1'),
+      });
+
+      expect(file).toBeUndefined();
+      expect(globalFile).toBeUndefined();
     });
+    it('should delete a file by id but global file not removed ', async () => {
+      DISABLE_REMOVE_GLOBAL_FILE = true;
+      await fileModel.createGlobalFile({
+        hashId: '1',
+        url: 'https://example.com/file1.txt',
+        size: 100,
+        fileType: 'text/plain',
+      });
 
-    await fileModel.deleteMany([file1.id, file2.id]);
+      const { id } = await fileModel.create({
+        name: 'test-file.txt',
+        url: 'https://example.com/test-file.txt',
+        size: 100,
+        fileType: 'text/plain',
+        fileHash: '1',
+      });
+
+      await fileModel.delete(id);
+
+      const file = await serverDB.query.files.findFirst({ where: eq(files.id, id) });
+      const globalFile = await serverDB.query.globalFiles.findFirst({
+        where: eq(globalFiles.hashId, '1'),
+      });
 
-    const remainingFiles = await serverDB.query.files.findMany({ where: eq(files.userId, userId) });
-    expect(remainingFiles).toHaveLength(0);
+      expect(file).toBeUndefined();
+      expect(globalFile).toBeDefined();
+      DISABLE_REMOVE_GLOBAL_FILE = false;
+    });
   });
 
-  it('should clear all files for the user', async () => {
-    await fileModel.create({
-      name: 'test-file-1.txt',
-      url: 'https://example.com/test-file-1.txt',
-      size: 100,
-      fileType: 'text/plain',
+  describe('deleteMany', () => {
+    it('should delete multiple files', async () => {
+      await fileModel.createGlobalFile({
+        hashId: '1',
+        url: 'https://example.com/file1.txt',
+        size: 100,
+        fileType: 'text/plain',
+      });
+      await fileModel.createGlobalFile({
+        hashId: '2',
+        url: 'https://example.com/file2.txt',
+        size: 200,
+        fileType: 'text/plain',
+      });
+
+      const file1 = await fileModel.create({
+        name: 'file1.txt',
+        url: 'https://example.com/file1.txt',
+        size: 100,
+        fileHash: '1',
+        fileType: 'text/plain',
+      });
+      const file2 = await fileModel.create({
+        name: 'file2.txt',
+        url: 'https://example.com/file2.txt',
+        size: 200,
+        fileType: 'text/plain',
+        fileHash: '2',
+      });
+      const globalFilesResult = await serverDB.query.globalFiles.findMany({
+        where: inArray(globalFiles.hashId, ['1', '2']),
+      });
+      expect(globalFilesResult).toHaveLength(2);
+
+      await fileModel.deleteMany([file1.id, file2.id]);
+
+      const remainingFiles = await serverDB.query.files.findMany({
+        where: eq(files.userId, userId),
+      });
+      const globalFilesResult2 = await serverDB.query.globalFiles.findMany({
+        where: inArray(
+          globalFiles.hashId,
+          remainingFiles.map((i) => i.fileHash as string),
+        ),
+      });
+
+      expect(remainingFiles).toHaveLength(0);
+      expect(globalFilesResult2).toHaveLength(0);
     });
-    await fileModel.create({
-      name: 'test-file-2.txt',
-      url: 'https://example.com/test-file-2.txt',
-      size: 200,
-      fileType: 'text/plain',
+    it('should delete multiple files but not remove global files if DISABLE_REMOVE_GLOBAL_FILE=true', async () => {
+      DISABLE_REMOVE_GLOBAL_FILE = true;
+      await fileModel.createGlobalFile({
+        hashId: '1',
+        url: 'https://example.com/file1.txt',
+        size: 100,
+        fileType: 'text/plain',
+      });
+      await fileModel.createGlobalFile({
+        hashId: '2',
+        url: 'https://example.com/file2.txt',
+        size: 200,
+        fileType: 'text/plain',
+      });
+
+      const file1 = await fileModel.create({
+        name: 'file1.txt',
+        url: 'https://example.com/file1.txt',
+        size: 100,
+        fileType: 'text/plain',
+        fileHash: '1',
+      });
+      const file2 = await fileModel.create({
+        name: 'file2.txt',
+        url: 'https://example.com/file2.txt',
+        size: 200,
+        fileType: 'text/plain',
+        fileHash: '2',
+      });
+
+      const globalFilesResult = await serverDB.query.globalFiles.findMany({
+        where: inArray(globalFiles.hashId, ['1', '2']),
+      });
+
+      expect(globalFilesResult).toHaveLength(2);
+
+      await fileModel.deleteMany([file1.id, file2.id]);
+
+      const remainingFiles = await serverDB.query.files.findMany({
+        where: eq(files.userId, userId),
+      });
+      const globalFilesResult2 = await serverDB.query.globalFiles.findMany({
+        where: inArray(globalFiles.hashId, ['1', '2']),
+      });
+
+      expect(remainingFiles).toHaveLength(0);
+      expect(globalFilesResult2).toHaveLength(2);
+      DISABLE_REMOVE_GLOBAL_FILE = false;
     });
+  });
 
-    await fileModel.clear();
+  describe('clear', () => {
+    it('should clear all files for the user', async () => {
+      await fileModel.create({
+        name: 'test-file-1.txt',
+        url: 'https://example.com/test-file-1.txt',
+        size: 100,
+        fileType: 'text/plain',
+      });
+      await fileModel.create({
+        name: 'test-file-2.txt',
+        url: 'https://example.com/test-file-2.txt',
+        size: 200,
+        fileType: 'text/plain',
+      });
+
+      await fileModel.clear();
 
-    const userFiles = await serverDB.query.files.findMany({ where: eq(files.userId, userId) });
-    expect(userFiles).toHaveLength(0);
+      const userFiles = await serverDB.query.files.findMany({ where: eq(files.userId, userId) });
+      expect(userFiles).toHaveLength(0);
+    });
   });
 
   describe('Query', () => {
@@ -334,22 +477,24 @@ describe('FileModel', () => {
     });
   });
 
-  it('should find a file by id', async () => {
-    const { id } = await fileModel.create({
-      name: 'test-file.txt',
-      url: 'https://example.com/test-file.txt',
-      size: 100,
-      fileType: 'text/plain',
-    });
+  describe('findById', () => {
+    it('should find a file by id', async () => {
+      const { id } = await fileModel.create({
+        name: 'test-file.txt',
+        url: 'https://example.com/test-file.txt',
+        size: 100,
+        fileType: 'text/plain',
+      });
 
-    const file = await fileModel.findById(id);
-    expect(file).toMatchObject({
-      id,
-      name: 'test-file.txt',
-      url: 'https://example.com/test-file.txt',
-      size: 100,
-      fileType: 'text/plain',
-      userId,
+      const file = await fileModel.findById(id);
+      expect(file).toMatchObject({
+        id,
+        name: 'test-file.txt',
+        url: 'https://example.com/test-file.txt',
+        size: 100,
+        fileType: 'text/plain',
+        userId,
+      });
     });
   });
 

package/src/database/server/models/__tests__/knowledgeBase.test.ts

@@ -1,11 +1,18 @@
 // @vitest-environment node
 import { eq } from 'drizzle-orm';
-import { desc } from 'drizzle-orm/expressions';
+import { and, desc } from 'drizzle-orm/expressions';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
 import { getTestDBInstance } from '@/database/server/core/dbForTest';
 
-import { NewKnowledgeBase, knowledgeBases, users } from '../../schemas/lobechat';
+import {
+  NewKnowledgeBase,
+  files,
+  globalFiles,
+  knowledgeBaseFiles,
+  knowledgeBases,
+  users,
+} from '../../schemas/lobechat';
 import { KnowledgeBaseModel } from '../knowledgeBase';
 
 let serverDB = await getTestDBInstance();
@@ -21,6 +28,7 @@ const knowledgeBaseModel = new KnowledgeBaseModel(userId);
 
 beforeEach(async () => {
   await serverDB.delete(users);
+  await serverDB.delete(globalFiles);
   await serverDB.insert(users).values([{ id: userId }, { id: 'user2' }]);
 });
 
@@ -129,4 +137,122 @@ describe('KnowledgeBaseModel', () => {
       });
     });
   });
+
+  const fileList = [
+    {
+      id: 'file1',
+      name: 'document.pdf',
+      url: 'https://example.com/document.pdf',
+      fileHash: 'hash1',
+      size: 1000,
+      fileType: 'application/pdf',
+      userId,
+    },
+    {
+      id: 'file2',
+      name: 'image.jpg',
+      url: 'https://example.com/image.jpg',
+      fileHash: 'hash2',
+      size: 500,
+      fileType: 'image/jpeg',
+      userId,
+    },
+  ];
+
+  describe('addFilesToKnowledgeBase', () => {
+    it('should add files to a knowledge base', async () => {
+      await serverDB.insert(globalFiles).values([
+        {
+          hashId: 'hash1',
+          url: 'https://example.com/document.pdf',
+          size: 1000,
+          fileType: 'application/pdf',
+        },
+        {
+          hashId: 'hash2',
+          url: 'https://example.com/image.jpg',
+          size: 500,
+          fileType: 'image/jpeg',
+        },
+      ]);
+
+      await serverDB.insert(files).values(fileList);
+
+      const { id: knowledgeBaseId } = await knowledgeBaseModel.create({ name: 'Test Group' });
+      const fileIds = ['file1', 'file2'];
+
+      const result = await knowledgeBaseModel.addFilesToKnowledgeBase(knowledgeBaseId, fileIds);
+
+      expect(result).toHaveLength(2);
+      expect(result).toEqual(
+        expect.arrayContaining(
+          fileIds.map((fileId) => expect.objectContaining({ fileId, knowledgeBaseId })),
+        ),
+      );
+
+      const addedFiles = await serverDB.query.knowledgeBaseFiles.findMany({
+        where: eq(knowledgeBaseFiles.knowledgeBaseId, knowledgeBaseId),
+      });
+      expect(addedFiles).toHaveLength(2);
+    });
+  });
+
+  describe('removeFilesFromKnowledgeBase', () => {
+    it('should remove files from a knowledge base', async () => {
+      await serverDB.insert(globalFiles).values([
+        {
+          hashId: 'hash1',
+          url: 'https://example.com/document.pdf',
+          size: 1000,
+          fileType: 'application/pdf',
+        },
+        {
+          hashId: 'hash2',
+          url: 'https://example.com/image.jpg',
+          size: 500,
+          fileType: 'image/jpeg',
+        },
+      ]);
+
+      await serverDB.insert(files).values(fileList);
+
+      const { id: knowledgeBaseId } = await knowledgeBaseModel.create({ name: 'Test Group' });
+      const fileIds = ['file1', 'file2'];
+      await knowledgeBaseModel.addFilesToKnowledgeBase(knowledgeBaseId, fileIds);
+
+      const filesToRemove = ['file1'];
+      await knowledgeBaseModel.removeFilesFromKnowledgeBase(knowledgeBaseId, filesToRemove);
+
+      const remainingFiles = await serverDB.query.knowledgeBaseFiles.findMany({
+        where: and(eq(knowledgeBaseFiles.knowledgeBaseId, knowledgeBaseId)),
+      });
+      expect(remainingFiles).toHaveLength(1);
+      expect(remainingFiles[0].fileId).toBe('file2');
+    });
+  });
+
+  describe('static findById', () => {
+    it('should find a knowledge base by id without user restriction', async () => {
+      const { id } = await knowledgeBaseModel.create({ name: 'Test Group' });
+
+      const group = await KnowledgeBaseModel.findById(id);
+      expect(group).toMatchObject({
+        id,
+        name: 'Test Group',
+        userId,
+      });
+    });
+
+    it('should find a knowledge base created by another user', async () => {
+      const anotherKnowledgeBaseModel = new KnowledgeBaseModel('user2');
+      const { id } = await anotherKnowledgeBaseModel.create({ name: 'Another User Group' });
+
+      const group = await KnowledgeBaseModel.findById(id);
+      expect(group).toMatchObject({
+        id,
+        name: 'Another User Group',
+        userId: 'user2',
+      });
+    });
+  });
 });

package/src/database/server/models/__tests__/user.test.ts

@@ -4,7 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { INBOX_SESSION_ID } from '@/const/session';
 import { getTestDBInstance } from '@/database/server/core/dbForTest';
 import { KeyVaultsGateKeeper } from '@/server/modules/KeyVaultsEncrypt';
-import { UserPreference } from '@/types/user';
+import { UserGuide, UserPreference } from '@/types/user';
 import { UserSettings } from '@/types/user/settings';
 
 import { userSettings, users } from '../../schemas/lobechat';
@@ -167,6 +167,24 @@ describe('UserModel', () => {
       const { plaintext } = await gateKeeper.decrypt(updatedSettings!.keyVaults!);
       expect(JSON.parse(plaintext)).toEqual(settings.keyVaults);
     });
+
+    it('should update user settings with encrypted keyVaults', async () => {
+      const settings = {
+        general: { language: 'en-US' },
+      } as UserSettings;
+      await serverDB.insert(users).values({ id: userId });
+      await serverDB.insert(userSettings).values({ ...settings, keyVaults: '', id: userId });
+
+      const newSettings = {
+        general: { fontSize: 16, language: 'zh-CN', themeMode: 'dark' },
+      } as UserSettings;
+      await userModel.updateSetting(userId, newSettings);
+
+      const updatedSettings = await serverDB.query.userSettings.findFirst({
+        where: eq(users.id, userId),
+      });
+      expect(updatedSettings?.general).toEqual(newSettings.general);
+    });
   });
 
   describe('updatePreference', () => {
@@ -183,4 +201,21 @@ describe('UserModel', () => {
       expect(updatedUser?.preference).toEqual({ ...preference, ...newPreference });
     });
   });
+
+  describe('updateGuide', () => {
+    it('should update user guide', async () => {
+      const preference = { guide: { topic: false } } as UserGuide;
+      await serverDB.insert(users).values({ id: userId, preference });
+
+      const newGuide: Partial<UserGuide> = {
+        topic: true,
+        moveSettingsToAvatar: true,
+        uploadFileInKnowledgeBase: true,
+      };
+      await userModel.updateGuide(userId, newGuide);
+
+      const updatedUser = await serverDB.query.users.findFirst({ where: eq(users.id, userId) });
+      expect(updatedUser?.preference).toEqual({ ...preference, guide: newGuide });
+    });
+  });
 });

package/src/database/server/models/chunk.ts

@@ -1,5 +1,5 @@
 import { asc, cosineDistance, count, eq, inArray, sql } from 'drizzle-orm';
-import { and, desc } from 'drizzle-orm/expressions';
+import { and, desc, isNull } from 'drizzle-orm/expressions';
 
 import { serverDB } from '@/database/server';
 import { ChunkMetadata, FileChunk, SemanticSearchChunk } from '@/types/chunk';
@@ -43,6 +43,19 @@ export class ChunkModel {
     return serverDB.delete(chunks).where(and(eq(chunks.id, id), eq(chunks.userId, this.userId)));
   };
 
+  deleteOrphanChunks = async () => {
+    const orphanedChunks = await serverDB
+      .select({ chunkId: chunks.id })
+      .from(chunks)
+      .leftJoin(fileChunks, eq(chunks.id, fileChunks.chunkId))
+      .where(isNull(fileChunks.fileId));
+
+    const ids = orphanedChunks.map((chunk) => chunk.chunkId);
+    if (ids.length === 0) return;
+
+    await serverDB.delete(chunks).where(inArray(chunks.id, ids));
+  };
+
   findById = async (id: string) => {
     return serverDB.query.chunks.findFirst({
       where: and(eq(chunks.id, id)),

package/src/database/server/models/file.ts

@@ -1,6 +1,7 @@
 import { asc, count, eq, ilike, inArray, notExists } from 'drizzle-orm';
 import { and, desc } from 'drizzle-orm/expressions';
 
+import { serverDBEnv } from '@/config/db';
 import { serverDB } from '@/database/server/core/db';
 import { FilesTabs, QueryFileListParams, SortType } from '@/types/files';
 
@@ -77,7 +78,8 @@ export class FileModel {
       const fileCount = result[0].count;
 
       // delete the file from global file if it is not used by other files
-      if (fileCount === 0) {
+      // if `DISABLE_REMOVE_GLOBAL_FILE` is true, we will not remove the global file
+      if (fileCount === 0 && !serverDBEnv.DISABLE_REMOVE_GLOBAL_FILE) {
         await trx.delete(globalFiles).where(eq(globalFiles.hashId, fileHash));
 
         return file;
@@ -118,7 +120,7 @@ export class FileModel {
 
       const needToDeleteList = fileHashCounts.filter((item) => item.count === 0);
 
-      if (needToDeleteList.length === 0) return;
+      if (needToDeleteList.length === 0 || serverDBEnv.DISABLE_REMOVE_GLOBAL_FILE) return;
 
       // delete the file from global file if it is not used by other files
       await trx.delete(globalFiles).where(

package/src/server/routers/lambda/file.ts

@@ -150,6 +150,8 @@ export const fileRouter = router({
   removeFile: fileProcedure.input(z.object({ id: z.string() })).mutation(async ({ input, ctx }) => {
     const file = await ctx.fileModel.delete(input.id);
 
+    // delete the orphan chunks
+    await ctx.chunkModel.deleteOrphanChunks();
     if (!file) return;
 
     // delele the file from remove from S3 if it is not used by other files
@@ -162,6 +164,9 @@ export const fileRouter = router({
     .mutation(async ({ input, ctx }) => {
       const needToRemoveFileList = await ctx.fileModel.deleteMany(input.ids);
 
+      // delete the orphan chunks
+      await ctx.chunkModel.deleteOrphanChunks();
+
       if (!needToRemoveFileList || needToRemoveFileList.length === 0) return;
 
       // remove from S3