@lobehub/chat 1.55.4 → 1.56.0

package/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 # Changelog
 
+## [Version 1.56.0](https://github.com/lobehub/lobe-chat/compare/v1.55.4...v1.56.0)
+
+<sup>Released on **2025-02-15**</sup>
+
+#### ✨ Features
+
+- **misc**: Add configurable PDF processing method with Unstructured.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's improved
+
+- **misc**: Add configurable PDF processing method with Unstructured, closes [#5927](https://github.com/lobehub/lobe-chat/issues/5927) ([35fa3ee](https://github.com/lobehub/lobe-chat/commit/35fa3ee))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 1.55.4](https://github.com/lobehub/lobe-chat/compare/v1.55.3...v1.55.4)
 
 <sup>Released on **2025-02-15**</sup>

package/changelog/v1.json

@@ -1,4 +1,13 @@
 [
+  {
+    "children": {
+      "features": [
+        "Add configurable PDF processing method with Unstructured."
+      ]
+    },
+    "date": "2025-02-15",
+    "version": "1.56.0"
+  },
   {
     "children": {
       "improvements": [

package/docker-compose/local/docker-compose.yml

@@ -112,18 +112,34 @@ services:
           echo '⚠️Warining: Unable to fetch OIDC configuration from Casdoor'
           echo 'Request URL: ${AUTH_CASDOOR_ISSUER}/.well-known/openid-configuration'
           echo 'Read more at: https://lobehub.com/docs/self-hosting/server-database/docker-compose#necessary-configuration'
+          echo ''
+          echo '⚠️注意：无法从 Casdoor 获取 OIDC 配置'
+          echo '请求 URL: ${AUTH_CASDOOR_ISSUER}/.well-known/openid-configuration'
+          echo '了解更多：https://lobehub.com/zh/docs/self-hosting/server-database/docker-compose#necessary-configuration'
+          echo ''
         else
           if ! wget -O - --timeout=5 ${AUTH_CASDOOR_ISSUER}/.well-known/openid-configuration 2>&1 | grep 'issuer' | grep ${AUTH_CASDOOR_ISSUER}; then
             printf '❌Error: The Auth issuer is conflict, Issuer in OIDC configuration is: %s' \$(wget -O - --timeout=5 ${AUTH_CASDOOR_ISSUER}/.well-known/openid-configuration 2>&1 | grep -E 'issuer.*' | awk -F '\"' '{print \$4}')
             echo ' , but the issuer in .env file is: ${AUTH_CASDOOR_ISSUER} '
             echo 'Request URL: ${AUTH_CASDOOR_ISSUER}/.well-known/openid-configuration'
             echo 'Read more at: https://lobehub.com/docs/self-hosting/server-database/docker-compose#necessary-configuration'
+            echo ''
+            printf '❌错误：Auth 的 issuer 冲突，OIDC 配置中的 issuer 是：%s' \$(wget -O - --timeout=5 ${AUTH_CASDOOR_ISSUER}/.well-known/openid-configuration 2>&1 | grep -E 'issuer.*' | awk -F '\"' '{print \$4}')
+            echo ' , 但 .env 文件中的 issuer 是：${AUTH_CASDOOR_ISSUER} '
+            echo '请求 URL: ${AUTH_CASDOOR_ISSUER}/.well-known/openid-configuration'
+            echo '了解更多：https://lobehub.com/zh/docs/self-hosting/server-database/docker-compose#necessary-configuration'
+            echo ''
           fi
         fi
         if [ $(wget --timeout=5 --spider --server-response ${S3_ENDPOINT}/minio/health/live 2>&1 | grep -c 'HTTP/1.1 200 OK') -eq 0 ]; then
           echo '⚠️Warining: Unable to fetch MinIO health status'
           echo 'Request URL: ${S3_ENDPOINT}/minio/health/live'
           echo 'Read more at: https://lobehub.com/docs/self-hosting/server-database/docker-compose#necessary-configuration'
+          echo ''
+          echo '⚠️注意：无法获取 MinIO 健康状态'
+          echo '请求 URL: ${S3_ENDPOINT}/minio/health/live'
+          echo '了解更多：https://lobehub.com/zh/docs/self-hosting/server-database/docker-compose#necessary-configuration'
+          echo ''
         fi
         wait \$LOBE_PID
       "

package/docs/self-hosting/advanced/auth/next-auth/casdoor.mdx

@@ -109,7 +109,7 @@ If you are deploying using a public network, the following assumptions apply:
 
   Go to `Admin` -> `Webhooks`, add a webhook, and fill in the following fields:
 
-  - URL: `https://lobe.example.com/api/auth/webhooks/casdoor`
+  - URL: `https://lobe.example.com/api/webhooks/casdoor`
   - Method: `POST`
   - Content Type: `application/json`
   - Headers: `casdoor-secret`: `Your Webhook Secret`

package/docs/self-hosting/advanced/auth/next-auth/casdoor.zh-CN.mdx

@@ -98,7 +98,7 @@ tags:
 
   前往 `管理工具` -> `Webhooks`，创建一个 Webhook，添加一个 Webhook，填写以下字段：
 
-  - 链接：`http://lobe.example.com/api/auth/webhooks/casdoor`
+  - 链接：`http://lobe.example.com/api/webhooks/casdoor`
   - 方法：`POST`
   - 内容类型：`application/json`
   - 协议头：`casdoor-secret`: `你的Webhook密钥`

package/docs/self-hosting/advanced/auth/next-auth/logto.mdx

@@ -49,7 +49,7 @@ If you are using Logto Cloud, assume its endpoint domain is `https://example.log
 
   Go to `Webhooks`, create a Webhook, and fill in the following fields:
 
-  - Endpoint URL: `https://lobe.example.com/api/auth/webhooks/logto`
+  - Endpoint URL: `https://lobe.example.com/api/webhooks/logto`
   - Events: `User.Data.Updated`
 
   After successful creation, copy the Webhook's `Signing Key` and fill it in the `LOGTO_WEBHOOK_SIGNING_KEY` environment variable.

package/docs/self-hosting/advanced/auth/next-auth/logto.zh-CN.mdx

@@ -46,7 +46,7 @@ tags:
 
   前往 `Webhooks` ，创建一个 Webhook，填写以下字段：
 
-  - 端点 URL： `https://lobe.example.com/api/auth/webhooks/logto`
+  - 端点 URL： `https://lobe.example.com/api/webhooks/logto`
   - 事件： `User.Data.Updated`
 
   创建成功后，复制 Webhook 的 `签名密钥`。填写到环境变量中的 `LOGTO_WEBHOOK_SIGNING_KEY`。

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.55.4",
+  "version": "1.56.0",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",

package/scripts/vercelIgnoredBuildStep.js

@@ -5,7 +5,7 @@ const branchName = process.env.VERCEL_GIT_COMMIT_REF || '';
 
 function shouldProceedBuild() {
   // 如果是 lighthouse 分支或以 testgru 开头的分支，取消构建
-  if (branchName === 'lighthouse' || branchName.startsWith('testgru')) {
+  if (branchName === 'lighthouse' || branchName.startsWith('gru/')) {
     return false;
   }
 

package/src/config/knowledge.ts

@@ -1,19 +1,17 @@
 import { createEnv } from '@t3-oss/env-nextjs';
 import { z } from 'zod';
 
-export const getKnowledgeConfig = () => {
-  return createEnv({
-    runtimeEnv: {
-      DEFAULT_FILES_CONFIG: process.env.DEFAULT_FILES_CONFIG,
-      UNSTRUCTURED_API_KEY: process.env.UNSTRUCTURED_API_KEY,
-      UNSTRUCTURED_SERVER_URL: process.env.UNSTRUCTURED_SERVER_URL,
-    },
-    server: {
-      DEFAULT_FILES_CONFIG: z.string().optional(),
-      UNSTRUCTURED_API_KEY: z.string().optional(),
-      UNSTRUCTURED_SERVER_URL: z.string().optional(),
-    },
-  });
-};
-
-export const knowledgeEnv = getKnowledgeConfig();
+export const knowledgeEnv = createEnv({
+  runtimeEnv: {
+    DEFAULT_FILES_CONFIG: process.env.DEFAULT_FILES_CONFIG,
+    FILE_TYPE_CHUNKING_RULES: process.env.FILE_TYPE_CHUNKING_RULES,
+    UNSTRUCTURED_API_KEY: process.env.UNSTRUCTURED_API_KEY,
+    UNSTRUCTURED_SERVER_URL: process.env.UNSTRUCTURED_SERVER_URL,
+  },
+  server: {
+    DEFAULT_FILES_CONFIG: z.string().optional(),
+    FILE_TYPE_CHUNKING_RULES: z.string().optional(),
+    UNSTRUCTURED_API_KEY: z.string().optional(),
+    UNSTRUCTURED_SERVER_URL: z.string().optional(),
+  },
+});

package/src/server/modules/ContentChunk/index.ts

@@ -1,9 +1,13 @@
 import { ChunkingLoader } from 'src/libs/langchain';
 import { Strategy } from 'unstructured-client/sdk/models/shared';
 
-import { NewChunkItem, NewUnstructuredChunkItem } from '@/database/schemas';
+import { knowledgeEnv } from '@/config/knowledge';
+import type { NewChunkItem, NewUnstructuredChunkItem } from '@/database/schemas';
 import { ChunkingStrategy, Unstructured } from '@/libs/unstructured';
 
+import { ChunkingRuleParser } from './rules';
+import type { ChunkingService } from './rules';
+
 export interface ChunkContentParams {
   content: Uint8Array;
   fileType: string;
@@ -19,23 +23,57 @@ interface ChunkResult {
 export class ContentChunk {
   private unstructuredClient: Unstructured;
   private langchainClient: ChunkingLoader;
+  private chunkingRules: Record<string, ChunkingService[]>;
 
   constructor() {
     this.unstructuredClient = new Unstructured();
     this.langchainClient = new ChunkingLoader();
+    this.chunkingRules = ChunkingRuleParser.parse(knowledgeEnv.FILE_TYPE_CHUNKING_RULES || '');
   }
 
-  isUsingUnstructured(params: ChunkContentParams) {
-    return params.fileType === 'application/pdf' && params.mode === 'hi-res';
+  private getChunkingServices(fileType: string): ChunkingService[] {
+    const ext = fileType.split('/').pop()?.toLowerCase() || '';
+    return this.chunkingRules[ext] || ['default'];
   }
 
   async chunkContent(params: ChunkContentParams): Promise<ChunkResult> {
-    if (this.isUsingUnstructured(params))
-      return await this.chunkByUnstructured(params.filename, params.content);
-
+    const services = this.getChunkingServices(params.fileType);
+
+    for (const service of services) {
+      try {
+        switch (service) {
+          case 'unstructured': {
+            if (this.canUseUnstructured()) {
+              return await this.chunkByUnstructured(params.filename, params.content);
+            }
+            break;
+          }
+
+          case 'doc2x': {
+            // Future implementation
+            break;
+          }
+
+          default: {
+            return await this.chunkByLangChain(params.filename, params.content);
+          }
+        }
+      } catch (error) {
+        // If this is the last service, throw the error
+        if (service === services.at(-1)) throw error;
+        // Otherwise continue to next service
+        console.error(`Chunking failed with service ${service}:`, error);
+      }
+    }
+
+    // Fallback to langchain if no service succeeded
     return await this.chunkByLangChain(params.filename, params.content);
   }
 
+  private canUseUnstructured(): boolean {
+    return !!(knowledgeEnv.UNSTRUCTURED_API_KEY && knowledgeEnv.UNSTRUCTURED_SERVER_URL);
+  }
+
   private chunkByUnstructured = async (
     filename: string,
     content: Uint8Array,

package/src/server/modules/ContentChunk/rules.test.ts

@@ -0,0 +1,81 @@
+import { describe, expect, it } from 'vitest';
+import { ChunkingRuleParser } from './rules';
+
+describe('ChunkingRuleParser', () => {
+  describe('parse', () => {
+    it('should parse a single file type rule correctly', () => {
+      const input = 'pdf=unstructured,default';
+      const result = ChunkingRuleParser.parse(input);
+
+      expect(result).toEqual({
+        pdf: ['unstructured', 'default'],
+      });
+    });
+
+    it('should parse multiple file type rules correctly', () => {
+      const input = 'pdf=unstructured,default;doc=doc2x,default;txt=default';
+      const result = ChunkingRuleParser.parse(input);
+
+      expect(result).toEqual({
+        pdf: ['unstructured', 'default'],
+        doc: ['doc2x', 'default'],
+        txt: ['default'],
+      });
+    });
+
+    it('should convert file types to lowercase', () => {
+      const input = 'PDF=unstructured;DOC=doc2x';
+      const result = ChunkingRuleParser.parse(input);
+
+      expect(result).toEqual({
+        pdf: ['unstructured'],
+        doc: ['doc2x'],
+      });
+    });
+
+    it('should filter out invalid service names', () => {
+      const input = 'pdf=unstructured,invalid,default,wrongservice';
+      const result = ChunkingRuleParser.parse(input);
+
+      expect(result).toEqual({
+        pdf: ['unstructured', 'default'],
+      });
+    });
+
+    it('should handle empty string input', () => {
+      const input = '';
+      const result = ChunkingRuleParser.parse(input);
+
+      expect(result).toEqual({});
+    });
+
+    it('should skip invalid rule formats', () => {
+      const input = 'pdf=unstructured;invalid;doc=doc2x;=default;txt';
+      const result = ChunkingRuleParser.parse(input);
+
+      expect(result).toEqual({
+        pdf: ['unstructured'],
+        doc: ['doc2x'],
+      });
+    });
+
+    it('should handle whitespace in service names', () => {
+      const input = 'pdf= unstructured , default ;doc=doc2x';
+      const result = ChunkingRuleParser.parse(input);
+
+      expect(result).toEqual({
+        pdf: ['unstructured', 'default'],
+        doc: ['doc2x'],
+      });
+    });
+
+    it('should handle duplicate services for same file type', () => {
+      const input = 'pdf=unstructured,default,unstructured';
+      const result = ChunkingRuleParser.parse(input);
+
+      expect(result).toEqual({
+        pdf: ['unstructured', 'default', 'unstructured'],
+      });
+    });
+  });
+});

package/src/server/modules/ContentChunk/rules.ts

@@ -0,0 +1,23 @@
+export type ChunkingService = 'unstructured' | 'doc2x' | 'default';
+
+export const ChunkingRuleParser = {
+  parse(rulesStr: string): Record<string, ChunkingService[]> {
+    const rules: Record<string, ChunkingService[]> = {};
+
+    // Split by semicolon for different file types
+    const fileTypeRules = rulesStr.split(';');
+
+    for (const rule of fileTypeRules) {
+      const [fileType, services] = rule.split('=');
+      if (!fileType || !services) continue;
+
+      // Split services by comma and validate each service
+      rules[fileType.toLowerCase()] = services
+        .split(',')
+        .map((s) => s.trim().toLowerCase())
+        .filter((s): s is ChunkingService => ['unstructured', 'doc2x', 'default'].includes(s));
+    }
+
+    return rules;
+  },
+} as const;