@lobehub/chat 1.36.22 → 1.36.24

package/CHANGELOG.md

@@ -2,6 +2,56 @@
 
 # Changelog
 
+### [Version 1.36.24](https://github.com/lobehub/lobe-chat/compare/v1.36.23...v1.36.24)
+
+<sup>Released on **2024-12-14**</sup>
+
+#### ♻ Code Refactoring
+
+- **misc**: Refactor file Url query in message model.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### Code refactoring
+
+- **misc**: Refactor file Url query in message model, closes [#5019](https://github.com/lobehub/lobe-chat/issues/5019) ([edf78f4](https://github.com/lobehub/lobe-chat/commit/edf78f4))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
+### [Version 1.36.23](https://github.com/lobehub/lobe-chat/compare/v1.36.22...v1.36.23)
+
+<sup>Released on **2024-12-13**</sup>
+
+#### 💄 Styles
+
+- **misc**: Support csv chunking.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### Styles
+
+- **misc**: Support csv chunking, closes [#4123](https://github.com/lobehub/lobe-chat/issues/4123) ([1b993a8](https://github.com/lobehub/lobe-chat/commit/1b993a8))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 1.36.22](https://github.com/lobehub/lobe-chat/compare/v1.36.21...v1.36.22)
 
 <sup>Released on **2024-12-13**</sup>

package/changelog/v1.json

@@ -1,4 +1,22 @@
 [
+  {
+    "children": {
+      "improvements": [
+        "Refactor file Url query in message model."
+      ]
+    },
+    "date": "2024-12-14",
+    "version": "1.36.24"
+  },
+  {
+    "children": {
+      "improvements": [
+        "Support csv chunking."
+      ]
+    },
+    "date": "2024-12-13",
+    "version": "1.36.23"
+  },
   {
     "children": {},
     "date": "2024-12-13",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.36.22",
+  "version": "1.36.24",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",

package/src/database/server/models/__tests__/message.test.ts

@@ -209,15 +209,19 @@ describe('MessageModel', () => {
         ]);
       });
 
+      const domain = 'http://abc.com';
       // 调用 query 方法
-      const result = await messageModel.query();
+      const result = await messageModel.query(
+        {},
+        { postProcessUrl: async (path) => `${domain}/${path}` },
+      );
 
       // 断言结果
       expect(result).toHaveLength(2);
       expect(result[0].id).toBe('1');
       expect(result[0].imageList).toEqual([
-        { alt: 'file-1', id: 'f-0', url: expect.stringContaining('/abc') },
-        { alt: 'file-3', id: 'f-3', url: expect.stringContaining('/abc') },
+        { alt: 'file-1', id: 'f-0', url: `${domain}/abc` },
+        { alt: 'file-3', id: 'f-3', url: `${domain}/abc` },
       ]);
 
       expect(result[1].id).toBe('2');

package/src/database/server/models/message.ts

@@ -3,7 +3,6 @@ import { and, asc, desc, eq, gte, inArray, isNull, like, lt } from 'drizzle-orm/
 
 import { LobeChatDatabase } from '@/database/type';
 import { idGenerator } from '@/database/utils/idGenerator';
-import { getFullFileUrl } from '@/server/utils/files';
 import {
   ChatFileItem,
   ChatImageItem,
@@ -47,12 +46,10 @@ export class MessageModel {
   }
 
   // **************** Query *************** //
-  async query({
-    current = 0,
-    pageSize = 1000,
-    sessionId,
-    topicId,
-  }: QueryMessageParams = {}): Promise<MessageItem[]> {
+  async query(
+    { current = 0, pageSize = 1000, sessionId, topicId }: QueryMessageParams = {},
+    options: { postProcessUrl?: (path: string | null) => Promise<string> } = {},
+  ): Promise<MessageItem[]> {
     const offset = current * pageSize;
 
     // 1. get basic messages
@@ -133,7 +130,7 @@ export class MessageModel {
     const relatedFileList = await Promise.all(
       rawRelatedFileList.map(async (file) => ({
         ...file,
-        url: await getFullFileUrl(file.url),
+        url: options.postProcessUrl ? await options.postProcessUrl(file.url) : (file.url as string),
       })),
     );
 

package/src/libs/langchain/loaders/csv/__tests__/__snapshots__/index.test.ts.snap

@@ -0,0 +1,422 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`CSVLoader > should run 1`] = `
+[
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 1,
+      "source": "blob",
+    },
+    "pageContent": ": 1
+Hair: Black
+Eye: Brown
+Sex: Male
+Freq: 32",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 2,
+      "source": "blob",
+    },
+    "pageContent": ": 2
+Hair: Brown
+Eye: Brown
+Sex: Male
+Freq: 53",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 3,
+      "source": "blob",
+    },
+    "pageContent": ": 3
+Hair: Red
+Eye: Brown
+Sex: Male
+Freq: 10",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 4,
+      "source": "blob",
+    },
+    "pageContent": ": 4
+Hair: Blond
+Eye: Brown
+Sex: Male
+Freq: 3",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 5,
+      "source": "blob",
+    },
+    "pageContent": ": 5
+Hair: Black
+Eye: Blue
+Sex: Male
+Freq: 11",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 6,
+      "source": "blob",
+    },
+    "pageContent": ": 6
+Hair: Brown
+Eye: Blue
+Sex: Male
+Freq: 50",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 7,
+      "source": "blob",
+    },
+    "pageContent": ": 7
+Hair: Red
+Eye: Blue
+Sex: Male
+Freq: 10",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 8,
+      "source": "blob",
+    },
+    "pageContent": ": 8
+Hair: Blond
+Eye: Blue
+Sex: Male
+Freq: 30",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 9,
+      "source": "blob",
+    },
+    "pageContent": ": 9
+Hair: Black
+Eye: Hazel
+Sex: Male
+Freq: 10",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 10,
+      "source": "blob",
+    },
+    "pageContent": ": 10
+Hair: Brown
+Eye: Hazel
+Sex: Male
+Freq: 25",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 11,
+      "source": "blob",
+    },
+    "pageContent": ": 11
+Hair: Red
+Eye: Hazel
+Sex: Male
+Freq: 7",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 12,
+      "source": "blob",
+    },
+    "pageContent": ": 12
+Hair: Blond
+Eye: Hazel
+Sex: Male
+Freq: 5",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 13,
+      "source": "blob",
+    },
+    "pageContent": ": 13
+Hair: Black
+Eye: Green
+Sex: Male
+Freq: 3",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 14,
+      "source": "blob",
+    },
+    "pageContent": ": 14
+Hair: Brown
+Eye: Green
+Sex: Male
+Freq: 15",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 15,
+      "source": "blob",
+    },
+    "pageContent": ": 15
+Hair: Red
+Eye: Green
+Sex: Male
+Freq: 7",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 16,
+      "source": "blob",
+    },
+    "pageContent": ": 16
+Hair: Blond
+Eye: Green
+Sex: Male
+Freq: 8",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 17,
+      "source": "blob",
+    },
+    "pageContent": ": 17
+Hair: Black
+Eye: Brown
+Sex: Female
+Freq: 36",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 18,
+      "source": "blob",
+    },
+    "pageContent": ": 18
+Hair: Brown
+Eye: Brown
+Sex: Female
+Freq: 66",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 19,
+      "source": "blob",
+    },
+    "pageContent": ": 19
+Hair: Red
+Eye: Brown
+Sex: Female
+Freq: 16",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 20,
+      "source": "blob",
+    },
+    "pageContent": ": 20
+Hair: Blond
+Eye: Brown
+Sex: Female
+Freq: 4",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 21,
+      "source": "blob",
+    },
+    "pageContent": ": 21
+Hair: Black
+Eye: Blue
+Sex: Female
+Freq: 9",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 22,
+      "source": "blob",
+    },
+    "pageContent": ": 22
+Hair: Brown
+Eye: Blue
+Sex: Female
+Freq: 34",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 23,
+      "source": "blob",
+    },
+    "pageContent": ": 23
+Hair: Red
+Eye: Blue
+Sex: Female
+Freq: 7",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 24,
+      "source": "blob",
+    },
+    "pageContent": ": 24
+Hair: Blond
+Eye: Blue
+Sex: Female
+Freq: 64",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 25,
+      "source": "blob",
+    },
+    "pageContent": ": 25
+Hair: Black
+Eye: Hazel
+Sex: Female
+Freq: 5",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 26,
+      "source": "blob",
+    },
+    "pageContent": ": 26
+Hair: Brown
+Eye: Hazel
+Sex: Female
+Freq: 29",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 27,
+      "source": "blob",
+    },
+    "pageContent": ": 27
+Hair: Red
+Eye: Hazel
+Sex: Female
+Freq: 7",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 28,
+      "source": "blob",
+    },
+    "pageContent": ": 28
+Hair: Blond
+Eye: Hazel
+Sex: Female
+Freq: 5",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 29,
+      "source": "blob",
+    },
+    "pageContent": ": 29
+Hair: Black
+Eye: Green
+Sex: Female
+Freq: 2",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 30,
+      "source": "blob",
+    },
+    "pageContent": ": 30
+Hair: Brown
+Eye: Green
+Sex: Female
+Freq: 14",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 31,
+      "source": "blob",
+    },
+    "pageContent": ": 31
+Hair: Red
+Eye: Green
+Sex: Female
+Freq: 7",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 32,
+      "source": "blob",
+    },
+    "pageContent": ": 32
+Hair: Blond
+Eye: Green
+Sex: Female
+Freq: 8",
+  },
+]
+`;

package/src/libs/langchain/loaders/csv/__tests__/demo.csv

@@ -0,0 +1,33 @@
+"","Hair","Eye","Sex","Freq"
+"1","Black","Brown","Male",32
+"2","Brown","Brown","Male",53
+"3","Red","Brown","Male",10
+"4","Blond","Brown","Male",3
+"5","Black","Blue","Male",11
+"6","Brown","Blue","Male",50
+"7","Red","Blue","Male",10
+"8","Blond","Blue","Male",30
+"9","Black","Hazel","Male",10
+"10","Brown","Hazel","Male",25
+"11","Red","Hazel","Male",7
+"12","Blond","Hazel","Male",5
+"13","Black","Green","Male",3
+"14","Brown","Green","Male",15
+"15","Red","Green","Male",7
+"16","Blond","Green","Male",8
+"17","Black","Brown","Female",36
+"18","Brown","Brown","Female",66
+"19","Red","Brown","Female",16
+"20","Blond","Brown","Female",4
+"21","Black","Blue","Female",9
+"22","Brown","Blue","Female",34
+"23","Red","Blue","Female",7
+"24","Blond","Blue","Female",64
+"25","Black","Hazel","Female",5
+"26","Brown","Hazel","Female",29
+"27","Red","Hazel","Female",7
+"28","Blond","Hazel","Female",5
+"29","Black","Green","Female",2
+"30","Brown","Green","Female",14
+"31","Red","Green","Female",7
+"32","Blond","Green","Female",8

package/src/libs/langchain/loaders/csv/__tests__/index.test.ts

@@ -0,0 +1,18 @@
+// @vitest-environment node
+import * as fs from 'node:fs';
+import { join } from 'node:path';
+import { expect } from 'vitest';
+
+import { CsVLoader } from '../index';
+
+describe('CSVLoader', () => {
+  it('should run', async () => {
+    const content = fs.readFileSync(join(__dirname, `./demo.csv`), 'utf-8');
+
+    const fileBlob = new Blob([Buffer.from(content)]);
+
+    const data = await CsVLoader(fileBlob);
+
+    expect(data).toMatchSnapshot();
+  });
+});

package/src/libs/langchain/loaders/csv/index.ts

@@ -0,0 +1,7 @@
+import { CSVLoader } from '@langchain/community/document_loaders/fs/csv';
+
+export const CsVLoader = async (fileBlob: Blob) => {
+  const loader = new CSVLoader(fileBlob);
+
+  return await loader.load();
+};

package/src/libs/langchain/loaders/index.ts

@@ -13,6 +13,7 @@ import { MarkdownLoader } from './markdown';
 import { PdfLoader } from './pdf';
 import { PPTXLoader } from './pptx';
 import { TextLoader } from './txt';
+import { CsVLoader} from './csv';
 
 class LangChainError extends Error {
   constructor(message: string) {
@@ -59,6 +60,10 @@ export class ChunkingLoader {
           return await TextLoader(txt);
         }
 
+        case 'csv': {
+          return await CsVLoader(fileBlob);
+        }
+
         default: {
           throw new Error(
             `Unsupported file type [${type}], please check your file is supported, or create report issue here: https://github.com/lobehub/lobe-chat/discussions/3550`,
@@ -91,6 +96,10 @@ export class ChunkingLoader {
       return 'markdown';
     }
 
+    if (filename.endsWith('csv') ) {
+      return 'csv';
+    }
+
     const ext = filename.split('.').pop();
 
     if (ext && SupportedTextSplitterLanguages.includes(ext as SupportedTextSplitterLanguage)) {

package/src/libs/langchain/types.ts

@@ -1 +1 @@
-export type LangChainLoaderType = 'code' | 'ppt' | 'pdf' | 'markdown' | 'doc' | 'text' | 'latex';
+export type LangChainLoaderType = 'code' | 'ppt' | 'pdf' | 'markdown' | 'doc' | 'text' | 'latex' | 'csv';

package/src/server/routers/lambda/message.ts

@@ -1,9 +1,10 @@
 import { z } from 'zod';
 
+import { updateMessagePluginSchema } from '@/database/schemas';
 import { serverDB } from '@/database/server';
 import { MessageModel } from '@/database/server/models/message';
-import { updateMessagePluginSchema } from '@/database/schemas';
 import { authedProcedure, publicProcedure, router } from '@/libs/trpc';
+import { getFullFileUrl } from '@/server/utils/files';
 import { ChatMessage } from '@/types/message';
 import { BatchTaskResult } from '@/types/service';
 
@@ -70,7 +71,7 @@ export const messageRouter = router({
 
       const messageModel = new MessageModel(serverDB, ctx.userId);
 
-      return messageModel.query(input);
+      return messageModel.query(input, { postProcessUrl: (path) => getFullFileUrl(path) });
     }),
 
   removeAllMessages: messageProcedure.mutation(async ({ ctx }) => {