@lobehub/chat 1.36.22 → 1.36.23

package/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 # Changelog
 
+### [Version 1.36.23](https://github.com/lobehub/lobe-chat/compare/v1.36.22...v1.36.23)
+
+<sup>Released on **2024-12-13**</sup>
+
+#### 💄 Styles
+
+- **misc**: Support csv chunking.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### Styles
+
+- **misc**: Support csv chunking, closes [#4123](https://github.com/lobehub/lobe-chat/issues/4123) ([1b993a8](https://github.com/lobehub/lobe-chat/commit/1b993a8))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 1.36.22](https://github.com/lobehub/lobe-chat/compare/v1.36.21...v1.36.22)
 
 <sup>Released on **2024-12-13**</sup>

package/changelog/v1.json

@@ -1,4 +1,13 @@
 [
+  {
+    "children": {
+      "improvements": [
+        "Support csv chunking."
+      ]
+    },
+    "date": "2024-12-13",
+    "version": "1.36.23"
+  },
   {
     "children": {},
     "date": "2024-12-13",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.36.22",
+  "version": "1.36.23",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",

package/src/libs/langchain/loaders/csv/__tests__/__snapshots__/index.test.ts.snap

@@ -0,0 +1,422 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`CSVLoader > should run 1`] = `
+[
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 1,
+      "source": "blob",
+    },
+    "pageContent": ": 1
+Hair: Black
+Eye: Brown
+Sex: Male
+Freq: 32",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 2,
+      "source": "blob",
+    },
+    "pageContent": ": 2
+Hair: Brown
+Eye: Brown
+Sex: Male
+Freq: 53",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 3,
+      "source": "blob",
+    },
+    "pageContent": ": 3
+Hair: Red
+Eye: Brown
+Sex: Male
+Freq: 10",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 4,
+      "source": "blob",
+    },
+    "pageContent": ": 4
+Hair: Blond
+Eye: Brown
+Sex: Male
+Freq: 3",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 5,
+      "source": "blob",
+    },
+    "pageContent": ": 5
+Hair: Black
+Eye: Blue
+Sex: Male
+Freq: 11",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 6,
+      "source": "blob",
+    },
+    "pageContent": ": 6
+Hair: Brown
+Eye: Blue
+Sex: Male
+Freq: 50",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 7,
+      "source": "blob",
+    },
+    "pageContent": ": 7
+Hair: Red
+Eye: Blue
+Sex: Male
+Freq: 10",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 8,
+      "source": "blob",
+    },
+    "pageContent": ": 8
+Hair: Blond
+Eye: Blue
+Sex: Male
+Freq: 30",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 9,
+      "source": "blob",
+    },
+    "pageContent": ": 9
+Hair: Black
+Eye: Hazel
+Sex: Male
+Freq: 10",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 10,
+      "source": "blob",
+    },
+    "pageContent": ": 10
+Hair: Brown
+Eye: Hazel
+Sex: Male
+Freq: 25",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 11,
+      "source": "blob",
+    },
+    "pageContent": ": 11
+Hair: Red
+Eye: Hazel
+Sex: Male
+Freq: 7",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 12,
+      "source": "blob",
+    },
+    "pageContent": ": 12
+Hair: Blond
+Eye: Hazel
+Sex: Male
+Freq: 5",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 13,
+      "source": "blob",
+    },
+    "pageContent": ": 13
+Hair: Black
+Eye: Green
+Sex: Male
+Freq: 3",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 14,
+      "source": "blob",
+    },
+    "pageContent": ": 14
+Hair: Brown
+Eye: Green
+Sex: Male
+Freq: 15",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 15,
+      "source": "blob",
+    },
+    "pageContent": ": 15
+Hair: Red
+Eye: Green
+Sex: Male
+Freq: 7",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 16,
+      "source": "blob",
+    },
+    "pageContent": ": 16
+Hair: Blond
+Eye: Green
+Sex: Male
+Freq: 8",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 17,
+      "source": "blob",
+    },
+    "pageContent": ": 17
+Hair: Black
+Eye: Brown
+Sex: Female
+Freq: 36",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 18,
+      "source": "blob",
+    },
+    "pageContent": ": 18
+Hair: Brown
+Eye: Brown
+Sex: Female
+Freq: 66",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 19,
+      "source": "blob",
+    },
+    "pageContent": ": 19
+Hair: Red
+Eye: Brown
+Sex: Female
+Freq: 16",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 20,
+      "source": "blob",
+    },
+    "pageContent": ": 20
+Hair: Blond
+Eye: Brown
+Sex: Female
+Freq: 4",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 21,
+      "source": "blob",
+    },
+    "pageContent": ": 21
+Hair: Black
+Eye: Blue
+Sex: Female
+Freq: 9",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 22,
+      "source": "blob",
+    },
+    "pageContent": ": 22
+Hair: Brown
+Eye: Blue
+Sex: Female
+Freq: 34",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 23,
+      "source": "blob",
+    },
+    "pageContent": ": 23
+Hair: Red
+Eye: Blue
+Sex: Female
+Freq: 7",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 24,
+      "source": "blob",
+    },
+    "pageContent": ": 24
+Hair: Blond
+Eye: Blue
+Sex: Female
+Freq: 64",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 25,
+      "source": "blob",
+    },
+    "pageContent": ": 25
+Hair: Black
+Eye: Hazel
+Sex: Female
+Freq: 5",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 26,
+      "source": "blob",
+    },
+    "pageContent": ": 26
+Hair: Brown
+Eye: Hazel
+Sex: Female
+Freq: 29",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 27,
+      "source": "blob",
+    },
+    "pageContent": ": 27
+Hair: Red
+Eye: Hazel
+Sex: Female
+Freq: 7",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 28,
+      "source": "blob",
+    },
+    "pageContent": ": 28
+Hair: Blond
+Eye: Hazel
+Sex: Female
+Freq: 5",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 29,
+      "source": "blob",
+    },
+    "pageContent": ": 29
+Hair: Black
+Eye: Green
+Sex: Female
+Freq: 2",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 30,
+      "source": "blob",
+    },
+    "pageContent": ": 30
+Hair: Brown
+Eye: Green
+Sex: Female
+Freq: 14",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 31,
+      "source": "blob",
+    },
+    "pageContent": ": 31
+Hair: Red
+Eye: Green
+Sex: Female
+Freq: 7",
+  },
+  Document {
+    "id": undefined,
+    "metadata": {
+      "blobType": "",
+      "line": 32,
+      "source": "blob",
+    },
+    "pageContent": ": 32
+Hair: Blond
+Eye: Green
+Sex: Female
+Freq: 8",
+  },
+]
+`;

package/src/libs/langchain/loaders/csv/__tests__/demo.csv

@@ -0,0 +1,33 @@
+"","Hair","Eye","Sex","Freq"
+"1","Black","Brown","Male",32
+"2","Brown","Brown","Male",53
+"3","Red","Brown","Male",10
+"4","Blond","Brown","Male",3
+"5","Black","Blue","Male",11
+"6","Brown","Blue","Male",50
+"7","Red","Blue","Male",10
+"8","Blond","Blue","Male",30
+"9","Black","Hazel","Male",10
+"10","Brown","Hazel","Male",25
+"11","Red","Hazel","Male",7
+"12","Blond","Hazel","Male",5
+"13","Black","Green","Male",3
+"14","Brown","Green","Male",15
+"15","Red","Green","Male",7
+"16","Blond","Green","Male",8
+"17","Black","Brown","Female",36
+"18","Brown","Brown","Female",66
+"19","Red","Brown","Female",16
+"20","Blond","Brown","Female",4
+"21","Black","Blue","Female",9
+"22","Brown","Blue","Female",34
+"23","Red","Blue","Female",7
+"24","Blond","Blue","Female",64
+"25","Black","Hazel","Female",5
+"26","Brown","Hazel","Female",29
+"27","Red","Hazel","Female",7
+"28","Blond","Hazel","Female",5
+"29","Black","Green","Female",2
+"30","Brown","Green","Female",14
+"31","Red","Green","Female",7
+"32","Blond","Green","Female",8

package/src/libs/langchain/loaders/csv/__tests__/index.test.ts

@@ -0,0 +1,18 @@
+// @vitest-environment node
+import * as fs from 'node:fs';
+import { join } from 'node:path';
+import { expect } from 'vitest';
+
+import { CsVLoader } from '../index';
+
+describe('CSVLoader', () => {
+  it('should run', async () => {
+    const content = fs.readFileSync(join(__dirname, `./demo.csv`), 'utf-8');
+
+    const fileBlob = new Blob([Buffer.from(content)]);
+
+    const data = await CsVLoader(fileBlob);
+
+    expect(data).toMatchSnapshot();
+  });
+});

package/src/libs/langchain/loaders/csv/index.ts

@@ -0,0 +1,7 @@
+import { CSVLoader } from '@langchain/community/document_loaders/fs/csv';
+
+export const CsVLoader = async (fileBlob: Blob) => {
+  const loader = new CSVLoader(fileBlob);
+
+  return await loader.load();
+};

package/src/libs/langchain/loaders/index.ts

@@ -13,6 +13,7 @@ import { MarkdownLoader } from './markdown';
 import { PdfLoader } from './pdf';
 import { PPTXLoader } from './pptx';
 import { TextLoader } from './txt';
+import { CsVLoader} from './csv';
 
 class LangChainError extends Error {
   constructor(message: string) {
@@ -59,6 +60,10 @@ export class ChunkingLoader {
           return await TextLoader(txt);
         }
 
+        case 'csv': {
+          return await CsVLoader(fileBlob);
+        }
+
         default: {
           throw new Error(
             `Unsupported file type [${type}], please check your file is supported, or create report issue here: https://github.com/lobehub/lobe-chat/discussions/3550`,
@@ -91,6 +96,10 @@ export class ChunkingLoader {
       return 'markdown';
     }
 
+    if (filename.endsWith('csv') ) {
+      return 'csv';
+    }
+
     const ext = filename.split('.').pop();
 
     if (ext && SupportedTextSplitterLanguages.includes(ext as SupportedTextSplitterLanguage)) {

package/src/libs/langchain/types.ts

@@ -1 +1 @@
-export type LangChainLoaderType = 'code' | 'ppt' | 'pdf' | 'markdown' | 'doc' | 'text' | 'latex';
+export type LangChainLoaderType = 'code' | 'ppt' | 'pdf' | 'markdown' | 'doc' | 'text' | 'latex' | 'csv';