@rws-framework/ai-tools 3.9.2 → 3.10.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@rws-framework/ai-tools",
   "private": false,
-  "version": "3.9.2",
+  "version": "3.10.0",
   "description": "",
   "main": "src/index.ts",
   "scripts": {},

package/src/services/LangChainEmbeddingService.ts

@@ -168,16 +168,7 @@ export class LangChainEmbeddingService {
         return TextChunker.chunkText(text, maxTokens, overlap, separators);
     }
 
-    async chunkCSV(rows: Record<string, any>[], ragOverride?: IChunkConfig): Promise<Document[]> {        
-            // Use safe token limits - the TextChunker handles token estimation internally
-        const maxTokens = ragOverride ? ragOverride.chunkSize : (this.chunkConfig?.chunkSize || 450); // Safe token limit for embedding models
-        const overlap = ragOverride ? ragOverride.chunkOverlap : (this.chunkConfig?.chunkOverlap || 50); // Character overlap, not token
-
-        const splitter = new RecursiveCharacterTextSplitter({
-            chunkSize: maxTokens,
-            chunkOverlap: overlap
-        });       
-
+    async chunkCSV(rows: Record<string, any>[], ragOverride?: IChunkConfig): Promise<Document[]> {                                 
         const docs = rows.map((row, i) => {
             const text = Object.entries(row)
             .map(([k, v]) => `${k}: ${v}`)
@@ -189,7 +180,7 @@ export class LangChainEmbeddingService {
             });
         });
 
-        return await splitter.splitDocuments(docs);
+        return docs;
     }
 
     /**