@rws-framework/ai-tools 3.9.1 → 3.10.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@rws-framework/ai-tools",
   "private": false,
-  "version": "3.9.1",
+  "version": "3.10.0",
   "description": "",
   "main": "src/index.ts",
   "scripts": {},

package/src/services/LangChainEmbeddingService.ts

@@ -87,14 +87,16 @@ export class LangChainEmbeddingService {
         this.ensureInitialized();
         
         if (this.config.rateLimiting) {
+            let doneFiles = 0;
             return await this.rateLimitingService.executeWithRateLimit(
                 docs,
                 async (batch: Document[]) => {
                     const embeddings = await this.embeddings.embedDocuments(batch.map(d => d.pageContent));
 
                     if(batchCallback){
-                        const fragments = batch.map(d => d.pageContent);
-                        const percentage = (batch.length / docs.length) * 100;
+                        const fragments = batch.map(d => d.pageContent);                        
+                        doneFiles += batch.length;
+                        const percentage = (doneFiles / docs.length) * 100;
                         await batchCallback(fragments, embeddings, percentage);
                     }
 
@@ -109,14 +111,15 @@ export class LangChainEmbeddingService {
 
     async embedTexts(texts: string[], batchCallback?: (fragments:string[], batch: number[][], percentage: number) => Promise<void>): Promise<number[][]> {
         this.ensureInitialized();
-        
+        let doneTexts = 0;
         if (this.config.rateLimiting) {
             return await this.rateLimitingService.executeWithRateLimit(
                 texts,
                 async (batch: string[]) => {
                     const embeddings = await this.embeddings.embedDocuments(batch);
                     if (batchCallback) {
-                        const percentage = (batch.length / texts.length) * 100;
+                        doneTexts += batch.length;
+                        const percentage = (doneTexts / texts.length) * 100;
                         await batchCallback(batch, embeddings, percentage);
                     }
                     return embeddings;
@@ -165,16 +168,7 @@ export class LangChainEmbeddingService {
         return TextChunker.chunkText(text, maxTokens, overlap, separators);
     }
 
-    async chunkCSV(rows: Record<string, any>[], ragOverride?: IChunkConfig): Promise<Document[]> {        
-            // Use safe token limits - the TextChunker handles token estimation internally
-        const maxTokens = ragOverride ? ragOverride.chunkSize : (this.chunkConfig?.chunkSize || 450); // Safe token limit for embedding models
-        const overlap = ragOverride ? ragOverride.chunkOverlap : (this.chunkConfig?.chunkOverlap || 50); // Character overlap, not token
-
-        const splitter = new RecursiveCharacterTextSplitter({
-            chunkSize: maxTokens,
-            chunkOverlap: overlap
-        });       
-
+    async chunkCSV(rows: Record<string, any>[], ragOverride?: IChunkConfig): Promise<Document[]> {                                 
         const docs = rows.map((row, i) => {
             const text = Object.entries(row)
             .map(([k, v]) => `${k}: ${v}`)
@@ -186,7 +180,7 @@ export class LangChainEmbeddingService {
             });
         });
 
-        return await splitter.splitDocuments(docs);
+        return docs;
     }
 
     /**