@rws-framework/ai-tools 3.7.0 → 3.9.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@rws-framework/ai-tools",
   "private": false,
-  "version": "3.7.0",
+  "version": "3.9.0",
   "description": "",
   "main": "src/index.ts",
   "scripts": {},

package/src/services/LangChainEmbeddingService.ts

@@ -7,7 +7,7 @@ import { IEmbeddingConfig, IChunkConfig } from '../types';
 import { TextChunker } from './TextChunker';
 import RWSVectorStore, { VectorDocType, IVectorStoreConfig } from '../models/convo/VectorStore';
 import { OpenAIRateLimitingService } from './OpenAIRateLimitingService';
-
+import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
 @Injectable()
 export class LangChainEmbeddingService {
     private embeddings: Embeddings;
@@ -83,17 +83,41 @@ export class LangChainEmbeddingService {
         // This method is kept for compatibility but doesn't initialize anything
     }
 
-        /**
-     * Generate embeddings for multiple texts with sophisticated rate limiting
-     */
-    async embedTexts(texts: string[]): Promise<number[][]> {
+     async embedDocs(docs: Document[], batchCallback?: (fragments:string[], batch: number[][]) => Promise<void>): Promise<number[][]> {
+        this.ensureInitialized();
+        
+        if (this.config.rateLimiting) {
+            return await this.rateLimitingService.executeWithRateLimit(
+                docs,
+                async (batch: Document[]) => {
+                    const embeddings = await this.embeddings.embedDocuments(batch.map(d => d.pageContent));
+
+                    if(batchCallback){
+                        const fragments = batch.map(d => d.pageContent);
+                        await batchCallback(fragments, embeddings);
+                    }
+
+                    return embeddings;
+                },
+                (doc: Document) => doc.pageContent
+            );
+        }
+        
+        return await this.embeddings.embedDocuments(docs.map(d => d.pageContent));
+    }
+
+    async embedTexts(texts: string[], batchCallback?: (fragments:string[], batch: number[][]) => Promise<void>): Promise<number[][]> {
         this.ensureInitialized();
         
         if (this.config.rateLimiting) {
             return await this.rateLimitingService.executeWithRateLimit(
                 texts,
                 async (batch: string[]) => {
-                    return await this.embeddings.embedDocuments(batch);
+                    const embeddings = await this.embeddings.embedDocuments(batch);
+                    if (batchCallback) {
+                        await batchCallback(batch, embeddings);
+                    }
+                    return embeddings;
                 },
                 (text: string) => text // Token extractor
             );
@@ -127,15 +151,40 @@ export class LangChainEmbeddingService {
     /**
      * Split text into chunks
      */
-    async chunkText(text: string): Promise<string[]> {
+    async chunkText(text: string, ragOverride?: IChunkConfig): Promise<string[]> {
         this.ensureInitialized();
         
         // Use our custom TextChunker instead of LangChain's splitter
         // Use safe token limits - the TextChunker handles token estimation internally
-        const maxTokens = this.chunkConfig?.chunkSize || 450; // Safe token limit for embedding models
-        const overlap = this.chunkConfig?.chunkOverlap || 50; // Character overlap, not token
+        const maxTokens = ragOverride ? ragOverride.chunkSize : (this.chunkConfig?.chunkSize || 450); // Safe token limit for embedding models
+        const overlap = ragOverride ? ragOverride.chunkOverlap : (this.chunkConfig?.chunkOverlap || 50); // Character overlap, not token
+        const separators = ragOverride?.separators || this.chunkConfig?.separators || TextChunker.DEFAULT_SEPARATORS; // Default separators
         
-        return TextChunker.chunkText(text, maxTokens, overlap);
+        return TextChunker.chunkText(text, maxTokens, overlap, separators);
+    }
+
+    async chunkCSV(rows: Record<string, any>[], ragOverride?: IChunkConfig): Promise<Document[]> {        
+            // Use safe token limits - the TextChunker handles token estimation internally
+        const maxTokens = ragOverride ? ragOverride.chunkSize : (this.chunkConfig?.chunkSize || 450); // Safe token limit for embedding models
+        const overlap = ragOverride ? ragOverride.chunkOverlap : (this.chunkConfig?.chunkOverlap || 50); // Character overlap, not token
+
+        const splitter = new RecursiveCharacterTextSplitter({
+            chunkSize: maxTokens,
+            chunkOverlap: overlap
+        });       
+
+        const docs = rows.map((row, i) => {
+            const text = Object.entries(row)
+            .map(([k, v]) => `${k}: ${v}`)
+            .join("\n");
+
+            return new Document({
+                pageContent: text,
+                metadata: { row: i }
+            });
+        });
+
+        return await splitter.splitDocuments(docs);
     }
 
     /**
@@ -205,6 +254,7 @@ export class LangChainEmbeddingService {
 
         return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
     }
+    
 
     /**
      * Ensure the service is initialized

package/src/services/LangChainRAGService.ts

@@ -43,6 +43,18 @@ export class LangChainRAGService {
     private isInitialized = false;
     private logger?: any; // Optional logger interface
 
+    static SheetMimeType: string[] = [
+        'text/csv',
+        'text/tab-separated-values',
+        'application/vnd.ms-excel',
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.template',
+        'application/vnd.ms-excel.sheet.macroEnabled.12',
+        'application/vnd.ms-excel.sheet.binary.macroEnabled.12',
+        'application/vnd.oasis.opendocument.spreadsheet',
+        'application/vnd.google-apps.spreadsheet',
+    ];
+
     constructor(
         private embeddingService: LangChainEmbeddingService,
         private vectorSearchService: OptimizedVectorSearchService
@@ -84,44 +96,57 @@ export class LangChainRAGService {
      */
     async indexKnowledge(
         fileId: string | number,
-        content: string,
-        metadata: Record<string, any> = {}
-    ): Promise<IRAGResponse<{ chunkIds: string[] }>> {
+        content: string | Record<string, any>[],
+        metadata: Record<string, any> = {},
+        batchCallback?: (fragments:string[], batch: number[][]) => Promise<void>,
+        ragOverride?: IChunkConfig
+    ): Promise<IRAGResponse<{ chunkCount: number }>> {
         this.log('log', `[INDEXING] Starting indexKnowledge for fileId: ${fileId}`);
-        this.log('debug', `[INDEXING] Content length: ${content.length} characters`);
+        this.log('debug', `[INDEXING] Content length: ${Array.isArray(content) ? content.map(r => Object.values(r).join(' ')).join('\n').length : content.length} characters`);
 
         try {
             await this.ensureInitialized();
 
-            // Chunk the content using the embedding service
-            const chunks = await this.embeddingService.chunkText(content);
-            this.log('debug', `[INDEXING] Split content into ${chunks.length} chunks for file ${fileId}`);
-
-            // Generate embeddings for all chunks at once (batch processing for speed)
-            const embeddings = await this.embeddingService.embedTexts(chunks);
-            this.log('debug', `[INDEXING] Generated embeddings for ${chunks.length} chunks`);
-
-            // Create chunk objects with embeddings
-            const chunksWithEmbeddings = chunks.map((chunkContent, index) => ({
-                content: chunkContent,
-                embedding: embeddings[index],
-                metadata: {
-                    ...metadata,
-                    fileId,
-                    chunkIndex: index,
-                    id: `knowledge_${fileId}_chunk_${index}`
-                }
-            }));
+            const mime = metadata.mime || null;
 
-            // Save to per-knowledge vector file
-            await this.saveKnowledgeVector(fileId, chunksWithEmbeddings);
+            let chunkTexts: string[] = undefined;
+            let embeddings: number[][] = undefined;
 
-            const chunkIds = chunksWithEmbeddings.map(chunk => chunk.metadata.id);
-            this.log('log', `[INDEXING] Successfully indexed file ${fileId} with ${chunkIds.length} chunks using optimized approach`);
+            if(mime && LangChainRAGService.isSheetDocument(mime)) {       
+                this.log('debug', `[INDEXING] SHEET extraction mode detected.`);
+         
+                const docs = await this.embeddingService.chunkCSV(content as Record<string, any>[], ragOverride);
+                embeddings = await this.embeddingService.embedDocs(docs, batchCallback);
+                chunkTexts = docs.map(d => d.pageContent);
+            }else{                
+                chunkTexts = await this.embeddingService.chunkText(content as string, ragOverride);
+                embeddings = await this.embeddingService.embedTexts(chunkTexts, batchCallback);
+            }
+            
+            this.log('debug', `[INDEXING] Generated embeddings for ${chunkTexts.length} chunks`);
+
+       
+
+            if(!batchCallback){
+                // Create chunk objects with embeddings
+                const chunksWithEmbeddings = chunkTexts.map((chunkContent, index) => ({
+                    content: chunkContent,
+                    embedding: embeddings[index],
+                    metadata: {
+                        ...metadata,
+                        fileId,
+                        chunkIndex: index,
+                        id: `knowledge_${fileId}_chunk_${index}`
+                    }
+                }));
+                await this.saveKnowledgeVector(fileId, chunksWithEmbeddings);
+            }            
+
+            this.log('log', `[INDEXING] Successfully indexed file ${fileId} with ${chunkTexts.length} chunks using optimized approach`);
 
             return {
                 success: true,
-                data: { chunkIds }
+                data: { chunkCount: chunkTexts.length }
             };
 
         } catch (error: any) {
@@ -134,6 +159,10 @@ export class LangChainRAGService {
         }
     }
 
+    static isSheetDocument(mime: string): boolean {
+        return LangChainRAGService.SheetMimeType.includes(mime);
+    }
+
     /**
      * Search for relevant knowledge chunks using optimized vector search
      */
@@ -239,6 +268,10 @@ export class LangChainRAGService {
         }
     }
 
+    embedQuery(query: string): Promise<number[]> {
+        return this.vectorSearchService.getQueryEmbedding(query);
+    }
+
     /**
      * Get statistics about the RAG system
      */
@@ -317,6 +350,7 @@ export class LangChainRAGService {
 
     /**
      * Save chunks to knowledge-specific vector file with embeddings
+     * Uses streaming JSON write to handle large embedding datasets
      */
     private async saveKnowledgeVector(fileId: string | number, chunks: Array<{ content: string; embedding: number[]; metadata: any }>): Promise<void> {
         const vectorFilePath = this.getKnowledgeVectorPath(fileId);
@@ -328,13 +362,24 @@ export class LangChainRAGService {
         }
 
         try {
-            const vectorData = {
-                fileId,
-                chunks,
-                timestamp: new Date().toISOString()
-            };
-
-            fs.writeFileSync(vectorFilePath, JSON.stringify(vectorData, null, 2));
+            // Stream JSON to avoid "Invalid string length" on large datasets
+            const writeStream = fs.createWriteStream(vectorFilePath);
+            
+            await new Promise<void>((resolve, reject) => {
+                writeStream.on('error', reject);
+                writeStream.on('finish', resolve);
+                
+                writeStream.write(`{"fileId":${JSON.stringify(fileId)},"timestamp":${JSON.stringify(new Date().toISOString())},"chunks":[`);
+                
+                for (let i = 0; i < chunks.length; i++) {
+                    if (i > 0) writeStream.write(',');
+                    writeStream.write(JSON.stringify(chunks[i]));
+                }
+                
+                writeStream.write(']}');
+                writeStream.end();
+            });
+            
             this.log('debug', `[SAVE] Successfully saved ${chunks.length} chunks with embeddings for file ${fileId} to: "${vectorFilePath}"`);
 
         } catch (error) {

package/src/services/OpenAIRateLimitingService.ts

@@ -95,6 +95,7 @@ export class OpenAIRateLimitingService {
         }
 
         const results = new Array(items.length);
+        let doneItems = 0;
 
         // Process all batches with queue concurrency control
         await Promise.all(batchStarts.map(meta =>
@@ -104,9 +105,14 @@ export class OpenAIRateLimitingService {
                 for (let attempt = 0; attempt < 6; attempt++) {
                     try {
                         const batchResults = await this.callWithRetry(() => executor(attemptBatch));
+
                         for (let i = 0; i < batchResults.length; i++) {
                             results[meta.start + i] = batchResults[i];
+                            doneItems++;
                         }
+
+                        this.logger.debug(`Embedding chunks done [${doneItems}/${items.length}]`);
+
                         break;
                     } catch (err: any) {
                         const status = err?.status || err?.response?.status;

package/src/services/OptimizedVectorSearchService.ts

@@ -108,7 +108,7 @@ export class OptimizedVectorSearchService {
     /**
      * Get query embedding with caching
      */
-    private async getQueryEmbedding(query: string): Promise<number[]> {
+    async getQueryEmbedding(query: string): Promise<number[]> {
         // Check cache first
         if (this.queryEmbeddingCache.has(query)) {
             return this.queryEmbeddingCache.get(query)!;

package/src/services/TextChunker.ts

@@ -16,7 +16,7 @@ export class TextChunker {
      * Default separators following LangChain RecursiveCharacterTextSplitter approach
      * Ordered by preference for breaking text
      */
-    private static readonly DEFAULT_SEPARATORS = [
+    static readonly DEFAULT_SEPARATORS = [
         '\n\n',  // Double newlines (paragraphs)
         '\n',    // Single newlines
         '. ',    // Sentence endings

package/tsconfig.json

@@ -14,11 +14,14 @@
     "allowSyntheticDefaultImports": true,
     "sourceMap": true,
     "declaration": true, 
+    "types": []
   },
   "include": [
     "src"
   ],
   "exclude": [  
-    "node_modules"  
+    "node_modules",
+    "**/*.d.ts",
+    "**/node_modules/**",
   ]
 }