@rws-framework/ai-tools 3.7.0 → 3.8.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@rws-framework/ai-tools",
   "private": false,
-  "version": "3.7.0",
+  "version": "3.8.0",
   "description": "",
   "main": "src/index.ts",
   "scripts": {},

package/src/services/LangChainEmbeddingService.ts

@@ -127,15 +127,15 @@ export class LangChainEmbeddingService {
     /**
      * Split text into chunks
      */
-    async chunkText(text: string): Promise<string[]> {
+    async chunkText(text: string, ragOverride?: IChunkConfig): Promise<string[]> {
         this.ensureInitialized();
         
         // Use our custom TextChunker instead of LangChain's splitter
         // Use safe token limits - the TextChunker handles token estimation internally
-        const maxTokens = this.chunkConfig?.chunkSize || 450; // Safe token limit for embedding models
-        const overlap = this.chunkConfig?.chunkOverlap || 50; // Character overlap, not token
-        
-        return TextChunker.chunkText(text, maxTokens, overlap);
+        const maxTokens = ragOverride ? ragOverride.chunkSize : (this.chunkConfig?.chunkSize || 450); // Safe token limit for embedding models
+        const overlap = ragOverride ? ragOverride.chunkOverlap : (this.chunkConfig?.chunkOverlap || 50); // Character overlap, not token
+        const separators = ragOverride?.separators || this.chunkConfig?.separators || TextChunker.DEFAULT_SEPARATORS; // Default separators
+        return TextChunker.chunkText(text, maxTokens, overlap, separators);
     }
 
     /**

package/src/services/LangChainRAGService.ts

@@ -85,7 +85,8 @@ export class LangChainRAGService {
     async indexKnowledge(
         fileId: string | number,
         content: string,
-        metadata: Record<string, any> = {}
+        metadata: Record<string, any> = {},
+        ragOverride?: IChunkConfig
     ): Promise<IRAGResponse<{ chunkIds: string[] }>> {
         this.log('log', `[INDEXING] Starting indexKnowledge for fileId: ${fileId}`);
         this.log('debug', `[INDEXING] Content length: ${content.length} characters`);
@@ -94,7 +95,7 @@ export class LangChainRAGService {
             await this.ensureInitialized();
 
             // Chunk the content using the embedding service
-            const chunks = await this.embeddingService.chunkText(content);
+            const chunks = await this.embeddingService.chunkText(content, ragOverride);
             this.log('debug', `[INDEXING] Split content into ${chunks.length} chunks for file ${fileId}`);
 
             // Generate embeddings for all chunks at once (batch processing for speed)