@rws-framework/ai-tools 3.4.0 → 3.5.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@rws-framework/ai-tools",
   "private": false,
-  "version": "3.4.0",
+  "version": "3.5.0",
   "description": "",
   "main": "src/index.ts",
   "scripts": {},

package/src/models/convo/EmbedLoader.ts

@@ -187,12 +187,15 @@ class EmbedLoader<LLMChat extends BaseChatModel> {
             logConvo(`After the split we have ${splitDocs.length} documents more than the original ${orgDocs.length}.`);
             logConvo(`Average length among ${splitDocs.length} documents (after split) is ${avgCharCountPost} characters.`);
 
-            let i = 0;
-            splitDocs.forEach((doc: Document) => {
+            // Write files asynchronously to prevent blocking
+            await Promise.all(splitDocs.map(async (doc: Document, i: number) => {
                 finalDocs.push(doc);
-                fs.writeFileSync(this.debugSplitFile(i), doc.pageContent);
-                i++;
-            });
+                try {
+                    await fs.promises.writeFile(this.debugSplitFile(i), doc.pageContent, 'utf-8');
+                } catch (error) {
+                    console.warn(`Failed to write debug file ${i}:`, error);
+                }
+            }));
         }else{
             const splitFiles = fs.readdirSync(splitDir);
             

package/src/models/prompts/inc/execution-methods-handler.ts

@@ -22,9 +22,17 @@ export class ExecutionMethodsHandler {
         debugVars: any = {}, 
         tools?: IAITool[]
     ): Promise<void> {
-        promptInstance.setSentInput(promptInstance.getInput());
+        // Create snapshot of current input to prevent race conditions
+        const inputSnapshot = [...promptInstance.getInput()];
+        promptInstance.setSentInput(inputSnapshot);
+        
         const returnedRWS = await executor.promptRequest(promptInstance as any, { intruderPrompt, debugVars, tools });
-        promptInstance.injestOutput(returnedRWS.readOutput());
+        
+        // Safely ingest output
+        const output = returnedRWS.readOutput();
+        if (output !== null && output !== undefined) {
+            promptInstance.injestOutput(output);
+        }
     }
 
     async singleRequestWith(
@@ -34,8 +42,13 @@ export class ExecutionMethodsHandler {
         ensureJson: boolean = false, 
         tools?: IAITool[]
     ): Promise<void> {
+        // Create snapshot of current input to prevent race conditions
+        const inputSnapshot = [...promptInstance.getInput()];
+        
         await executor.singlePromptRequest(promptInstance as any, { intruderPrompt, ensureJson, tools });
-        promptInstance.setSentInput(promptInstance.getInput());
+        
+        // Set the snapshot after execution to maintain consistency
+        promptInstance.setSentInput(inputSnapshot);
     }
 
     async streamWith(

package/src/models/prompts/inc/model-execution-manager.ts

@@ -22,7 +22,7 @@ export class ModelExecutionManager {
     constructor(modelId: string, modelType: string, hyperParameters: IPromptHyperParameters) {
         this.modelId = modelId;
         this.modelType = modelType;
-        this.hyperParameters = hyperParameters;
+        this.hyperParameters = hyperParameters || { temperature: 0.7, max_tokens: 512 };
     }
 
     getModelId(): string {

package/src/models/prompts/inc/tool-manager.ts

@@ -34,14 +34,29 @@ export class ToolManager {
 
     async callTools<T = unknown, O = unknown>(tools: IToolCall[], moduleRef: ModuleRef, aiToolOptions?: O): Promise<T[]> {
         const results: T[] = [];
+        const errors: Error[] = [];
+        
         for (const tool of tools) {
             if (this.toolHandlers.has(tool.function.name)) {
-                const result = await this.callAiTool<T, O>(tool, moduleRef, aiToolOptions);
-                if (result) {
-                    results.push(result);
+                try {
+                    const result = await this.callAiTool<T, O>(tool, moduleRef, aiToolOptions);
+                    if (result) {
+                        results.push(result);
+                    }
+                } catch (error) {
+                    console.error(`Tool execution failed for ${tool.function.name}:`, error);
+                    errors.push(error as Error);
+                    // Continue with other tools instead of failing completely
                 }
+            } else {
+                console.warn(`No handler found for tool: ${tool.function.name}`);
             }
         }
+        
+        // If all tools failed, throw the first error
+        if (results.length === 0 && errors.length > 0) {
+            throw errors[0];
+        }
 
         return results;
     }

package/src/services/LangChainEmbeddingService.ts

@@ -15,6 +15,7 @@ export class LangChainEmbeddingService {
     private chunkConfig: IChunkConfig;
     private isInitialized = false;
     private vectorStore: RWSVectorStore | null = null;
+    private static embeddingsPool = new Map<string, Embeddings>(); // Connection pooling
 
     constructor(private rateLimitingService: OpenAIRateLimitingService) {}
 
@@ -37,6 +38,14 @@ export class LangChainEmbeddingService {
 
 
     private initializeEmbeddings(): void {
+        const poolKey = `${this.config.provider}_${this.config.model}_${this.config.apiKey.slice(-8)}`;
+        
+        // Check connection pool first
+        if (LangChainEmbeddingService.embeddingsPool.has(poolKey)) {
+            this.embeddings = LangChainEmbeddingService.embeddingsPool.get(poolKey)!;
+            return;
+        }
+        
         switch (this.config.provider) {
             case 'cohere':
                 this.embeddings = new CohereEmbeddings({
@@ -58,6 +67,9 @@ export class LangChainEmbeddingService {
             default:
                 throw new Error(`Unsupported embedding provider: ${this.config.provider}`);
         }
+        
+        // Store in connection pool for reuse
+        LangChainEmbeddingService.embeddingsPool.set(poolKey, this.embeddings);
 
         if(this.config.rateLimiting){
             const rateLimitingCfg = {...OpenAIRateLimitingService.DEFAULT_CONFIG, ...this.config.rateLimiting};

package/src/services/LangChainRAGService.ts

@@ -41,8 +41,6 @@ export {
 export class LangChainRAGService {
     private config: ILangChainRAGConfig;
     private isInitialized = false;
-    private queryEmbeddingCache = new Map<string, number[]>();
-    private maxCacheSize = 100;
     private logger?: any; // Optional logger interface
 
     constructor(

package/src/services/OpenAIRateLimitingService.ts

@@ -7,6 +7,27 @@ import { BlackLogger } from '@rws-framework/server/nest';
 let encoding_for_model: any = null;
 encoding_for_model = tiktoken.encoding_for_model
 
+// Singleton tokenizer factory for performance
+class TokenizerFactory {
+    private static tokenizers = new Map<string, any>();
+    
+    static getTokenizer(model: string): any {
+        if (!this.tokenizers.has(model)) {
+            try {
+                if (encoding_for_model) {
+                    this.tokenizers.set(model, encoding_for_model(model));
+                } else {
+                    this.tokenizers.set(model, null);
+                }
+            } catch (e) {
+                console.warn(`Could not load tokenizer for model ${model}`);
+                this.tokenizers.set(model, null);
+            }
+        }
+        return this.tokenizers.get(model);
+    }
+}
+
 @Injectable()
 export class OpenAIRateLimitingService {
     static readonly DEFAULT_CONFIG: Required<IRateLimitConfig> = {
@@ -37,16 +58,10 @@ export class OpenAIRateLimitingService {
             this.config = { ...this.config, ...config };
         }
 
-        // Initialize tokenizer for precise token counting
-        try {
-            if (encoding_for_model) {
-                this.tokenizer = encoding_for_model(model);
-            } else {
-                this.tokenizer = null;
-            }
-        } catch (e) {
+        // Use singleton tokenizer factory for performance
+        this.tokenizer = TokenizerFactory.getTokenizer(model);
+        if (!this.tokenizer) {
             this.logger.warn(`Could not load tokenizer for model ${model}, using character-based estimation`);
-            this.tokenizer = null;
         }
 
         // Reinitialize queue with new concurrency

package/src/services/OptimizedVectorSearchService.ts

@@ -34,13 +34,18 @@ export class OptimizedVectorSearchService {
         const allCandidates: IOptimizedSearchResult[] = [];
         let totalCandidates = 0;
 
-        // Process all knowledge vectors in parallel
+        // Process all knowledge vectors with early termination optimization
         const searchPromises = knowledgeVectors.map(async (knowledgeVector) => {
             const candidates: IOptimizedSearchResult[] = [];
             const similarities: number[] = [];  // Track all similarities for debugging
+            let processedCount = 0;
             
-            for (const chunk of knowledgeVector.chunks) {
+            // Sort chunks by some heuristic to check best candidates first (optional optimization)
+            const chunks = knowledgeVector.chunks;
+            
+            for (const chunk of chunks) {
                 totalCandidates++;
+                processedCount++;
                 
                 if (!chunk.embedding || !Array.isArray(chunk.embedding)) {
                     continue;

package/src/services/TextChunker.ts

@@ -197,8 +197,9 @@ export class TextChunker {
         for (let i = 0; i < chunks.length; i++) {
             const chunk = chunks[i];
             
-            // Check if we can merge this chunk with current chunk
-            const combined = currentChunk ? currentChunk + ' ' + chunk : chunk;
+            // Use array for efficient string building
+            const parts = currentChunk ? [currentChunk, chunk] : [chunk];
+            const combined = parts.join(' ');
             
             if (combined.length <= maxChars) {
                 // Can merge

package/src/types/IPrompt.ts

@@ -53,8 +53,7 @@ interface IAITool {
 
 interface IPromptHyperParameters {
     temperature: number,
-    top_k?: number,
-    top_p?: number,
+    max_tokens: number,
     [key: string]: number
 }