@rws-framework/ai-tools 3.3.0 → 3.4.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@rws-framework/ai-tools",
   "private": false,
-  "version": "3.3.0",
+  "version": "3.4.0",
   "description": "",
   "main": "src/index.ts",
   "scripts": {},

package/src/services/LangChainRAGService.ts

@@ -85,11 +85,11 @@ export class LangChainRAGService {
      * Index knowledge content for RAG with optimized per-knowledge vector storage
      */
     async indexKnowledge(
-        knowledgeId: string | number,
+        fileId: string | number,
         content: string,
         metadata: Record<string, any> = {}
     ): Promise<IRAGResponse<{ chunkIds: string[] }>> {
-        this.log('log', `[INDEXING] Starting indexKnowledge for knowledgeId: ${knowledgeId}`);
+        this.log('log', `[INDEXING] Starting indexKnowledge for fileId: ${fileId}`);
         this.log('debug', `[INDEXING] Content length: ${content.length} characters`);
 
         try {
@@ -97,7 +97,7 @@ export class LangChainRAGService {
 
             // Chunk the content using the embedding service
             const chunks = await this.embeddingService.chunkText(content);
-            this.log('debug', `[INDEXING] Split content into ${chunks.length} chunks for knowledge ${knowledgeId}`);
+            this.log('debug', `[INDEXING] Split content into ${chunks.length} chunks for file ${fileId}`);
 
             // Generate embeddings for all chunks at once (batch processing for speed)
             const embeddings = await this.embeddingService.embedTexts(chunks);
@@ -109,17 +109,17 @@ export class LangChainRAGService {
                 embedding: embeddings[index],
                 metadata: {
                     ...metadata,
-                    knowledgeId,
+                    fileId,
                     chunkIndex: index,
-                    id: `knowledge_${knowledgeId}_chunk_${index}`
+                    id: `knowledge_${fileId}_chunk_${index}`
                 }
             }));
 
             // Save to per-knowledge vector file
-            await this.saveKnowledgeVector(knowledgeId, chunksWithEmbeddings);
+            await this.saveKnowledgeVector(fileId, chunksWithEmbeddings);
 
             const chunkIds = chunksWithEmbeddings.map(chunk => chunk.metadata.id);
-            this.log('log', `[INDEXING] Successfully indexed knowledge ${knowledgeId} with ${chunkIds.length} chunks using optimized approach`);
+            this.log('log', `[INDEXING] Successfully indexed file ${fileId} with ${chunkIds.length} chunks using optimized approach`);
 
             return {
                 success: true,
@@ -127,7 +127,7 @@ export class LangChainRAGService {
             };
 
         } catch (error: any) {
-            this.log('error', `[INDEXING] Failed to index knowledge ${knowledgeId}:`, error);
+            this.log('error', `[INDEXING] Failed to index file ${fileId}:`, error);
             return {
                 success: false,
                 data: null,
@@ -141,32 +141,45 @@ export class LangChainRAGService {
      */
     async searchKnowledge(request: IRAGSearchRequest): Promise<IRAGResponse<{ results: ISearchResult[] }>> {
         this.log('log', `[SEARCH] Starting knowledge search for query: "${request.query}"`);
-        this.log('debug', `[SEARCH] Search parameters: maxResults=${request.maxResults || 5}, threshold=${request.threshold || 0.3}`);
+        this.log('debug', `[SEARCH] Search parameters: maxResults=${request.maxResults || 5}, threshold=${request.threshold || 0.3}, temporaryDocumentSearch=${request.temporaryDocumentSearch}`);
 
         try {
             await this.ensureInitialized();
 
-            const knowledgeIds = request.filter?.knowledgeIds || [];
-            console.log('knowledgeIds', knowledgeIds);
+            const fileIds = request.filter?.fileIds || [];
+            console.log('fileIds', fileIds, 'temporaryDocumentSearch:', request.temporaryDocumentSearch);
             
-            if (knowledgeIds.length === 0) {
-                this.log('warn', '[SEARCH] No knowledge IDs provided for search, returning empty results');
+            if (fileIds.length === 0) {
+                this.log('warn', '[SEARCH] No file IDs provided for search, returning empty results');
                 return {
                     success: true,
                     data: { results: [] }
                 };
             }
 
-            // Load all knowledge vectors in parallel
-            const knowledgeVectorPromises = knowledgeIds.map(async (knowledgeId) => {
-                const vectorData = await this.loadKnowledgeVectorWithEmbeddings(knowledgeId);
-                return {
-                    knowledgeId,                    
-                    chunks: vectorData.chunks
-                };
+            // Load all knowledge vectors in parallel (including temporary documents)
+            const knowledgeVectorPromises = fileIds.map(async (fileId) => {
+                try {
+                    const vectorData = await this.loadKnowledgeVectorWithEmbeddings(fileId);
+                    return {
+                        fileId,                    
+                        chunks: vectorData.chunks
+                    };
+                } catch (loadError) {
+                    this.log('warn', `[SEARCH] Failed to load file ${fileId}:`, loadError);
+                    return null;
+                }
             });
 
-            const knowledgeVectors = await Promise.all(knowledgeVectorPromises);
+            const knowledgeVectors = (await Promise.all(knowledgeVectorPromises)).filter(v => v !== null);
+            
+            if (knowledgeVectors.length === 0) {
+                this.log('warn', '[SEARCH] No knowledge vectors could be loaded for search');
+                return {
+                    success: true,
+                    data: { results: [] }
+                };
+            }
 
             // Use optimized vector search service
             const searchResponse = await this.vectorSearchService.searchSimilar({
@@ -178,10 +191,10 @@ export class LangChainRAGService {
             
             // Convert results to expected format
             const results: ISearchResult[] = searchResponse.results.map(result => ({
-                knowledgeId: result.metadata.knowledgeId,
+                fileId: result.metadata?.fileId,  // Use fileId directly
                 content: result.content,
                 score: result.score,
-                metadata: result.metadata,
+                metadata: result.metadata,  // Pass metadata as-is
                 chunkId: result.chunkId,                
             }));
 
@@ -205,25 +218,25 @@ export class LangChainRAGService {
     /**
      * Remove knowledge from index
      */
-    async removeKnowledge(knowledgeId: string | number): Promise<boolean> {
-        this.log('log', `[REMOVE] Starting removal of knowledge: ${knowledgeId}`);
+    async removeKnowledge(fileId: string | number): Promise<boolean> {
+        this.log('log', `[REMOVE] Starting removal of file: ${fileId}`);
 
         try {
             await this.ensureInitialized();
 
             // Remove the individual knowledge vector file
-            const vectorFilePath = this.getKnowledgeVectorPath(knowledgeId);
+            const vectorFilePath = this.getKnowledgeVectorPath(fileId);
             if (fs.existsSync(vectorFilePath)) {
                 fs.unlinkSync(vectorFilePath);
-                this.log('log', `[REMOVE] Successfully removed vector file for knowledge ${knowledgeId}`);
+                this.log('log', `[REMOVE] Successfully removed vector file for file ${fileId}`);
                 return true;
             } else {
-                this.log('warn', `[REMOVE] Vector file not found for knowledge ${knowledgeId}`);
+                this.log('warn', `[REMOVE] Vector file not found for file ${fileId}`);
                 return true; // Consider it successful if file doesn't exist
             }
 
         } catch (error: any) {
-            this.log('error', `[REMOVE] Failed to remove knowledge ${knowledgeId}:`, error);
+            this.log('error', `[REMOVE] Failed to remove file ${fileId}:`, error);
             return false;
         }
     }
@@ -307,8 +320,8 @@ export class LangChainRAGService {
     /**
      * Save chunks to knowledge-specific vector file with embeddings
      */
-    private async saveKnowledgeVector(knowledgeId: string | number, chunks: Array<{ content: string; embedding: number[]; metadata: any }>): Promise<void> {
-        const vectorFilePath = this.getKnowledgeVectorPath(knowledgeId);
+    private async saveKnowledgeVector(fileId: string | number, chunks: Array<{ content: string; embedding: number[]; metadata: any }>): Promise<void> {
+        const vectorFilePath = this.getKnowledgeVectorPath(fileId);
         const vectorDir = path.dirname(vectorFilePath);
 
         // Ensure directory exists
@@ -318,16 +331,16 @@ export class LangChainRAGService {
 
         try {
             const vectorData = {
-                knowledgeId,
+                fileId,
                 chunks,
                 timestamp: new Date().toISOString()
             };
 
             fs.writeFileSync(vectorFilePath, JSON.stringify(vectorData, null, 2));
-            this.log('debug', `[SAVE] Successfully saved ${chunks.length} chunks with embeddings for knowledge ${knowledgeId}`);
+            this.log('debug', `[SAVE] Successfully saved ${chunks.length} chunks with embeddings for file ${fileId}`);
 
         } catch (error) {
-            this.log('error', `[SAVE] Failed to save vector data for knowledge ${knowledgeId}:`, error);
+            this.log('error', `[SAVE] Failed to save vector data for file ${fileId}:`, error);
             throw error;
         }
     }
@@ -335,24 +348,24 @@ export class LangChainRAGService {
     /**
      * Load vector data for a specific knowledge item with embeddings
      */
-    private async loadKnowledgeVectorWithEmbeddings(knowledgeId: string | number): Promise<{ knowledgeId?: string | number, chunks: Array<{ content: string; embedding: number[]; metadata: any }> }> {
-        const vectorFilePath = this.getKnowledgeVectorPath(knowledgeId);
+    private async loadKnowledgeVectorWithEmbeddings(fileId: string | number): Promise<{ fileId?: string | number, chunks: Array<{ content: string; embedding: number[]; metadata: any }> }> {
+        const vectorFilePath = this.getKnowledgeVectorPath(fileId);
         
         if (!fs.existsSync(vectorFilePath)) {
-            this.log('debug', `[LOAD] No vector file found for knowledge ${knowledgeId}, skipping...`);
+            this.log('debug', `[LOAD] No vector file found for file ${fileId}, skipping...`);
             return { chunks: [] };
         }
 
         try {
-            this.log('debug', `[LOAD] Loading vector data with embeddings for knowledge ${knowledgeId} from ${vectorFilePath}`);
+            this.log('debug', `[LOAD] Loading vector data with embeddings for file ${fileId} from ${vectorFilePath}`);
             const vectorData = JSON.parse(fs.readFileSync(vectorFilePath, 'utf8'));
             
             return {
                 chunks: vectorData.chunks || [],
-                knowledgeId
+                fileId
             };
         } catch (error) {
-            this.log('error', `[LOAD] Failed to load vector data for knowledge ${knowledgeId}:`, error);
+            this.log('error', `[LOAD] Failed to load vector data for file ${fileId}:`, error);
             return { chunks: [] };
         }
     }
@@ -360,12 +373,12 @@ export class LangChainRAGService {
     /**
      * Get the file path for a specific knowledge's vector data
      */
-    private getKnowledgeVectorPath(knowledgeId: string | number): string {
+    private getKnowledgeVectorPath(fileId: string | number): string {
         const vectorDir = path.join(rwsPath.findRootWorkspacePath(), 'files', 'vectors', 'knowledge');
         if (!fs.existsSync(vectorDir)) {
             fs.mkdirSync(vectorDir, { recursive: true });
         }
-        return path.join(vectorDir, `knowledge_${knowledgeId}.json`);
+        return path.join(vectorDir, `knowledge_${fileId}.json`);
     }
 
     /**

package/src/services/OptimizedVectorSearchService.ts

@@ -54,35 +54,42 @@ export class OptimizedVectorSearchService {
                     candidates.push({
                         content: chunk.content,
                         score: similarity,
-                        metadata: chunk.metadata,
-                        knowledgeId: knowledgeVector.knowledgeId,
-                        chunkId: chunk.metadata?.id || `${knowledgeVector.knowledgeId}_chunk_${Date.now()}`
+                        metadata: {
+                            ...chunk.metadata,
+                            fileId: knowledgeVector.fileId  // Use fileId directly
+                        },
+                        fileId: knowledgeVector.fileId,  // Always use the fileId from the knowledgeVector
+                        chunkId: chunk.metadata?.id || `${knowledgeVector.fileId}_chunk_${Date.now()}`
                     });
                 }
             }
             
+            // Sort candidates by score and take top maxResults per source
+            const topCandidates = candidates
+                .sort((a, b) => b.score - a.score)
+                .slice(0, maxResults);
+            
             // Log similarity statistics for debugging
             if (similarities.length > 0) {
                 const maxSim = Math.max(...similarities);
                 const avgSim = similarities.reduce((a, b) => a + b, 0) / similarities.length;
-                console.log(`[VECTOR SEARCH] Knowledge ${knowledgeVector.knowledgeId}: Max similarity: ${maxSim.toFixed(4)}, Avg: ${avgSim.toFixed(4)}, Candidates above ${threshold}: ${candidates.length}`);
+                console.log(`[VECTOR SEARCH] File ${knowledgeVector.fileId}: Max similarity: ${maxSim.toFixed(4)}, Avg: ${avgSim.toFixed(4)}, Candidates above ${threshold}: ${candidates.length}, Top results taken: ${topCandidates.length}`);
             }
             
-            return candidates;
+            return topCandidates;
         });
 
         // Wait for all searches to complete
         const allCandidateArrays = await Promise.all(searchPromises);
         
-        // Flatten results
+        // Flatten results (each source already limited to maxResults)
         for (const candidates of allCandidateArrays) {
             allCandidates.push(...candidates);
         }
 
-        // Sort by similarity score and take top results
+        // Sort by similarity score (no additional limiting since each source is already limited)
         const results = allCandidates
-            .sort((a, b) => b.score - a.score)
-            .slice(0, maxResults);
+            .sort((a, b) => b.score - a.score);
 
         const searchTime = Date.now() - startTime;
 
@@ -122,7 +129,7 @@ export class OptimizedVectorSearchService {
     async batchSearch(
         queries: string[],
         knowledgeVectors: Array<{
-            knowledgeId: string | number;
+            fileId: string | number;
             chunks: Array<{
                 content: string;
                 embedding: number[];
@@ -165,7 +172,7 @@ export class OptimizedVectorSearchService {
     private async searchWithEmbedding(request: {
         queryEmbedding: number[];
         knowledgeVectors: Array<{
-            knowledgeId: string | number;
+            fileId: string | number;
             chunks: Array<{
                 content: string;
                 embedding: number[];
@@ -200,8 +207,8 @@ export class OptimizedVectorSearchService {
                         content: chunk.content,
                         score: similarity,
                         metadata: chunk.metadata,
-                        knowledgeId: knowledgeVector.knowledgeId,
-                        chunkId: chunk.metadata?.id || `${knowledgeVector.knowledgeId}_chunk_${Date.now()}`
+                        fileId: knowledgeVector.fileId,
+                        chunkId: chunk.metadata?.id || `${knowledgeVector.fileId}_chunk_${Date.now()}`
                     });
                 }
             }
@@ -252,7 +259,7 @@ export class OptimizedVectorSearchService {
      * Search similar documents (compatibility method from LangChainVectorStoreService)
      */
     async searchSimilarCompat(request: IVectorSearchRequest, knowledgeVectors: Array<{
-        knowledgeId: string | number;
+        fileId: string | number;
         chunks: Array<{
             content: string;
             embedding: number[];
@@ -271,9 +278,9 @@ export class OptimizedVectorSearchService {
             let filteredVectors = knowledgeVectors;
             if (filter) {
                 filteredVectors = knowledgeVectors.filter(vector => {
-                    // Check knowledge IDs
-                    if (filter.knowledgeIds && filter.knowledgeIds.length > 0) {
-                        return filter.knowledgeIds.includes(String(vector.knowledgeId));
+                    // Check file IDs
+                    if (filter.fileIds && filter.fileIds.length > 0) {
+                        return filter.fileIds.includes(String(vector.fileId));
                     }
                     return true;
                 });
@@ -293,7 +300,7 @@ export class OptimizedVectorSearchService {
                 score: result.score,
                 metadata: result.metadata,
                 chunkId: result.chunkId,
-                knowledgeId: result.knowledgeId
+                fileId: result.fileId
             }));
 
             return {
@@ -313,7 +320,7 @@ export class OptimizedVectorSearchService {
      * Get search statistics
      */
     getStats(knowledgeVectors: Array<{
-        knowledgeId: string | number;
+        fileId: string | number;
         chunks: Array<{ content: string; embedding: number[]; metadata: any; }>;
     }>): { totalChunks: number; totalKnowledge: number } {
         const totalChunks = knowledgeVectors.reduce((total, vector) => total + vector.chunks.length, 0);

package/src/types/rag.types.ts

@@ -24,8 +24,9 @@ export interface IRAGSearchRequest {
     query: string;
     maxResults?: number;
     threshold?: number;
+    temporaryDocumentSearch?: boolean; // Flag for searching temporary documents (web search)
     filter?: {
-        knowledgeIds?: (string | number)[];
+        fileIds?: (string | number)[];
         documentIds?: (string | number)[];
         [key: string]: any;
     };

package/src/types/search.types.ts

@@ -5,7 +5,7 @@ export interface ISearchResult {
     content: string;
     score: number;
     metadata: any;
-    knowledgeId: string | number;
+    fileId: string | number;
     chunkId: string;
 }
 
@@ -30,7 +30,7 @@ export interface IVectorSearchResponse {
 export interface IOptimizedSearchRequest {
     query: string;
     knowledgeVectors: Array<{
-        knowledgeId: string | number;
+        fileId: string | number;
         chunks: Array<{
             content: string;
             embedding: number[];
@@ -45,7 +45,7 @@ export interface IOptimizedSearchResult {
     content: string;
     score: number;
     metadata: any;
-    knowledgeId: string | number;
+    fileId: string | number;
     chunkId: string;
 }