@rws-framework/ai-tools 2.2.0 → 3.0.0

package/src/services/LangChainVectorStoreService.ts

@@ -0,0 +1,378 @@
+import { Injectable } from '@rws-framework/server/nest';
+import { Document } from '@langchain/core/documents';
+import { LangChainEmbeddingService } from './LangChainEmbeddingService';
+import RWSVectorStore, { VectorDocType, IVectorStoreConfig as RWSVectorStoreConfig } from '../models/convo/VectorStore';
+
+export interface ISearchResult {
+    content: string;
+    score: number;
+    metadata: any;
+    chunkId: string;
+}
+
+export interface IVectorStoreConfig {
+    type: 'memory' | 'faiss';
+    maxResults?: number;
+    autoSave?: boolean;
+    similarityThreshold?: number;
+    persistPath?: string;
+}
+
+export interface IDocumentChunk {
+    id: string;
+    content: string;
+    embedding?: number[];
+    metadata?: {
+        documentId: string;
+        chunkIndex: number;
+        source?: string;
+        title?: string;
+        knowledgeId?: string;
+        [key: string]: any;
+    };
+}
+
+export interface IVectorSearchRequest {
+    query: string;
+    maxResults?: number;
+    similarityThreshold?: number;
+    filter?: {
+        knowledgeIds?: string[];
+        documentIds?: string[];
+        [key: string]: any;
+    };
+}
+
+export interface IVectorSearchResponse {
+    results: ISearchResult[];
+    totalResults: number;
+}
+
+/**
+ * LangChain-based vector store service that provides document storage and similarity search
+ * Now uses RWSVectorStore for unified vector storage like the LangChain tutorial
+ */
+@Injectable()
+export class LangChainVectorStoreService {
+    private vectorStore: RWSVectorStore;
+    private documents: Map<string, Document> = new Map();
+    private config: IVectorStoreConfig;
+    private isInitialized = false;
+    private documentCount = 0;
+    private embeddingService: LangChainEmbeddingService;
+
+    constructor() {    
+        // Empty constructor for NestJS dependency injection
+    }
+
+    /**
+     * Initialize the service with configuration
+     */
+    async initialize(embeddingService: LangChainEmbeddingService, config: IVectorStoreConfig): Promise<void> {
+        if (this.isInitialized) {
+            return;
+        }
+        
+        this.embeddingService = embeddingService;
+        this.config = {
+            type: 'memory',
+            maxResults: 10,
+            similarityThreshold: 0.1,  // Use lower threshold like we configured
+            ...config,
+        };
+
+        const embeddings = this.embeddingService.getEmbeddingsInstance();
+        
+        // Initialize with empty documents first, then create RWSVectorStore
+        const initialDocs: VectorDocType = [];
+        this.vectorStore = await new RWSVectorStore(
+            initialDocs, 
+            embeddings, 
+            { 
+                type: this.config.type,
+                persistPath: this.config.persistPath 
+            }
+        ).init();
+        
+        console.log(`Created new ${this.config.type} vector store using RWSVectorStore`);
+        this.isInitialized = true;
+    }
+
+    /**
+     * Add documents to the vector store
+     */
+    async addDocuments(documents: Document[]): Promise<string[]> {
+        this.ensureInitialized();
+
+        try {
+            const ids = await this.vectorStore.addDocuments(documents);
+            const docIds: string[] = [];
+            
+            // Store documents in our map for retrieval
+            documents.forEach((doc, index) => {
+                const id = `doc_${Date.now()}_${index}`;
+                this.documents.set(id, doc);
+                docIds.push(id);
+            });
+
+            this.documentCount += documents.length;
+            return docIds;
+        } catch (error) {
+            console.error('Failed to add documents to vector store:', error);
+            throw error;
+        }
+    }
+
+    /**
+     * Index a single document (split into chunks if needed)
+     */
+    async indexDocument(
+        content: string,
+        documentId: string,
+        metadata: Record<string, any> = {}
+    ): Promise<{ success: boolean; chunkIds: string[]; error?: string }> {
+        try {
+            this.ensureInitialized();
+
+            // Remove existing chunks for this document
+            await this.deleteDocument(documentId);
+
+            // Create document chunks
+            const documents = await this.embeddingService.createDocuments(content, {
+                documentId,
+                ...metadata,
+            });
+
+            // Add to vector store
+            const chunkIds = await this.addDocuments(documents);
+
+            return {
+                success: true,
+                chunkIds,
+            };
+        } catch (error: any) {
+            console.error(`Failed to index document ${documentId}:`, error);
+            return {
+                success: false,
+                chunkIds: [],
+                error: error?.message || 'Unknown error',
+            };
+        }
+    }
+
+    /**
+     * Search for similar documents using RWSVectorStore like the LangChain tutorial
+     */
+    async searchSimilar(request: IVectorSearchRequest): Promise<IVectorSearchResponse> {
+        this.ensureInitialized();
+
+        try {
+            const {
+                query,
+                maxResults = this.config.maxResults,
+                similarityThreshold = this.config.similarityThreshold,
+                filter,
+            } = request;
+
+            // Use RWSVectorStore's similaritySearchWithScore method like the tutorial
+            const searchResults = await this.vectorStore.similaritySearchWithScore(
+                query,
+                maxResults || 10
+            );
+
+            // Filter by similarity threshold and metadata filters
+            let filteredResults = searchResults
+                .filter(([_doc, score]: [any, any]) => score >= (similarityThreshold || 0.1));
+
+            // Apply knowledge/document ID filters if provided
+            if (filter) {
+                filteredResults = filteredResults.filter(([doc, _score]: [any, any]) => {
+                    // Check knowledge IDs
+                    if (filter.knowledgeIds && filter.knowledgeIds.length > 0) {
+                        const docKnowledgeId = doc.metadata?.knowledgeId;
+                        if (!docKnowledgeId || !filter.knowledgeIds.includes(docKnowledgeId)) {
+                            return false;
+                        }
+                    }
+                    
+                    // Check document IDs
+                    if (filter.documentIds && filter.documentIds.length > 0) {
+                        const docId = doc.metadata?.documentId;
+                        if (!docId || !filter.documentIds.includes(docId)) {
+                            return false;
+                        }
+                    }
+                    
+                    return true;
+                });
+            }
+
+            // Format results to match our interface
+            const results: ISearchResult[] = filteredResults
+                .map(([doc, score]: [any, any], index: number) => ({
+                    content: doc.pageContent,
+                    score,
+                    metadata: doc.metadata,
+                    chunkId: doc.metadata?.id || `chunk_${index}`,
+                }));
+
+            return {
+                results,
+                totalResults: results.length,
+            };
+        } catch (error) {
+            console.error('Failed to search similar documents:', error);
+            return {
+                results: [],
+                totalResults: 0,
+            };
+        }
+    }
+
+    /**
+     * Delete a document and all its chunks
+     */
+    async deleteDocument(documentId: string): Promise<boolean> {
+        try {
+            await this.ensureInitialized();
+
+            // Find all chunks for this document
+            const docsToDelete: string[] = [];
+            for (const [id, doc] of this.documents.entries()) {
+                if (doc.metadata?.documentId === documentId) {
+                    docsToDelete.push(id);
+                }
+            }
+
+            if (docsToDelete.length > 0) {
+                // Use RWSVectorStore's deleteDocuments method
+                await this.vectorStore.deleteDocuments(docsToDelete);
+
+                // Remove from our document map
+                docsToDelete.forEach(id => this.documents.delete(id));
+
+                if (this.config.autoSave) {
+                    await this.save();
+                }
+            }
+
+            return true;
+        } catch (error) {
+            console.error(`Failed to delete document ${documentId}:`, error);
+            return false;
+        }
+    }
+
+    /**
+     * Get documents by filter
+     */
+    async getDocuments(filter?: Record<string, any>): Promise<Document[]> {
+        const docs: Document[] = [];
+        
+        for (const doc of this.documents.values()) {
+            if (!filter) {
+                docs.push(doc);
+                continue;
+            }
+
+            let matches = true;
+            for (const [key, value] of Object.entries(filter)) {
+                if (doc.metadata?.[key] !== value) {
+                    matches = false;
+                    break;
+                }
+            }
+
+            if (matches) {
+                docs.push(doc);
+            }
+        }
+
+        return docs;
+    }
+
+    /**
+     * Save the vector store to disk (not needed for memory store)
+     */
+    async save(): Promise<boolean> {
+        return true; // Memory store doesn't need saving
+    }
+
+    /**
+     * Get statistics about the vector store
+     */
+    getStats(): { totalChunks: number; totalDocuments: number } {
+        const documentIds = new Set<string>();
+        
+        for (const doc of this.documents.values()) {
+            if (doc.metadata?.documentId) {
+                documentIds.add(doc.metadata.documentId);
+            }
+        }
+
+        return {
+            totalChunks: this.documentCount,
+            totalDocuments: documentIds.size,
+        };
+    }
+
+    /**
+     * Clear all documents from the vector store
+     */
+    async clear(): Promise<boolean> {
+        try {
+            await this.ensureInitialized();
+
+            // Recreate RWSVectorStore with empty documents
+            const embeddings = this.embeddingService.getEmbeddingsInstance();
+            const initialDocs: VectorDocType = [];
+            this.vectorStore = await new RWSVectorStore(
+                initialDocs, 
+                embeddings, 
+                { 
+                    type: this.config.type,
+                    persistPath: this.config.persistPath 
+                }
+            ).init();
+
+            // Clear our document map
+            this.documents.clear();
+            this.documentCount = 0;
+
+            return true;
+        } catch (error) {
+            console.error('Failed to clear vector store:', error);
+            return false;
+        }
+    }
+
+    /**
+     * Ensure the vector store is initialized
+     */
+    private ensureInitialized(): void {
+        if (!this.isInitialized) {
+            throw new Error('LangChainVectorStoreService not initialized. Call initialize() first.');
+        }
+    }
+
+    /**
+     * Get the underlying RWSVectorStore instance
+     */
+    getVectorStore(): RWSVectorStore {
+        return this.vectorStore;
+    }
+
+    /**
+     * Get the underlying LangChain vector store (memory or faiss)
+     */
+    getLangChainVectorStore() {
+        return this.vectorStore.getVectorStore();
+    }
+
+    /**
+     * Update configuration
+     */
+    updateConfig(newConfig: Partial<IVectorStoreConfig>): void {
+        this.config = { ...this.config, ...newConfig };
+    }
+}

package/src/services/OptimizedVectorSearchService.ts

@@ -0,0 +1,324 @@
+import { Injectable } from '@nestjs/common';
+import { LangChainEmbeddingService } from './LangChainEmbeddingService';
+import { 
+    IOptimizedSearchRequest, 
+    IOptimizedSearchResult, 
+    IOptimizedSearchResponse,
+    IVectorSearchRequest,
+    IVectorSearchResponse,
+    ISearchResult
+} from '../types';
+
+/**
+ * Optimized vector search service for lightning-fast similarity searches
+ * Uses pre-computed embeddings and direct cosine similarity calculations
+ */
+@Injectable()
+export class OptimizedVectorSearchService {
+    private queryEmbeddingCache = new Map<string, number[]>();
+    private maxCacheSize = 100;
+
+    constructor(private embeddingService: LangChainEmbeddingService) {}
+
+    /**
+     * Perform optimized similarity search across pre-computed vectors
+     */
+    async searchSimilar(request: IOptimizedSearchRequest): Promise<IOptimizedSearchResponse> {
+        const startTime = Date.now();
+        const { query, knowledgeVectors, maxResults = 5, threshold = 0.1 } = request;
+
+        // Get or compute query embedding
+        const queryEmbedding = await this.getQueryEmbedding(query);
+
+        // Collect all candidates with parallel processing
+        const allCandidates: IOptimizedSearchResult[] = [];
+        let totalCandidates = 0;
+
+        // Process all knowledge vectors in parallel
+        const searchPromises = knowledgeVectors.map(async (knowledgeVector) => {
+            const candidates: IOptimizedSearchResult[] = [];
+            const similarities: number[] = [];  // Track all similarities for debugging
+            
+            for (const chunk of knowledgeVector.chunks) {
+                totalCandidates++;
+                
+                if (!chunk.embedding || !Array.isArray(chunk.embedding)) {
+                    continue;
+                }
+
+                // Compute cosine similarity
+                const similarity = this.embeddingService.cosineSimilarity(queryEmbedding, chunk.embedding);
+                similarities.push(similarity);
+                
+                if (similarity >= threshold) {
+                    candidates.push({
+                        content: chunk.content,
+                        score: similarity,
+                        metadata: chunk.metadata,
+                        knowledgeId: knowledgeVector.knowledgeId,
+                        chunkId: chunk.metadata?.id || `${knowledgeVector.knowledgeId}_chunk_${Date.now()}`
+                    });
+                }
+            }
+            
+            // Log similarity statistics for debugging
+            if (similarities.length > 0) {
+                const maxSim = Math.max(...similarities);
+                const avgSim = similarities.reduce((a, b) => a + b, 0) / similarities.length;
+                console.log(`[VECTOR SEARCH] Knowledge ${knowledgeVector.knowledgeId}: Max similarity: ${maxSim.toFixed(4)}, Avg: ${avgSim.toFixed(4)}, Candidates above ${threshold}: ${candidates.length}`);
+            }
+            
+            return candidates;
+        });
+
+        // Wait for all searches to complete
+        const allCandidateArrays = await Promise.all(searchPromises);
+        
+        // Flatten results
+        for (const candidates of allCandidateArrays) {
+            allCandidates.push(...candidates);
+        }
+
+        // Sort by similarity score and take top results
+        const results = allCandidates
+            .sort((a, b) => b.score - a.score)
+            .slice(0, maxResults);
+
+        const searchTime = Date.now() - startTime;
+
+        return {
+            results,
+            searchTime,
+            totalCandidates
+        };
+    }
+
+    /**
+     * Get query embedding with caching
+     */
+    private async getQueryEmbedding(query: string): Promise<number[]> {
+        // Check cache first
+        if (this.queryEmbeddingCache.has(query)) {
+            return this.queryEmbeddingCache.get(query)!;
+        }
+
+        // Generate embedding
+        const embedding = await this.embeddingService.embedText(query);
+
+        // Cache the embedding (with size limit)
+        if (this.queryEmbeddingCache.size >= this.maxCacheSize) {
+            // Remove oldest entry
+            const firstKey = this.queryEmbeddingCache.keys().next().value;
+            this.queryEmbeddingCache.delete(firstKey);
+        }
+        
+        this.queryEmbeddingCache.set(query, embedding);
+        return embedding;
+    }
+
+    /**
+     * Batch search multiple queries efficiently
+     */
+    async batchSearch(
+        queries: string[],
+        knowledgeVectors: Array<{
+            knowledgeId: string | number;
+            chunks: Array<{
+                content: string;
+                embedding: number[];
+                metadata: any;
+            }>;
+        }>,
+        maxResults = 5,
+        threshold = 0.1  // Updated to match other defaults
+    ): Promise<Map<string, IOptimizedSearchResponse>> {
+        const results = new Map<string, IOptimizedSearchResponse>();
+
+        // Generate embeddings for all queries in batch
+        const queryEmbeddings = await this.embeddingService.embedTexts(queries);
+
+        // Process each query
+        for (let i = 0; i < queries.length; i++) {
+            const query = queries[i];
+            const queryEmbedding = queryEmbeddings[i];
+
+            // Cache the embedding
+            this.queryEmbeddingCache.set(query, queryEmbedding);
+
+            // Perform search with pre-computed embedding
+            const response = await this.searchWithEmbedding({
+                queryEmbedding,
+                knowledgeVectors,
+                maxResults,
+                threshold
+            });
+
+            results.set(query, response);
+        }
+
+        return results;
+    }
+
+    /**
+     * Search with pre-computed query embedding
+     */
+    private async searchWithEmbedding(request: {
+        queryEmbedding: number[];
+        knowledgeVectors: Array<{
+            knowledgeId: string | number;
+            chunks: Array<{
+                content: string;
+                embedding: number[];
+                metadata: any;
+            }>;
+        }>;
+        maxResults: number;
+        threshold: number;
+    }): Promise<IOptimizedSearchResponse> {
+        const startTime = Date.now();
+        const { queryEmbedding, knowledgeVectors, maxResults, threshold } = request;
+
+        const allCandidates: IOptimizedSearchResult[] = [];
+        let totalCandidates = 0;
+
+        // Process all knowledge vectors in parallel
+        const searchPromises = knowledgeVectors.map(async (knowledgeVector) => {
+            const candidates: IOptimizedSearchResult[] = [];
+            
+            for (const chunk of knowledgeVector.chunks) {
+                totalCandidates++;
+                
+                if (!chunk.embedding || !Array.isArray(chunk.embedding)) {
+                    continue;
+                }
+
+                // Compute cosine similarity
+                const similarity = this.embeddingService.cosineSimilarity(queryEmbedding, chunk.embedding);
+                
+                if (similarity >= threshold) {
+                    candidates.push({
+                        content: chunk.content,
+                        score: similarity,
+                        metadata: chunk.metadata,
+                        knowledgeId: knowledgeVector.knowledgeId,
+                        chunkId: chunk.metadata?.id || `${knowledgeVector.knowledgeId}_chunk_${Date.now()}`
+                    });
+                }
+            }
+            
+            return candidates;
+        });
+
+        // Wait for all searches to complete
+        const allCandidateArrays = await Promise.all(searchPromises);
+        
+        // Flatten results
+        for (const candidates of allCandidateArrays) {
+            allCandidates.push(...candidates);
+        }
+
+        // Sort by similarity score and take top results
+        const results = allCandidates
+            .sort((a, b) => b.score - a.score)
+            .slice(0, maxResults);
+
+        const searchTime = Date.now() - startTime;
+
+        return {
+            results,
+            searchTime,
+            totalCandidates
+        };
+    }
+
+    /**
+     * Clear query embedding cache
+     */
+    clearCache(): void {
+        this.queryEmbeddingCache.clear();
+    }
+
+    /**
+     * Get cache statistics
+     */
+    getCacheStats(): { size: number; maxSize: number } {
+        return {
+            size: this.queryEmbeddingCache.size,
+            maxSize: this.maxCacheSize
+        };
+    }
+
+    /**
+     * Search similar documents (compatibility method from LangChainVectorStoreService)
+     */
+    async searchSimilarCompat(request: IVectorSearchRequest, knowledgeVectors: Array<{
+        knowledgeId: string | number;
+        chunks: Array<{
+            content: string;
+            embedding: number[];
+            metadata: any;
+        }>;
+    }>): Promise<IVectorSearchResponse> {
+        try {
+            const {
+                query,
+                maxResults = 10,
+                similarityThreshold = 0.1,  // Updated to match other defaults
+                filter,
+            } = request;
+
+            // Filter knowledge vectors if needed
+            let filteredVectors = knowledgeVectors;
+            if (filter) {
+                filteredVectors = knowledgeVectors.filter(vector => {
+                    // Check knowledge IDs
+                    if (filter.knowledgeIds && filter.knowledgeIds.length > 0) {
+                        return filter.knowledgeIds.includes(String(vector.knowledgeId));
+                    }
+                    return true;
+                });
+            }
+
+            // Use our optimized search
+            const searchResponse = await this.searchSimilar({
+                query,
+                knowledgeVectors: filteredVectors,
+                maxResults,
+                threshold: similarityThreshold
+            });
+
+            // Convert to IVectorSearchResponse format
+            const results: ISearchResult[] = searchResponse.results.map(result => ({
+                content: result.content,
+                score: result.score,
+                metadata: result.metadata,
+                chunkId: result.chunkId
+            }));
+
+            return {
+                results,
+                totalResults: results.length,
+            };
+        } catch (error) {
+            console.error('Failed to search similar documents:', error);
+            return {
+                results: [],
+                totalResults: 0,
+            };
+        }
+    }
+
+    /**
+     * Get search statistics
+     */
+    getStats(knowledgeVectors: Array<{
+        knowledgeId: string | number;
+        chunks: Array<{ content: string; embedding: number[]; metadata: any; }>;
+    }>): { totalChunks: number; totalKnowledge: number } {
+        const totalChunks = knowledgeVectors.reduce((total, vector) => total + vector.chunks.length, 0);
+        return {
+            totalChunks,
+            totalKnowledge: knowledgeVectors.length,
+        };
+    }
+}