@rws-framework/ai-tools 2.2.1 → 3.1.0

package/src/models/convo/EmbedLoader.ts

@@ -4,17 +4,19 @@ import { ConsoleService, RWSConfigService, RWSErrorCodes} from '@rws-framework/s
 import { InjectServices } from '@rws-framework/server/src/services/_inject';
 import RWSPrompt from '../prompts/_prompt';
 import { IRWSPromptJSON, ILLMChunk } from '../../types/IPrompt';
-import {VectorStoreService} from '../../services/VectorStoreService';
-import RWSVectorStore, { VectorDocType } from './VectorStore';
+
+import RWSVectorStore, { VectorDocType, IVectorStoreConfig } from './VectorStore';
 
 import { Document } from '@langchain/core/documents';
 import { UnstructuredLoader } from '@langchain/community/document_loaders/fs/unstructured';
+import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
 
 import { BaseChatModel  } from "@langchain/core/language_models/chat_models";
 import { BaseLanguageModelInterface, BaseLanguageModelInput } from '@langchain/core/language_models/base';
 import { Runnable } from '@langchain/core/runnables';
 import { BaseMessage } from '@langchain/core/messages';
-
+import { EmbeddingsInterface } from '@langchain/core/embeddings';
+import { CohereEmbeddings } from '@langchain/cohere';
 
 import { v4 as uuid } from 'uuid';
 import xml2js from 'xml2js';
@@ -59,6 +61,12 @@ interface IChainCallOutput {
     text: string
 }
 
+interface IEmbeddingsConfig {
+    provider: 'cohere';
+    apiKey: string;
+    model?: string;
+}
+
 interface IEmbeddingsHandler<T extends object> {
     generateEmbeddings: (text?: string) => Promise<T>
     storeEmbeddings: (embeddings: any, convoId: string) => Promise<void>
@@ -66,41 +74,64 @@ interface IEmbeddingsHandler<T extends object> {
 
 type LLMType = BaseLanguageModelInterface | Runnable<BaseLanguageModelInput, string> | Runnable<BaseLanguageModelInput, BaseMessage>;
 
-@InjectServices([VectorStoreService])
 class EmbedLoader<LLMChat extends BaseChatModel> {
     private loader: UnstructuredLoader;
-    private embeddings: IEmbeddingsHandler<any>;
+    private embeddings: EmbeddingsInterface;
+    private docSplitter: RecursiveCharacterTextSplitter;
 
     private docs: Document[] = [];
     private _initiated = false;
     private convo_id: string;        
     private llmChat: LLMChat;
-    private chatConstructor: new (config: any) => LLMChat;
+    
     private thePrompt: RWSPrompt;
-
-    vectorStoreService: VectorStoreService;
-    configService: RWSConfigService<IAiCfg>;
+    private vectorStoreConfig: IVectorStoreConfig;
+    
+    configService: RWSConfigService<any>;
 
     public _baseSplitterParams: ISplitterParams;    
     
-    constructor(
-        chatConstructor: new (config: any) => LLMChat, 
-        embeddings: IEmbeddingsHandler<any> | null = null, 
+    constructor(        
+        embeddingsConfig: IEmbeddingsConfig | null = null, 
         convoId: string | null = null, 
         baseSplitterParams: ISplitterParams = {
             chunkSize: 400, 
             chunkOverlap: 80, 
             separators: ['/n/n','.']
-        }
+        },
+        vectorStoreConfig: IVectorStoreConfig = { type: 'memory' }
     ) {
-        this.embeddings = embeddings;
+        if (embeddingsConfig) {
+            this.initializeEmbeddings(embeddingsConfig);
+        }
+        
         if(convoId === null) {
             this.convo_id = EmbedLoader.uuid();
         } else {
             this.convo_id = convoId;
         }                        
-        this.chatConstructor = chatConstructor;    
-        this._baseSplitterParams = baseSplitterParams;  
+        
+        this._baseSplitterParams = baseSplitterParams;
+        this.vectorStoreConfig = vectorStoreConfig;
+
+        this.docSplitter = new RecursiveCharacterTextSplitter({
+            chunkSize: baseSplitterParams.chunkSize,
+            chunkOverlap: baseSplitterParams.chunkOverlap,
+            separators: baseSplitterParams.separators
+        });
+    }
+
+    private initializeEmbeddings(config: IEmbeddingsConfig): void {
+        switch (config.provider) {
+            case 'cohere':
+                this.embeddings = new CohereEmbeddings({
+                    apiKey: config.apiKey,
+                    model: config.model || 'embed-english-v3.0'
+                });
+                break;
+            default:
+                throw new Error(`Unsupported embedding provider: ${config.provider}`);
+        }
     }
 
     static uuid(): string
@@ -133,7 +164,6 @@ class EmbedLoader<LLMChat extends BaseChatModel> {
 
     async splitDocs(filePath: string, params: ISplitterParams): Promise<RWSVectorStore>
     {
-
         if(!this.embeddings){
             throw new Error('No embeddings provided for ConvoLoader\'s constructor. ConvoLoader.splitDocs aborting...');
         }
@@ -145,23 +175,17 @@ class EmbedLoader<LLMChat extends BaseChatModel> {
             console.log(`Split dir ${ConsoleService.color().magentaBright(splitDir)} doesn't exist. Splitting docs...`);
             this.loader = new UnstructuredLoader(filePath);
 
-            // this.docSplitter = new RecursiveCharacterTextSplitter({
-            //     chunkSize: params.chunkSize, // The size of the chunk that should be split.
-            //     chunkOverlap: params.chunkOverlap, // Adding overalap so that if a text is broken inbetween, next document may have part of the previous document 
-            //     separators: params.separators // In this case we are assuming that /n/n would mean one whole sentence. In case there is no nearing /n/n then "." will be used instead. This can be anything that helps derive a complete sentence .
-            // });
-
             fs.mkdirSync(splitDir, { recursive: true });
             
             const orgDocs = await this.loader.load();
-            const splitDocs: any[] = [];//await this.docSplitter.splitDocuments(orgDocs);
+            const splitDocs = await this.docSplitter.splitDocuments(orgDocs);
 
             const avgCharCountPre = this.avgDocLength(orgDocs);
             const avgCharCountPost = this.avgDocLength(splitDocs);
 
             logConvo(`Average length among ${orgDocs.length} documents loaded is ${avgCharCountPre} characters.`);
             logConvo(`After the split we have ${splitDocs.length} documents more than the original ${orgDocs.length}.`);
-            logConvo(`Average length among ${orgDocs.length} documents (after split) is ${avgCharCountPost} characters.`);
+            logConvo(`Average length among ${splitDocs.length} documents (after split) is ${avgCharCountPost} characters.`);
 
             let i = 0;
             splitDocs.forEach((doc: Document) => {
@@ -177,17 +201,75 @@ class EmbedLoader<LLMChat extends BaseChatModel> {
                 finalDocs.push(new Document({ pageContent: txt }));              
             }
         }
-        
-        return await this.vectorStoreService.createStore(finalDocs, await this.embeddings.generateEmbeddings());
+                
+        const vectorStore = new RWSVectorStore(finalDocs, this.embeddings, this.vectorStoreConfig);
+        return await vectorStore.init();
     }
 
     async similaritySearch(query: string, splitCount: number, store: RWSVectorStore): Promise<string>
     {
         console.log('Store is ready. Searching for embedds...');            
-        const texts = await store.getFaiss().similaritySearchWithScore(`${query}`, splitCount);
+        const texts = await store.similaritySearchWithScore(query, splitCount);
         console.log('Found best parts: ' + texts.length);
         return texts.map(([doc, score]: [any, number]) => `${doc['pageContent']}`).join('\n\n');    
     }
+
+    /**
+     * Index text content directly without file loading
+     */
+    async indexTextContent(
+        content: string, 
+        documentId: string | number, 
+        metadata: Record<string, any> = {}
+    ): Promise<RWSVectorStore> {
+        if (!this.embeddings) {
+            throw new Error('No embeddings provided for ConvoLoader. Cannot index text content.');
+        }
+
+        // Split the content into chunks
+        const docs = await this.docSplitter.createDocuments([content], [{
+            documentId,
+            ...metadata
+        }]);
+
+        // Create and initialize vector store
+        const vectorStore = new RWSVectorStore(docs, this.embeddings, this.vectorStoreConfig);
+        return await vectorStore.init();
+    }
+
+    /**
+     * Search for similar content with detailed results
+     */
+    async searchSimilarWithDetails(
+        query: string, 
+        store: RWSVectorStore, 
+        maxResults: number = 5,
+        threshold: number = 0.7
+    ): Promise<Array<{ content: string; score: number; metadata: any }>> {
+        const results = await store.similaritySearchWithScore(query, maxResults);
+        
+        return results
+            .filter(([_, score]) => score >= threshold)
+            .map(([doc, score]) => ({
+                content: doc.pageContent,
+                score,
+                metadata: doc.metadata || {}
+            }));
+    }
+
+    /**
+     * Get or create embeddings instance
+     */
+    getEmbeddings(): EmbeddingsInterface {
+        return this.embeddings;
+    }
+
+    /**
+     * Update embeddings configuration
+     */
+    updateEmbeddingsConfig(config: IEmbeddingsConfig): void {
+        this.initializeEmbeddings(config);
+    }
     
     private async debugCall(debugCallback: (debugData: IConvoDebugXMLData) => Promise<IConvoDebugXMLData> = null)
     {
@@ -300,4 +382,4 @@ class EmbedLoader<LLMChat extends BaseChatModel> {
 
 }
 
-export { EmbedLoader, IChainCallOutput, IConvoDebugXMLData, IEmbeddingsHandler, ISplitterParams, IBaseLangchainHyperParams };
+export { EmbedLoader, IChainCallOutput, IConvoDebugXMLData, IEmbeddingsHandler, IEmbeddingsConfig, ISplitterParams, IBaseLangchainHyperParams };

package/src/models/convo/VectorStore.ts

@@ -1,36 +1,114 @@
 import { FaissStore } from '@langchain/community/vectorstores/faiss';
+import { MemoryVectorStore } from 'langchain/vectorstores/memory';
 import { EmbeddingsInterface } from '@langchain/core/embeddings';
 import { Document } from '@langchain/core/documents';
 
 type VectorDocType = Document<Record<string, any>>[];
 
+export interface IVectorStoreConfig {
+    type: 'faiss' | 'memory';
+    persistPath?: string;
+}
+
 export default class RWSVectorStore 
 {
-    private faiss: FaissStore;
+    private vectorStore: FaissStore | MemoryVectorStore;
     private docs: VectorDocType;
     private embeddings: EmbeddingsInterface;
+    private config: IVectorStoreConfig;
 
-    constructor(docs: VectorDocType, embeddings: EmbeddingsInterface){
+    constructor(docs: VectorDocType, embeddings: EmbeddingsInterface, config: IVectorStoreConfig = { type: 'memory' }){
         this.docs = docs;
         this.embeddings = embeddings;
+        this.config = config;
     }
 
     async init(): Promise<RWSVectorStore>
     {
-        this.faiss = await FaissStore.fromDocuments(this.docs, this.embeddings);
+        if (this.config.type === 'faiss') {
+            this.vectorStore = await FaissStore.fromDocuments(this.docs, this.embeddings);
+        } else {
+            this.vectorStore = await MemoryVectorStore.fromDocuments(this.docs, this.embeddings);
+        }
 
         return this;
     }
 
+    getVectorStore(): FaissStore | MemoryVectorStore
+    {
+        return this.vectorStore;
+    }
+
     getFaiss(): FaissStore
     {
-        return this.faiss;
+        if (this.vectorStore instanceof FaissStore) {
+            return this.vectorStore;
+        }
+        throw new Error('Vector store is not a FAISS instance');
+    }
+
+    getMemoryStore(): MemoryVectorStore
+    {
+        if (this.vectorStore instanceof MemoryVectorStore) {
+            return this.vectorStore;
+        }
+        throw new Error('Vector store is not a Memory instance');
     }
 
     getDocs()
     {
         return this.docs;
     }
+
+    /**
+     * Add more documents to the vector store
+     */
+    async addDocuments(newDocs: VectorDocType): Promise<void> {
+        await this.vectorStore.addDocuments(newDocs);
+        this.docs.push(...newDocs);
+    }
+
+    /**
+     * Search for similar documents
+     */
+    async similaritySearchWithScore(query: string, k: number = 4): Promise<[Document, number][]> {
+        return await this.vectorStore.similaritySearchWithScore(query, k);
+    }
+
+    /**
+     * Search for similar documents using vector
+     */
+    async similaritySearchVectorWithScore(embedding: number[], k: number = 4): Promise<[Document, number][]> {
+        return await this.vectorStore.similaritySearchVectorWithScore(embedding, k);
+    }
+
+    /**
+     * Delete documents (if supported)
+     */
+    async deleteDocuments(ids: string[]): Promise<void> {
+        if ('delete' in this.vectorStore) {
+            await (this.vectorStore as any).delete({ ids });
+        }
+    }
+
+    /**
+     * Save the vector store (FAISS only)
+     */
+    async save(path?: string): Promise<void> {
+        if (this.vectorStore instanceof FaissStore) {
+            await this.vectorStore.save(path || this.config.persistPath || './vector_store');
+        }
+    }
+
+    /**
+     * Load a vector store from disk (FAISS only)
+     */
+    static async load(path: string, embeddings: EmbeddingsInterface): Promise<RWSVectorStore> {
+        const faissStore = await FaissStore.load(path, embeddings);
+        const vectorStore = new RWSVectorStore([], embeddings, { type: 'faiss', persistPath: path });
+        vectorStore.vectorStore = faissStore;
+        return vectorStore;
+    }
 }
 
 export {

package/src/models/prompts/_prompt.ts

@@ -69,7 +69,7 @@ class RWSPrompt implements IPromptInstance {
     }
 
     // Delegation methods for input/output management
-    listen(source: string, stream: boolean = true): RWSPrompt {
+    listen(source: string | object, stream: boolean = true): RWSPrompt {        
         this.ioManager.listen(source, stream);
         return this;
     }
@@ -108,7 +108,7 @@ class RWSPrompt implements IPromptInstance {
         return this;
     }
 
-    readOutput(): string {
+    readOutput(): string | object {
         return this.ioManager.readOutput();
     }
 

package/src/models/prompts/inc/execution-methods-handler.ts

@@ -7,11 +7,11 @@ import {
 } from './types';
 
 export interface IPromptInstance {
-    readOutput(): string;
+    readOutput(): string | object;
     readInput(): any[];
     getInput(): any[];
     setSentInput(input: any[]): void;
-    injestOutput(content: string): void;
+    injestOutput(content: string | object): void;
 }
 
 export class ExecutionMethodsHandler {

package/src/models/prompts/inc/input-output-manager.ts

@@ -5,7 +5,7 @@ export class InputOutputManager {
     private enhancedInput: IPromptEnchantment[] = [];
     private sentInput: CompoundInput[] = [];
     private originalInput: CompoundInput[] = [];
-    private output: string = '';
+    private output: string | object = null;
     private onStream: (chunk: string) => void = () => {};
 
     constructor(input: CompoundInput[]) {
@@ -54,18 +54,20 @@ export class InputOutputManager {
         this.output = content;
     }
 
-    readOutput(): string {
+    readOutput(): string | object{
         return this.output;
     }
 
-    listen(source: string, stream: boolean = true): void {
-        this.output = '';
+    listen(source: string | object, stream: boolean = true): void {
+        if(stream){
+            this.output = '';
+        }        
 
         if (!stream) {
             this.output = source;
         } else {
-            this.output += source;
-            this.onStream(source);
+            this.output += source as string;
+            this.onStream(source as string);
         }
     }
 
@@ -114,7 +116,7 @@ export class InputOutputManager {
         return this.originalInput;
     }
 
-    getOutput(): string {
+    getOutput(): string | object {
         return this.output;
     }
 }

package/src/services/LangChainEmbeddingService.ts

@@ -0,0 +1,222 @@
+import { Injectable } from '@nestjs/common';
+import { Embeddings } from '@langchain/core/embeddings';
+import { CohereEmbeddings } from '@langchain/cohere';
+import { Document } from '@langchain/core/documents';
+import { IEmbeddingConfig, IChunkConfig } from '../types';
+import { TextChunker } from './TextChunker';
+import RWSVectorStore, { VectorDocType, IVectorStoreConfig } from '../models/convo/VectorStore';
+
+@Injectable()
+export class LangChainEmbeddingService {
+    private embeddings: Embeddings;
+    private config: IEmbeddingConfig;
+    private chunkConfig: IChunkConfig;
+    private isInitialized = false;
+    private vectorStore: RWSVectorStore | null = null;
+
+    constructor() {
+        // Empty constructor for NestJS dependency injection
+    }
+
+    /**
+     * Initialize the service with configuration
+     */
+    async initialize(config: IEmbeddingConfig, chunkConfig?: IChunkConfig): Promise<void> {
+        if (this.isInitialized) {
+            return;
+        }
+
+        this.config = config;
+        this.chunkConfig = chunkConfig || {
+            chunkSize: 1000,
+            chunkOverlap: 200
+        };
+        this.initializeEmbeddings();
+        this.isInitialized = true;
+    }
+
+    /**
+     * Alternative constructor-like method for backward compatibility
+     */
+    static create(config: IEmbeddingConfig, chunkConfig?: IChunkConfig): LangChainEmbeddingService {
+        const service = new LangChainEmbeddingService();
+        service.config = config;
+        service.chunkConfig = chunkConfig || {
+            chunkSize: 1000,
+            chunkOverlap: 200
+        };
+        service.initializeEmbeddings();
+        service.isInitialized = true;
+        return service;
+    }
+
+    private initializeEmbeddings(): void {
+        switch (this.config.provider) {
+            case 'cohere':
+                this.embeddings = new CohereEmbeddings({
+                    apiKey: this.config.apiKey,
+                    model: this.config.model || 'embed-v4.0',
+                    batchSize: this.config.batchSize || 96
+                });
+                break;
+                
+            default:
+                throw new Error(`Unsupported embedding provider: ${this.config.provider}`);
+        }
+    }
+
+    private initializeTextSplitter(chunkConfig?: IChunkConfig): void {
+        // Text chunking is now handled by TextChunker class
+        // This method is kept for compatibility but doesn't initialize anything
+    }
+
+    /**
+     * Generate embeddings for multiple texts
+     */
+    async embedTexts(texts: string[]): Promise<number[][]> {
+        this.ensureInitialized();
+        return await this.embeddings.embedDocuments(texts);
+    }
+
+    /**
+     * Generate embedding for a single text
+     */
+    async embedText(text: string): Promise<number[]> {
+        this.ensureInitialized();
+        return await this.embeddings.embedQuery(text);
+    }
+
+    /**
+     * Split text into chunks
+     */
+    async chunkText(text: string): Promise<string[]> {
+        this.ensureInitialized();
+        
+        // Use our custom TextChunker instead of LangChain's splitter
+        // Use safe token limits - the TextChunker handles token estimation internally
+        const maxTokens = 450; // Safe token limit for embedding models
+        const overlap = this.chunkConfig?.chunkOverlap || 50; // Character overlap, not token
+        
+        return TextChunker.chunkText(text, maxTokens, overlap);
+    }
+
+    /**
+     * Split text and generate embeddings for chunks
+     */
+    async chunkAndEmbed(text: string): Promise<{ text: string; embedding: number[] }[]> {
+        this.ensureInitialized();
+        const chunks = await this.chunkText(text);
+        const embeddings = await this.embedTexts(chunks);
+        
+        return chunks.map((chunk, index) => ({
+            text: chunk,
+            embedding: embeddings[index]
+        }));
+    }
+
+    /**
+     * Create LangChain documents from text with metadata
+     */
+    async createDocuments(text: string, metadata: Record<string, any> = {}): Promise<Document[]> {
+        this.ensureInitialized();
+        const chunks = await this.chunkText(text);
+        
+        return chunks.map((chunk, index) => new Document({
+            pageContent: chunk,
+            metadata: {
+                ...metadata,
+                chunkIndex: index,
+                id: `${metadata.documentId || 'doc'}_chunk_${index}`
+            }
+        }));
+    }
+
+    /**
+     * Get the underlying LangChain embeddings instance
+     */
+    getEmbeddingsInstance(): Embeddings {
+        this.ensureInitialized();
+        return this.embeddings;
+    }
+
+    /**
+     * Update configuration and reinitialize
+     */
+    updateConfig(newConfig: Partial<IEmbeddingConfig>): void {
+        this.config = { ...this.config, ...newConfig };
+        this.initializeEmbeddings();
+    }
+
+    /**
+     * Calculate cosine similarity between two vectors
+     */
+    cosineSimilarity(vecA: number[], vecB: number[]): number {
+        if (vecA.length !== vecB.length) {
+            throw new Error('Vectors must have the same length');
+        }
+
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+
+        for (let i = 0; i < vecA.length; i++) {
+            dotProduct += vecA[i] * vecB[i];
+            normA += vecA[i] * vecA[i];
+            normB += vecB[i] * vecB[i];
+        }
+
+        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
+
+    /**
+     * Ensure the service is initialized
+     */
+    private ensureInitialized(): void {
+        if (!this.isInitialized) {
+            throw new Error('LangChainEmbeddingService not initialized. Call initialize() first.');
+        }
+    }
+
+    /**
+     * Create a vector store for similarity search like the LangChain tutorial
+     * This allows us to use vectorStore.similaritySearch() just like in the tutorial
+     */
+    async createVectorStore(documents: Document[], config?: { type?: 'memory' | 'faiss'; persistPath?: string }): Promise<RWSVectorStore> {
+        this.ensureInitialized();
+        
+        const vectorStoreConfig = {
+            type: config?.type || 'memory' as const,
+            persistPath: config?.persistPath
+        };
+        
+        const vectorStore = await new RWSVectorStore(
+            documents, 
+            this.embeddings, 
+            vectorStoreConfig
+        ).init();
+        
+        return vectorStore;
+    }
+
+    /**
+     * Perform similarity search on a vector store (tutorial-style interface)
+     * Usage: const results = await embeddingService.similaritySearch(vectorStore, query, k)
+     */
+    async similaritySearch(vectorStore: RWSVectorStore, query: string, k: number = 4): Promise<Document[]> {
+        this.ensureInitialized();
+        
+        // Use RWSVectorStore's similarity search (returns documents without scores)
+        const resultsWithScores = await vectorStore.similaritySearchWithScore(query, k);
+        return resultsWithScores.map(([doc, _score]) => doc);
+    }
+
+    /**
+     * Perform similarity search with scores (tutorial-style interface)
+     * Usage: const results = await embeddingService.similaritySearchWithScore(vectorStore, query, k)
+     */
+    async similaritySearchWithScore(vectorStore: RWSVectorStore, query: string, k: number = 4): Promise<[Document, number][]> {
+        this.ensureInitialized();
+        
+        return await vectorStore.similaritySearchWithScore(query, k);
+    }
+}