@rws-framework/ai-tools 2.2.0 → 3.0.0

package/docs/tutorial-style-rag.md

@@ -0,0 +1,124 @@
+# Tutorial-Style RAG with RWSVectorStore
+
+This document shows how to use our ai-tools services in the same way as the LangChain tutorial, but with RWSVectorStore as the backend.
+
+## Quick Start
+
+```typescript
+import { LangChainEmbeddingService } from '@rws-framework/ai-tools';
+import { Document } from '@langchain/core/documents';
+
+// Initialize embedding service
+const embeddingService = new LangChainEmbeddingService();
+await embeddingService.initialize({
+    provider: 'cohere',
+    apiKey: process.env.COHERE_API_KEY,
+    model: 'embed-v4.0'
+});
+
+// Create documents (like tutorial's document loading)
+const documents = [
+    new Document({
+        pageContent: "Task decomposition breaks complex tasks into steps.",
+        metadata: { source: 'tutorial' }
+    }),
+    // ... more documents
+];
+
+// Create vector store (tutorial-style)
+const vectorStore = await embeddingService.createVectorStore(documents);
+
+// Similarity search (tutorial-style)
+const results = await embeddingService.similaritySearch(vectorStore, "What is task decomposition?", 3);
+```
+
+## Comparison with LangChain Tutorial
+
+### LangChain Tutorial Approach:
+```typescript
+// Tutorial code
+const vectorStore = new MemoryVectorStore(embeddings);
+await vectorStore.addDocuments(documents);
+const results = await vectorStore.similaritySearch(query, k);
+```
+
+### Our AI-Tools Approach:
+```typescript
+// Our equivalent code using RWSVectorStore
+const vectorStore = await embeddingService.createVectorStore(documents);
+const results = await embeddingService.similaritySearch(vectorStore, query, k);
+```
+
+## Available Methods
+
+### 1. Simple Similarity Search
+```typescript
+const docs = await embeddingService.similaritySearch(vectorStore, query, k);
+// Returns: Document[]
+```
+
+### 2. Similarity Search with Scores
+```typescript
+const results = await embeddingService.similaritySearchWithScore(vectorStore, query, k);
+// Returns: [Document, number][]
+```
+
+### 3. Enhanced Search with Filters
+```typescript
+const results = await vectorStoreService.searchSimilar({
+    query: "your query",
+    maxResults: 5,
+    similarityThreshold: 0.1,
+    filter: {
+        knowledgeIds: ['28'],
+        documentIds: ['doc1', 'doc2']
+    }
+});
+// Returns: IVectorSearchResponse
+```
+
+## Integration with RAG Module
+
+The RAG module already uses these services:
+
+```typescript
+// In backend/src/app/rag_module/rag.service.ts
+constructor(
+    private embeddingService: LangChainEmbeddingService,
+    private langChainRAGService: LangChainRAGService
+) {}
+```
+
+## Benefits of Our Approach
+
+1. **Tutorial Compatibility**: Same interface as LangChain tutorial
+2. **RWSVectorStore Backend**: Uses our proven vector storage system  
+3. **Memory & FAISS Support**: Can use both in-memory and persistent storage
+4. **Knowledge Filtering**: Built-in support for multi-tenant scenarios
+5. **Polish Content Optimized**: Similarity thresholds tuned for non-English content
+
+## Configuration
+
+```typescript
+// Centralized configuration (from RAG service)
+static EMBEDDING_CONFIG = {
+    provider: 'cohere',
+    model: 'embed-v4.0',
+    batchSize: 96
+};
+
+static RAG_CONFIG = {
+    search: {
+        defaultSimilarityThreshold: 0.1,  // Optimized for Polish content
+        maxResults: 5
+    },
+    vectorStore: {
+        type: 'memory',  // or 'faiss' for persistence
+        autoSave: true
+    }
+};
+```
+
+## Examples
+
+See `/examples/tutorial-style-rag.ts` for complete working examples.

package/examples/test-recursive-chunker.ts

@@ -0,0 +1,167 @@
+/**
+ * Test the new RecursiveCharacterTextSplitter approach in TextChunker
+ * This verifies the tutorial-style chunking is working correctly
+ */
+
+import { TextChunker } from '../src/services/TextChunker';
+
+function testRecursiveChunking() {
+    console.log('🧪 Testing RecursiveCharacterTextSplitter approach in TextChunker\n');
+
+    // Test document with various separator types
+    const testDocument = `
+# Introduction to AI
+
+Artificial Intelligence (AI) is a rapidly evolving field that encompasses various technologies and methodologies. AI systems can process vast amounts of data, recognize patterns, and make decisions.
+
+## Machine Learning
+
+Machine learning is a subset of AI that focuses on algorithms that can learn from data. There are several types of machine learning:
+
+1. Supervised Learning: Uses labeled data to train models
+2. Unsupervised Learning: Finds patterns in unlabeled data  
+3. Reinforcement Learning: Learns through interaction with environment
+
+### Deep Learning
+
+Deep learning uses neural networks with multiple layers to process information. These networks can automatically extract features from raw data. Common applications include:
+
+- Image recognition and computer vision
+- Natural language processing and understanding
+- Speech recognition and synthesis
+- Autonomous vehicles and robotics
+
+## Applications and Impact
+
+AI technologies are transforming various industries. Healthcare uses AI for diagnosis and treatment planning. Finance leverages AI for fraud detection and algorithmic trading. Manufacturing employs AI for quality control and predictive maintenance.
+
+The ethical implications of AI are increasingly important! Questions about bias, privacy, and job displacement need careful consideration? As AI becomes more prevalent, ensuring responsible development and deployment is crucial; we must balance innovation with ethical considerations.
+    `.trim();
+
+    console.log('📄 Original document:');
+    console.log(`Length: ${testDocument.length} characters`);
+    console.log(`Estimated tokens: ${TextChunker.estimateTokens(testDocument)}`);
+    console.log('First 200 chars:', testDocument.substring(0, 200) + '...\n');
+
+    // Test different chunking configurations
+    const testConfigs = [
+        { maxTokens: 150, overlap: 20, name: 'Small chunks with overlap' },
+        { maxTokens: 300, overlap: 50, name: 'Medium chunks with overlap' },
+        { maxTokens: 100, overlap: 0, name: 'Small chunks no overlap' }
+    ];
+
+    for (const config of testConfigs) {
+        console.log(`\n🔧 Testing: ${config.name}`);
+        console.log(`Max tokens: ${config.maxTokens}, Overlap: ${config.overlap}`);
+        
+        const chunks = TextChunker.chunkText(testDocument, config.maxTokens, config.overlap);
+        
+        console.log(`\n✅ Generated ${chunks.length} chunks:`);
+        
+        chunks.forEach((chunk, index) => {
+            const tokens = TextChunker.estimateTokens(chunk);
+            const withinLimit = tokens <= config.maxTokens;
+            const status = withinLimit ? '✅' : '❌';
+            
+            console.log(`${status} Chunk ${index + 1}: ${tokens} tokens, ${chunk.length} chars`);
+            console.log(`   Preview: "${chunk.substring(0, 80)}..."`);
+            
+            if (!withinLimit) {
+                console.log(`   ⚠️  WARNING: Chunk exceeds token limit (${tokens} > ${config.maxTokens})`);
+            }
+        });
+
+        // Check for overlaps if configured
+        if (config.overlap > 0 && chunks.length > 1) {
+            console.log('\n🔄 Checking overlaps:');
+            for (let i = 1; i < chunks.length; i++) {
+                const prevChunk = chunks[i - 1];
+                const currentChunk = chunks[i];
+                
+                // Simple overlap detection - check if chunks share common words
+                const prevWords = prevChunk.split(' ').slice(-10);
+                const currentWords = currentChunk.split(' ').slice(0, 10);
+                
+                const commonWords = prevWords.filter(word => 
+                    currentWords.some(cWord => 
+                        word.length > 3 && cWord.includes(word.substring(0, Math.min(word.length, 5)))
+                    )
+                );
+                
+                if (commonWords.length > 0) {
+                    console.log(`   📎 Chunk ${i} has overlap with chunk ${i}: "${commonWords.slice(0, 3).join(', ')}"...`);
+                } else {
+                    console.log(`   ❓ Chunk ${i} may not have sufficient overlap with previous chunk`);
+                }
+            }
+        }
+    }
+}
+
+function testDocumentCreation() {
+    console.log('\n\n📋 Testing Document Creation (Tutorial Style)');
+    
+    const sampleText = `
+Artificial Intelligence is transforming the world. Machine learning algorithms can process vast amounts of data and identify patterns that humans might miss.
+
+Deep learning, a subset of machine learning, uses neural networks to solve complex problems. These networks consist of multiple layers that can automatically extract features from raw data.
+    `.trim();
+
+    const documents = TextChunker.createDocumentsFromChunks(
+        sampleText,
+        { 
+            documentId: 'ai-intro',
+            source: 'tutorial',
+            category: 'technology'
+        },
+        200, // maxTokens
+        30   // overlap
+    );
+
+    console.log(`\n📄 Created ${documents.length} documents:`);
+    
+    documents.forEach((doc, index) => {
+        console.log(`\nDocument ${index + 1}:`);
+        console.log(`  ID: ${doc.metadata.id}`);
+        console.log(`  Chunk: ${doc.metadata.chunkIndex + 1}/${doc.metadata.totalChunks}`);
+        console.log(`  Content: "${doc.pageContent.substring(0, 100)}..."`);
+        console.log(`  Tokens: ${TextChunker.estimateTokens(doc.pageContent)}`);
+    });
+}
+
+function testEdgeCases() {
+    console.log('\n\n🧪 Testing Edge Cases');
+
+    const testCases = [
+        { name: 'Empty string', text: '' },
+        { name: 'Very short text', text: 'Hello world!' },
+        { name: 'Single long word', text: 'Supercalifragilisticexpialidocious'.repeat(20) },
+        { name: 'No separators', text: 'abcdefghijklmnopqrstuvwxyz'.repeat(50) },
+        { name: 'Only separators', text: '\n\n\n. . . ! ! ! ? ? ?' }
+    ];
+
+    testCases.forEach(testCase => {
+        console.log(`\n🔬 Testing: ${testCase.name}`);
+        try {
+            const chunks = TextChunker.chunkText(testCase.text, 100, 20);
+            console.log(`   ✅ Generated ${chunks.length} chunks`);
+            if (chunks.length > 0) {
+                console.log(`   First chunk: "${chunks[0].substring(0, 50)}${chunks[0].length > 50 ? '...' : ''}"`);
+            }
+        } catch (error) {
+            console.log(`   ❌ Error: ${error.message}`);
+        }
+    });
+}
+
+// Run all tests
+console.log('🚀 Starting RecursiveCharacterTextSplitter Tests\n');
+console.log('='.repeat(60));
+
+testRecursiveChunking();
+testDocumentCreation();
+testEdgeCases();
+
+console.log('\n' + '='.repeat(60));
+console.log('✅ Tests completed! The TextChunker now follows LangChain tutorial approach.');
+console.log('📚 It uses RecursiveCharacterTextSplitter-like logic with hierarchical separators.');

package/examples/tutorial-style-rag.ts

@@ -0,0 +1,153 @@
+/**
+ * Example: LangChain Tutorial-Style RAG with RWSVectorStore
+ * 
+ * This demonstrates how to use our ai-tools services in the same way
+ * as the LangChain tutorial, but with RWSVectorStore backend.
+ */
+
+import { LangChainEmbeddingService } from '../src/services/LangChainEmbeddingService';
+import { LangChainVectorStoreService } from '../src/services/LangChainVectorStoreService';
+import { Document } from '@langchain/core/documents';
+
+async function tutorialStyleRAGExample() {
+    // Initialize services like the tutorial
+    const embeddingService = new LangChainEmbeddingService();
+    const vectorStoreService = new LangChainVectorStoreService();
+
+    // Configure embeddings (Cohere instead of OpenAI)
+    await embeddingService.initialize({
+        provider: 'cohere',
+        apiKey: process.env.COHERE_API_KEY || '',
+        model: 'embed-v4.0',
+        batchSize: 96
+    });
+
+    // Initialize vector store service
+    await vectorStoreService.initialize(embeddingService, {
+        type: 'memory',
+        similarityThreshold: 0.1,
+        maxResults: 5
+    });
+
+    // Sample documents (like the tutorial's blog post chunks)
+    const sampleTexts = [
+        "Task decomposition is the process of breaking down complex tasks into smaller, more manageable steps.",
+        "Chain of Thought (CoT) prompting helps models think step by step to solve complex problems.",
+        "Tree of Thoughts extends CoT by exploring multiple reasoning possibilities at each step.",
+        "RAG (Retrieval Augmented Generation) combines retrieval and generation for better answers.",
+        "Vector databases store embeddings to enable semantic search over documents."
+    ];
+
+    // Create documents from texts (like tutorial's document loading)
+    const documents: Document[] = [];
+    for (let i = 0; i < sampleTexts.length; i++) {
+        documents.push(new Document({
+            pageContent: sampleTexts[i],
+            metadata: {
+                id: `doc_${i}`,
+                source: 'tutorial_example',
+                chunkIndex: i
+            }
+        }));
+    }
+
+    // Add documents to vector store (like tutorial's vectorStore.addDocuments)
+    await vectorStoreService.addDocuments(documents);
+
+    // Create a vector store for similarity search (tutorial-style)
+    const vectorStore = await embeddingService.createVectorStore(documents, { type: 'memory' });
+
+    // Perform similarity search like the tutorial
+    const query = "What is task decomposition?";
+    console.log(`\n🔍 Searching for: "${query}"`);
+
+    // Method 1: Tutorial-style similarity search (returns documents only)
+    const similarDocs = await embeddingService.similaritySearch(vectorStore, query, 3);
+    console.log('\n📄 Similar documents (tutorial-style):');
+    similarDocs.forEach((doc, index) => {
+        console.log(`${index + 1}. ${doc.pageContent}`);
+    });
+
+    // Method 2: Similarity search with scores (tutorial-style)
+    const similarDocsWithScores = await embeddingService.similaritySearchWithScore(vectorStore, query, 3);
+    console.log('\n📊 Similar documents with scores:');
+    similarDocsWithScores.forEach(([doc, score], index) => {
+        console.log(`${index + 1}. Score: ${score.toFixed(4)} - ${doc.pageContent}`);
+    });
+
+    // Method 3: Using the vector store service (our enhanced approach)
+    const searchResults = await vectorStoreService.searchSimilar({
+        query,
+        maxResults: 3,
+        similarityThreshold: 0.1
+    });
+
+    console.log('\n🎯 Enhanced search results:');
+    searchResults.results.forEach((result, index) => {
+        console.log(`${index + 1}. Score: ${result.score.toFixed(4)} - ${result.content}`);
+        console.log(`   Chunk ID: ${result.chunkId}`);
+    });
+
+    return {
+        tutorialStyle: similarDocs,
+        withScores: similarDocsWithScores,
+        enhanced: searchResults.results
+    };
+}
+
+// Example usage with knowledge filtering (like our current RAG system)
+async function knowledgeFilteredExample() {
+    const embeddingService = new LangChainEmbeddingService();
+    const vectorStoreService = new LangChainVectorStoreService();
+
+    await embeddingService.initialize({
+        provider: 'cohere',
+        apiKey: process.env.COHERE_API_KEY || '',
+        model: 'embed-v4.0'
+    });
+
+    await vectorStoreService.initialize(embeddingService, {
+        type: 'memory',
+        similarityThreshold: 0.1
+    });
+
+    // Documents with knowledge IDs (like our current system)
+    const documents = [
+        new Document({
+            pageContent: "Testing prototypes is crucial for product development",
+            metadata: { knowledgeId: '28', documentId: 'test_doc', chunkIndex: 0 }
+        }),
+        new Document({
+            pageContent: "Quality assurance ensures product reliability",
+            metadata: { knowledgeId: '28', documentId: 'test_doc', chunkIndex: 1 }
+        }),
+        new Document({
+            pageContent: "User feedback drives iterative improvements",
+            metadata: { knowledgeId: '29', documentId: 'feedback_doc', chunkIndex: 0 }
+        })
+    ];
+
+    await vectorStoreService.addDocuments(documents);
+
+    // Search with knowledge filtering (like our RAG system does)
+    const results = await vectorStoreService.searchSimilar({
+        query: "opisz założenia dokumentu",
+        maxResults: 2,
+        similarityThreshold: 0.1,
+        filter: {
+            knowledgeIds: ['28']  // Only search in knowledge 28
+        }
+    });
+
+    console.log('\n🔍 Knowledge-filtered search results:');
+    results.results.forEach((result, index) => {
+        console.log(`${index + 1}. Score: ${result.score.toFixed(4)}`);
+        console.log(`   Content: ${result.content}`);
+        console.log(`   Knowledge ID: ${result.metadata.knowledgeId}`);
+    });
+
+    return results;
+}
+
+// Export for use in tests or other modules
+export { tutorialStyleRAGExample, knowledgeFilteredExample };

package/package.json

@@ -1,15 +1,16 @@
 {
   "name": "@rws-framework/ai-tools",
   "private": false,
-  "version": "2.2.0",
+  "version": "3.0.0",
   "description": "",
   "main": "src/index.ts",
   "scripts": {},
   "author": "papablack",
   "license": "ISC",
   "dependencies": {
-    "@langchain/community": "0.3.28",
-    "@langchain/core": "0.3.37",
+    "@langchain/cohere": "^0.3.4",
+    "@langchain/community": "^0.3.55",
+    "@langchain/core": "^0.3.75",
     "@rws-framework/server": "3.*",
     "@rws-framework/db": "*",
     "@rws-framework/console": "*",

package/src/index.ts

@@ -3,13 +3,15 @@ import RWSPrompt, { IChainCallOutput } from '@rws-framework/ai-tools/src/models/
 import { ILLMChunk, IRWSPromptRequestExecutor, IRWSSinglePromptRequestExecutor, IRWSPromptStreamExecutor, IRWSPromptJSON, ChainStreamType, IAIRequestOptions, IAITool, IAIToolSchema, IAIToolParameter, IToolCall, ToolHandler } from './types/IPrompt';
 import { EmbedLoader as RWSEmbed, IConvoDebugXMLData, IEmbeddingsHandler, ISplitterParams } from './models/convo/EmbedLoader';
 import RWSVectorStore from './models/convo/VectorStore';
-import { VectorStoreService } from './services/VectorStoreService';
+import { LangChainEmbeddingService } from './services/LangChainEmbeddingService';
+import { LangChainVectorStoreService, IVectorStoreConfig, IDocumentChunk, IVectorSearchRequest, IVectorSearchResponse, ISearchResult } from './services/LangChainVectorStoreService';
+import { LangChainRAGService, ILangChainRAGConfig, IRAGIndexRequest, IRAGSearchRequest, IRAGResponse, IRAGStats } from './services/LangChainRAGService';
 import { IContextToken } from './types/IContextToken';
+import { IEmbeddingConfig, IChunkConfig } from './types';
 import type { IAiCfg } from './types/IAiCfg';
 
 export {    
     IAiCfg,
-    VectorStoreService,
     RWSVectorStore,
     RWSEmbed,
     RWSPrompt,
@@ -29,5 +31,22 @@ export {
     IAIToolSchema,
     IAIToolParameter,
     IToolCall,
-    ToolHandler
+    ToolHandler,
+    // New LangChain-based services
+    LangChainEmbeddingService,
+    LangChainVectorStoreService, 
+    LangChainRAGService,
+    // Types
+    IEmbeddingConfig,
+    IChunkConfig,
+    IVectorStoreConfig,
+    IDocumentChunk,
+    IVectorSearchRequest,
+    IVectorSearchResponse,
+    ISearchResult,
+    ILangChainRAGConfig,
+    IRAGIndexRequest,
+    IRAGSearchRequest,
+    IRAGResponse,
+    IRAGStats
 };