dspy 0.34.2 → 0.34.4

data/lib/dspy/memory/in_memory_store.rb

@@ -1,216 +0,0 @@
-# frozen_string_literal: true
-
-require 'sorbet-runtime'
-require_relative 'memory_store'
-
-module DSPy
-  module Memory
-    # In-memory implementation of MemoryStore for development and testing
-    class InMemoryStore < MemoryStore
-      extend T::Sig
-
-      sig { void }
-      def initialize
-        @memories = T.let({}, T::Hash[String, MemoryRecord])
-        @mutex = T.let(Mutex.new, Mutex)
-      end
-
-      sig { override.params(record: MemoryRecord).returns(T::Boolean) }
-      def store(record)
-        @mutex.synchronize do
-          @memories[record.id] = record
-          true
-        end
-      end
-
-      sig { override.params(id: String).returns(T.nilable(MemoryRecord)) }
-      def retrieve(id)
-        @mutex.synchronize do
-          record = @memories[id]
-          record&.record_access!
-          record
-        end
-      end
-
-      sig { override.params(record: MemoryRecord).returns(T::Boolean) }
-      def update(record)
-        @mutex.synchronize do
-          if @memories.key?(record.id)
-            @memories[record.id] = record
-            true
-          else
-            false
-          end
-        end
-      end
-
-      sig { override.params(id: String).returns(T::Boolean) }
-      def delete(id)
-        @mutex.synchronize do
-          @memories.delete(id) ? true : false
-        end
-      end
-
-      sig { override.params(user_id: T.nilable(String), limit: T.nilable(Integer), offset: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
-      def list(user_id: nil, limit: nil, offset: nil)
-        @mutex.synchronize do
-          records = @memories.values
-          
-          # Filter by user_id if provided
-          records = records.select { |r| r.user_id == user_id } if user_id
-          
-          # Sort by created_at (newest first)
-          records = records.sort_by(&:created_at).reverse
-          
-          # Apply offset and limit
-          records = records.drop(offset) if offset
-          records = records.take(limit) if limit
-          
-          records
-        end
-      end
-
-      sig { override.params(query: String, user_id: T.nilable(String), limit: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
-      def search(query, user_id: nil, limit: nil)
-        @mutex.synchronize do
-          regex = Regexp.new(Regexp.escape(query), Regexp::IGNORECASE)
-          
-          records = @memories.values.select do |record|
-            # Filter by user_id if provided
-            next false if user_id && record.user_id != user_id
-            
-            # Search in content and tags
-            record.content.match?(regex) || record.tags.any? { |tag| tag.match?(regex) }
-          end
-          
-          # Sort by relevance (exact matches first, then by recency)
-          records = records.sort_by do |record|
-            exact_match = record.content.downcase.include?(query.downcase) ? 0 : 1
-            [exact_match, -record.created_at.to_f]
-          end
-          
-          records = records.take(limit) if limit
-          records
-        end
-      end
-
-      sig { override.params(tags: T::Array[String], user_id: T.nilable(String), limit: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
-      def search_by_tags(tags, user_id: nil, limit: nil)
-        @mutex.synchronize do
-          records = @memories.values.select do |record|
-            # Filter by user_id if provided
-            next false if user_id && record.user_id != user_id
-            
-            # Check if record has any of the specified tags
-            tags.any? { |tag| record.has_tag?(tag) }
-          end
-          
-          # Sort by number of matching tags, then by recency
-          records = records.sort_by do |record|
-            matching_tags = tags.count { |tag| record.has_tag?(tag) }
-            [-matching_tags, -record.created_at.to_f]
-          end
-          
-          records = records.take(limit) if limit
-          records
-        end
-      end
-
-      sig { override.params(embedding: T::Array[Float], user_id: T.nilable(String), limit: T.nilable(Integer), threshold: T.nilable(Float)).returns(T::Array[MemoryRecord]) }
-      def vector_search(embedding, user_id: nil, limit: nil, threshold: nil)
-        @mutex.synchronize do
-          records_with_similarity = []
-          
-          @memories.values.each do |record|
-            # Filter by user_id if provided
-            next if user_id && record.user_id != user_id
-            
-            # Skip records without embeddings
-            next unless record.embedding
-            
-            # Calculate cosine similarity
-            similarity = cosine_similarity(embedding, record.embedding)
-            
-            # Apply threshold if provided
-            next if threshold && similarity < threshold
-            
-            records_with_similarity << [record, similarity]
-          end
-          
-          # Sort by similarity (highest first)
-          records_with_similarity.sort_by! { |_, similarity| -similarity }
-          
-          # Apply limit
-          records_with_similarity = records_with_similarity.take(limit) if limit
-          
-          # Return just the records
-          records_with_similarity.map(&:first)
-        end
-      end
-
-      sig { override.params(user_id: T.nilable(String)).returns(Integer) }
-      def count(user_id: nil)
-        @mutex.synchronize do
-          if user_id
-            @memories.values.count { |record| record.user_id == user_id }
-          else
-            @memories.size
-          end
-        end
-      end
-
-      sig { override.params(user_id: T.nilable(String)).returns(Integer) }
-      def clear(user_id: nil)
-        @mutex.synchronize do
-          if user_id
-            count = @memories.values.count { |record| record.user_id == user_id }
-            @memories.reject! { |_, record| record.user_id == user_id }
-            count
-          else
-            count = @memories.size
-            @memories.clear
-            count
-          end
-        end
-      end
-
-      sig { override.returns(T::Boolean) }
-      def supports_vector_search?
-        true
-      end
-
-      sig { override.returns(T::Hash[Symbol, T.untyped]) }
-      def stats
-        @mutex.synchronize do
-          total = @memories.size
-          with_embeddings = @memories.values.count(&:embedding)
-          users = @memories.values.map(&:user_id).compact.uniq.size
-          
-          {
-            total_memories: total,
-            memories_with_embeddings: with_embeddings,
-            unique_users: users,
-            supports_vector_search: supports_vector_search?,
-            avg_access_count: total > 0 ? @memories.values.sum(&:access_count) / total.to_f : 0
-          }
-        end
-      end
-
-      private
-
-      # Calculate cosine similarity between two vectors
-      sig { params(a: T::Array[Float], b: T::Array[Float]).returns(Float) }
-      def cosine_similarity(a, b)
-        return 0.0 if a.empty? || b.empty? || a.size != b.size
-        
-        dot_product = a.zip(b).sum { |x, y| x * y }
-        magnitude_a = Math.sqrt(a.sum { |x| x * x })
-        magnitude_b = Math.sqrt(b.sum { |x| x * x })
-        
-        return 0.0 if magnitude_a == 0.0 || magnitude_b == 0.0
-        
-        dot_product / (magnitude_a * magnitude_b)
-      end
-    end
-  end
-end

data/lib/dspy/memory/local_embedding_engine.rb

@@ -1,244 +0,0 @@
-# frozen_string_literal: true
-
-require 'sorbet-runtime'
-
-require 'informers'
-
-require_relative 'embedding_engine'
-
-module DSPy
-  module Memory
-    # Local embedding engine using ankane/informers for privacy-preserving embeddings
-    class LocalEmbeddingEngine < EmbeddingEngine
-      extend T::Sig
-
-      # Default models supported by informers
-      DEFAULT_MODEL = 'Xenova/all-MiniLM-L6-v2'
-      SUPPORTED_MODELS = [
-        'Xenova/all-MiniLM-L6-v2',
-        'Xenova/all-MiniLM-L12-v2',
-        'Xenova/multi-qa-MiniLM-L6-cos-v1',
-        'Xenova/paraphrase-MiniLM-L6-v2'
-      ].freeze
-
-      sig { returns(String) }
-      attr_reader :model_name
-
-      sig { params(model_name: String).void }
-      def initialize(model_name = DEFAULT_MODEL)
-        @model_name = model_name
-        @model = T.let(nil, T.nilable(T.untyped))
-        @embedding_dim = T.let(nil, T.nilable(Integer))
-        @ready = T.let(false, T::Boolean)
-        
-        load_model!
-      end
-
-      sig { override.params(text: String).returns(T::Array[Float]) }
-      def embed(text)
-        DSPy::Context.with_span(
-          operation: 'embedding.generate',
-          **DSPy::ObservationType::Embedding.langfuse_attributes,
-          'embedding.model' => @model_name,
-          'embedding.input' => text[0..200], # Truncate for logging
-          'embedding.input_length' => text.length
-        ) do |span|
-          ensure_ready!
-          
-          # Preprocess text
-          cleaned_text = preprocess_text(text)
-          
-          # Generate embedding
-          result = @model.call(cleaned_text)
-          
-          # Extract embedding array and normalize
-          embedding = result.first.to_a
-          normalized = normalize_vector(embedding)
-          
-          # Add embedding metadata to span
-          if span
-            span.set_attribute('embedding.dimension', normalized.length)
-            span.set_attribute('embedding.magnitude', Math.sqrt(normalized.sum { |x| x * x }))
-          end
-          
-          normalized
-        end
-      end
-
-      sig { override.params(texts: T::Array[String]).returns(T::Array[T::Array[Float]]) }
-      def embed_batch(texts)
-        ensure_ready!
-        
-        # Generate embeddings one by one (informers doesn't support true batch processing)
-        texts.map do |text|
-          embed(text)
-        end
-      end
-
-      sig { override.returns(Integer) }
-      def embedding_dimension
-        @embedding_dim || load_model_info!
-      end
-
-      sig { override.returns(String) }
-      def model_name
-        @model_name
-      end
-
-      sig { override.returns(T::Boolean) }
-      def ready?
-        @ready
-      end
-
-      sig { override.returns(T::Hash[Symbol, T.untyped]) }
-      def stats
-        {
-          model_name: @model_name,
-          embedding_dimension: embedding_dimension,
-          ready: ready?,
-          supported_models: SUPPORTED_MODELS,
-          backend: 'informers'
-        }
-      end
-
-      # Check if a model is supported
-      sig { params(model_name: String).returns(T::Boolean) }
-      def self.model_supported?(model_name)
-        SUPPORTED_MODELS.include?(model_name)
-      end
-
-      # List all supported models
-      sig { returns(T::Array[String]) }
-      def self.supported_models
-        SUPPORTED_MODELS
-      end
-
-      private
-
-      # Load the embedding model
-      sig { void }
-      def load_model!
-        begin
-          @model = Informers.pipeline('feature-extraction', @model_name)
-          @ready = true
-          load_model_info!
-        rescue => e
-          @ready = false
-          raise "Failed to load embedding model '#{@model_name}': #{e.message}"
-        end
-      end
-
-      # Load model information (dimension, etc.)
-      sig { returns(Integer) }
-      def load_model_info!
-        return @embedding_dim if @embedding_dim
-        
-        # Test with a simple string to get dimension
-        test_result = @model.call("test")
-        @embedding_dim = test_result.first.size
-      end
-
-      # Ensure the model is ready
-      sig { void }
-      def ensure_ready!
-        unless @ready
-          raise "Embedding engine not ready. Model '#{@model_name}' failed to load."
-        end
-      end
-
-      # Preprocess text for better embeddings
-      sig { params(text: String).returns(String) }
-      def preprocess_text(text)
-        # Basic text preprocessing
-        cleaned = text.strip
-        
-        # Remove excessive whitespace
-        cleaned = cleaned.gsub(/\s+/, ' ')
-        
-        # Truncate if too long (most models have token limits)
-        if cleaned.length > 8192  # Conservative limit
-          cleaned = cleaned[0..8191]
-        end
-        
-        cleaned
-      end
-    end
-
-    # Fallback embedding engine when informers is not available
-    class NoOpEmbeddingEngine < EmbeddingEngine
-      extend T::Sig
-
-      sig { override.params(text: String).returns(T::Array[Float]) }
-      def embed(text)
-        # Return a simple hash-based embedding for basic functionality
-        simple_hash_embedding(text)
-      end
-
-      sig { override.params(texts: T::Array[String]).returns(T::Array[T::Array[Float]]) }
-      def embed_batch(texts)
-        texts.map { |text| embed(text) }
-      end
-
-      sig { override.returns(Integer) }
-      def embedding_dimension
-        128  # Fixed dimension for hash-based embeddings
-      end
-
-      sig { override.returns(String) }
-      def model_name
-        'simple-hash'
-      end
-
-      sig { override.returns(T::Boolean) }
-      def ready?
-        true
-      end
-
-      private
-
-      # Generate a simple hash-based embedding that captures semantic similarity
-      sig { params(text: String).returns(T::Array[Float]) }
-      def simple_hash_embedding(text)
-        # Create a deterministic but semantically aware embedding
-        words = text.downcase.split(/\W+/).reject(&:empty?)
-        
-        # Initialize embedding vector
-        embedding = Array.new(128, 0.0)
-        
-        # Create base embedding from all words
-        words.each_with_index do |word, word_idx|
-          word_hash = word.sum(&:ord)
-          
-          # Distribute word influence across dimensions
-          (0..7).each do |i|
-            dim = (word_hash + i * 13) % 128
-            weight = Math.sin(word_hash + i) * 0.2
-            embedding[dim] += weight / Math.sqrt(words.length + 1)
-          end
-        end
-        
-        # Add semantic clusters for common words
-        semantic_clusters = {
-          ['programming', 'code', 'software', 'development'] => (0..15),
-          ['ruby', 'python', 'java', 'javascript'] => (16..31),
-          ['work', 'project', 'task', 'job'] => (32..47),
-          ['tutorial', 'guide', 'learning', 'education'] => (48..63),
-          ['memory', 'storage', 'data', 'information'] => (64..79),
-          ['personal', 'private', 'individual', 'own'] => (80..95),
-          ['important', 'critical', 'key', 'essential'] => (96..111),
-          ['test', 'testing', 'spec', 'example'] => (112..127)
-        }
-        
-        semantic_clusters.each do |cluster_words, range|
-          cluster_weight = words.count { |word| cluster_words.include?(word) }
-          if cluster_weight > 0
-            range.each { |dim| embedding[dim] += cluster_weight * 0.3 }
-          end
-        end
-        
-        # Normalize to unit vector
-        normalize_vector(embedding)
-      end
-    end
-  end
-end