dspy 0.5.0 → 0.6.0

data/lib/dspy/lm.rb

@@ -39,15 +39,12 @@ module DSPy
       input_text = messages.map { |m| m[:content] }.join(' ')
       input_size = input_text.length
       
-      # Check trace level to decide instrumentation strategy
-      trace_level = DSPy.config.instrumentation.trace_level
-      
-      # Extract token usage and prepare consolidated payload
+      # Use smart consolidation: emit LM events only when not in nested context
       response = nil
       token_usage = {}
       
-      if should_emit_lm_events?(trace_level)
-        # Detailed mode: emit all LM events as before
+      if should_emit_lm_events?
+        # Emit all LM events when not in nested context
         response = Instrumentation.instrument('dspy.lm.request', {
           gen_ai_operation_name: 'chat',
           gen_ai_system: provider,
@@ -92,19 +89,10 @@ module DSPy
 
     private
 
-    # Determines if LM-level events should be emitted based on trace level
-    def should_emit_lm_events?(trace_level)
-      case trace_level
-      when :minimal
-        false  # Never emit LM events in minimal mode
-      when :standard
-        # In standard mode, emit LM events only if we're not in a nested context
-        !is_nested_context?
-      when :detailed
-        true  # Always emit LM events in detailed mode
-      else
-        true
-      end
+    # Determines if LM-level events should be emitted using smart consolidation
+    def should_emit_lm_events?
+      # Emit LM events only if we're not in a nested context (smart consolidation)
+      !is_nested_context?
     end
 
     # Determines if we're in a nested context where higher-level events are being emitted
@@ -151,11 +139,16 @@ module DSPy
       # Try to parse the response as JSON
       content = response.content
 
-      # Extract JSON if it's in a code block
-      if content.include?('```json')
-        content = content.split('```json').last.split('```').first.strip
-      elsif content.include?('```')
-        content = content.split('```').last.split('```').first.strip
+      # Let adapters handle their own extraction logic if available
+      if adapter && adapter.respond_to?(:extract_json_from_response, true)
+        content = adapter.send(:extract_json_from_response, content)
+      else
+        # Fallback: Extract JSON if it's in a code block (legacy behavior)
+        if content.include?('```json')
+          content = content.split('```json').last.split('```').first.strip
+        elsif content.include?('```')
+          content = content.split('```').last.split('```').first.strip
+        end
       end
 
       begin
@@ -164,8 +157,17 @@ module DSPy
         # For Sorbet signatures, just return the parsed JSON
         # The Predict will handle validation
         json_payload
-      rescue JSON::ParserError
-        raise "Failed to parse LLM response as JSON: #{content}"
+      rescue JSON::ParserError => e
+        # Enhanced error message with debugging information
+        error_details = {
+          original_content: response.content,
+          extracted_content: content,
+          provider: provider,
+          model: model
+        }
+        
+        DSPy.logger.debug("JSON parsing failed: #{error_details}")
+        raise "Failed to parse LLM response as JSON: #{e.message}. Original content length: #{response.content&.length || 0} chars"
       end
     end
   end

data/lib/dspy/memory/embedding_engine.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module DSPy
+  module Memory
+    # Abstract base class for embedding engines
+    class EmbeddingEngine
+      extend T::Sig
+      extend T::Helpers
+      abstract!
+
+      # Generate embeddings for a single text
+      sig { abstract.params(text: String).returns(T::Array[Float]) }
+      def embed(text); end
+
+      # Generate embeddings for multiple texts (batch processing)
+      sig { abstract.params(texts: T::Array[String]).returns(T::Array[T::Array[Float]]) }
+      def embed_batch(texts); end
+
+      # Get the dimension of embeddings produced by this engine
+      sig { abstract.returns(Integer) }
+      def embedding_dimension; end
+
+      # Get the model name/identifier
+      sig { abstract.returns(String) }
+      def model_name; end
+
+      # Check if the engine is ready to use
+      sig { returns(T::Boolean) }
+      def ready?
+        true
+      end
+
+      # Get engine statistics
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      def stats
+        {
+          model_name: model_name,
+          embedding_dimension: embedding_dimension,
+          ready: ready?
+        }
+      end
+
+      # Normalize a vector to unit length
+      sig { params(vector: T::Array[Float]).returns(T::Array[Float]) }
+      def normalize_vector(vector)
+        magnitude = Math.sqrt(vector.sum { |x| x * x })
+        return vector if magnitude == 0.0
+        vector.map { |x| x / magnitude }
+      end
+
+      # Calculate cosine similarity between two vectors
+      sig { params(a: T::Array[Float], b: T::Array[Float]).returns(Float) }
+      def cosine_similarity(a, b)
+        return 0.0 if a.empty? || b.empty? || a.size != b.size
+        
+        dot_product = a.zip(b).sum { |x, y| x * y }
+        magnitude_a = Math.sqrt(a.sum { |x| x * x })
+        magnitude_b = Math.sqrt(b.sum { |x| x * x })
+        
+        return 0.0 if magnitude_a == 0.0 || magnitude_b == 0.0
+        
+        dot_product / (magnitude_a * magnitude_b)
+      end
+    end
+  end
+end

data/lib/dspy/memory/in_memory_store.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+require_relative 'memory_store'
+
+module DSPy
+  module Memory
+    # In-memory implementation of MemoryStore for development and testing
+    class InMemoryStore < MemoryStore
+      extend T::Sig
+
+      sig { void }
+      def initialize
+        @memories = T.let({}, T::Hash[String, MemoryRecord])
+        @mutex = T.let(Mutex.new, Mutex)
+      end
+
+      sig { override.params(record: MemoryRecord).returns(T::Boolean) }
+      def store(record)
+        @mutex.synchronize do
+          @memories[record.id] = record
+          true
+        end
+      end
+
+      sig { override.params(id: String).returns(T.nilable(MemoryRecord)) }
+      def retrieve(id)
+        @mutex.synchronize do
+          record = @memories[id]
+          record&.record_access!
+          record
+        end
+      end
+
+      sig { override.params(record: MemoryRecord).returns(T::Boolean) }
+      def update(record)
+        @mutex.synchronize do
+          if @memories.key?(record.id)
+            @memories[record.id] = record
+            true
+          else
+            false
+          end
+        end
+      end
+
+      sig { override.params(id: String).returns(T::Boolean) }
+      def delete(id)
+        @mutex.synchronize do
+          @memories.delete(id) ? true : false
+        end
+      end
+
+      sig { override.params(user_id: T.nilable(String), limit: T.nilable(Integer), offset: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
+      def list(user_id: nil, limit: nil, offset: nil)
+        @mutex.synchronize do
+          records = @memories.values
+          
+          # Filter by user_id if provided
+          records = records.select { |r| r.user_id == user_id } if user_id
+          
+          # Sort by created_at (newest first)
+          records = records.sort_by(&:created_at).reverse
+          
+          # Apply offset and limit
+          records = records.drop(offset) if offset
+          records = records.take(limit) if limit
+          
+          records
+        end
+      end
+
+      sig { override.params(query: String, user_id: T.nilable(String), limit: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
+      def search(query, user_id: nil, limit: nil)
+        @mutex.synchronize do
+          regex = Regexp.new(Regexp.escape(query), Regexp::IGNORECASE)
+          
+          records = @memories.values.select do |record|
+            # Filter by user_id if provided
+            next false if user_id && record.user_id != user_id
+            
+            # Search in content and tags
+            record.content.match?(regex) || record.tags.any? { |tag| tag.match?(regex) }
+          end
+          
+          # Sort by relevance (exact matches first, then by recency)
+          records = records.sort_by do |record|
+            exact_match = record.content.downcase.include?(query.downcase) ? 0 : 1
+            [exact_match, -record.created_at.to_f]
+          end
+          
+          records = records.take(limit) if limit
+          records
+        end
+      end
+
+      sig { override.params(tags: T::Array[String], user_id: T.nilable(String), limit: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
+      def search_by_tags(tags, user_id: nil, limit: nil)
+        @mutex.synchronize do
+          records = @memories.values.select do |record|
+            # Filter by user_id if provided
+            next false if user_id && record.user_id != user_id
+            
+            # Check if record has any of the specified tags
+            tags.any? { |tag| record.has_tag?(tag) }
+          end
+          
+          # Sort by number of matching tags, then by recency
+          records = records.sort_by do |record|
+            matching_tags = tags.count { |tag| record.has_tag?(tag) }
+            [-matching_tags, -record.created_at.to_f]
+          end
+          
+          records = records.take(limit) if limit
+          records
+        end
+      end
+
+      sig { override.params(embedding: T::Array[Float], user_id: T.nilable(String), limit: T.nilable(Integer), threshold: T.nilable(Float)).returns(T::Array[MemoryRecord]) }
+      def vector_search(embedding, user_id: nil, limit: nil, threshold: nil)
+        @mutex.synchronize do
+          records_with_similarity = []
+          
+          @memories.values.each do |record|
+            # Filter by user_id if provided
+            next if user_id && record.user_id != user_id
+            
+            # Skip records without embeddings
+            next unless record.embedding
+            
+            # Calculate cosine similarity
+            similarity = cosine_similarity(embedding, record.embedding)
+            
+            # Apply threshold if provided
+            next if threshold && similarity < threshold
+            
+            records_with_similarity << [record, similarity]
+          end
+          
+          # Sort by similarity (highest first)
+          records_with_similarity.sort_by! { |_, similarity| -similarity }
+          
+          # Apply limit
+          records_with_similarity = records_with_similarity.take(limit) if limit
+          
+          # Return just the records
+          records_with_similarity.map(&:first)
+        end
+      end
+
+      sig { override.params(user_id: T.nilable(String)).returns(Integer) }
+      def count(user_id: nil)
+        @mutex.synchronize do
+          if user_id
+            @memories.values.count { |record| record.user_id == user_id }
+          else
+            @memories.size
+          end
+        end
+      end
+
+      sig { override.params(user_id: T.nilable(String)).returns(Integer) }
+      def clear(user_id: nil)
+        @mutex.synchronize do
+          if user_id
+            count = @memories.values.count { |record| record.user_id == user_id }
+            @memories.reject! { |_, record| record.user_id == user_id }
+            count
+          else
+            count = @memories.size
+            @memories.clear
+            count
+          end
+        end
+      end
+
+      sig { override.returns(T::Boolean) }
+      def supports_vector_search?
+        true
+      end
+
+      sig { override.returns(T::Hash[Symbol, T.untyped]) }
+      def stats
+        @mutex.synchronize do
+          total = @memories.size
+          with_embeddings = @memories.values.count(&:embedding)
+          users = @memories.values.map(&:user_id).compact.uniq.size
+          
+          {
+            total_memories: total,
+            memories_with_embeddings: with_embeddings,
+            unique_users: users,
+            supports_vector_search: supports_vector_search?,
+            avg_access_count: total > 0 ? @memories.values.sum(&:access_count) / total.to_f : 0
+          }
+        end
+      end
+
+      private
+
+      # Calculate cosine similarity between two vectors
+      sig { params(a: T::Array[Float], b: T::Array[Float]).returns(Float) }
+      def cosine_similarity(a, b)
+        return 0.0 if a.empty? || b.empty? || a.size != b.size
+        
+        dot_product = a.zip(b).sum { |x, y| x * y }
+        magnitude_a = Math.sqrt(a.sum { |x| x * x })
+        magnitude_b = Math.sqrt(b.sum { |x| x * x })
+        
+        return 0.0 if magnitude_a == 0.0 || magnitude_b == 0.0
+        
+        dot_product / (magnitude_a * magnitude_b)
+      end
+    end
+  end
+end

data/lib/dspy/memory/local_embedding_engine.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+require 'informers'
+
+require_relative 'embedding_engine'
+
+module DSPy
+  module Memory
+    # Local embedding engine using ankane/informers for privacy-preserving embeddings
+    class LocalEmbeddingEngine < EmbeddingEngine
+      extend T::Sig
+
+      # Default models supported by informers
+      DEFAULT_MODEL = 'Xenova/all-MiniLM-L6-v2'
+      SUPPORTED_MODELS = [
+        'Xenova/all-MiniLM-L6-v2',
+        'Xenova/all-MiniLM-L12-v2',
+        'Xenova/multi-qa-MiniLM-L6-cos-v1',
+        'Xenova/paraphrase-MiniLM-L6-v2'
+      ].freeze
+
+      sig { returns(String) }
+      attr_reader :model_name
+
+      sig { params(model_name: String).void }
+      def initialize(model_name = DEFAULT_MODEL)
+        @model_name = model_name
+        @model = T.let(nil, T.nilable(T.untyped))
+        @embedding_dim = T.let(nil, T.nilable(Integer))
+        @ready = T.let(false, T::Boolean)
+        
+        load_model!
+      end
+
+      sig { override.params(text: String).returns(T::Array[Float]) }
+      def embed(text)
+        ensure_ready!
+        
+        # Preprocess text
+        cleaned_text = preprocess_text(text)
+        
+        # Generate embedding
+        result = @model.call(cleaned_text)
+        
+        # Extract embedding array and normalize
+        embedding = result.first.to_a
+        normalize_vector(embedding)
+      end
+
+      sig { override.params(texts: T::Array[String]).returns(T::Array[T::Array[Float]]) }
+      def embed_batch(texts)
+        ensure_ready!
+        
+        # Preprocess all texts
+        cleaned_texts = texts.map { |text| preprocess_text(text) }
+        
+        # Generate embeddings in batch
+        results = @model.call(cleaned_texts)
+        
+        # Extract and normalize embeddings
+        results.map do |result|
+          # Handle both single embeddings and batch results
+          embedding = case result
+                     when Array
+                       result.flatten  # Flatten in case of nested arrays
+                     else
+                       result.to_a.flatten
+                     end
+          normalize_vector(embedding)
+        end
+      end
+
+      sig { override.returns(Integer) }
+      def embedding_dimension
+        @embedding_dim || load_model_info!
+      end
+
+      sig { override.returns(String) }
+      def model_name
+        @model_name
+      end
+
+      sig { override.returns(T::Boolean) }
+      def ready?
+        @ready
+      end
+
+      sig { override.returns(T::Hash[Symbol, T.untyped]) }
+      def stats
+        {
+          model_name: @model_name,
+          embedding_dimension: embedding_dimension,
+          ready: ready?,
+          supported_models: SUPPORTED_MODELS,
+          backend: 'informers'
+        }
+      end
+
+      # Check if a model is supported
+      sig { params(model_name: String).returns(T::Boolean) }
+      def self.model_supported?(model_name)
+        SUPPORTED_MODELS.include?(model_name)
+      end
+
+      # List all supported models
+      sig { returns(T::Array[String]) }
+      def self.supported_models
+        SUPPORTED_MODELS
+      end
+
+      private
+
+      # Load the embedding model
+      sig { void }
+      def load_model!
+        begin
+          @model = Informers.pipeline('feature-extraction', @model_name)
+          @ready = true
+          load_model_info!
+        rescue => e
+          @ready = false
+          raise "Failed to load embedding model '#{@model_name}': #{e.message}"
+        end
+      end
+
+      # Load model information (dimension, etc.)
+      sig { returns(Integer) }
+      def load_model_info!
+        return @embedding_dim if @embedding_dim
+        
+        # Test with a simple string to get dimension
+        test_result = @model.call("test")
+        @embedding_dim = test_result.first.size
+      end
+
+      # Ensure the model is ready
+      sig { void }
+      def ensure_ready!
+        unless @ready
+          raise "Embedding engine not ready. Model '#{@model_name}' failed to load."
+        end
+      end
+
+      # Preprocess text for better embeddings
+      sig { params(text: String).returns(String) }
+      def preprocess_text(text)
+        # Basic text preprocessing
+        cleaned = text.strip
+        
+        # Remove excessive whitespace
+        cleaned = cleaned.gsub(/\s+/, ' ')
+        
+        # Truncate if too long (most models have token limits)
+        if cleaned.length > 8192  # Conservative limit
+          cleaned = cleaned[0..8191]
+        end
+        
+        cleaned
+      end
+    end
+
+    # Fallback embedding engine when informers is not available
+    class NoOpEmbeddingEngine < EmbeddingEngine
+      extend T::Sig
+
+      sig { override.params(text: String).returns(T::Array[Float]) }
+      def embed(text)
+        # Return a simple hash-based embedding for basic functionality
+        simple_hash_embedding(text)
+      end
+
+      sig { override.params(texts: T::Array[String]).returns(T::Array[T::Array[Float]]) }
+      def embed_batch(texts)
+        texts.map { |text| embed(text) }
+      end
+
+      sig { override.returns(Integer) }
+      def embedding_dimension
+        128  # Fixed dimension for hash-based embeddings
+      end
+
+      sig { override.returns(String) }
+      def model_name
+        'simple-hash'
+      end
+
+      sig { override.returns(T::Boolean) }
+      def ready?
+        true
+      end
+
+      private
+
+      # Generate a simple hash-based embedding that captures semantic similarity
+      sig { params(text: String).returns(T::Array[Float]) }
+      def simple_hash_embedding(text)
+        # Create a deterministic but semantically aware embedding
+        words = text.downcase.split(/\W+/).reject(&:empty?)
+        
+        # Initialize embedding vector
+        embedding = Array.new(128, 0.0)
+        
+        # Create base embedding from all words
+        words.each_with_index do |word, word_idx|
+          word_hash = word.sum(&:ord)
+          
+          # Distribute word influence across dimensions
+          (0..7).each do |i|
+            dim = (word_hash + i * 13) % 128
+            weight = Math.sin(word_hash + i) * 0.2
+            embedding[dim] += weight / Math.sqrt(words.length + 1)
+          end
+        end
+        
+        # Add semantic clusters for common words
+        semantic_clusters = {
+          ['programming', 'code', 'software', 'development'] => (0..15),
+          ['ruby', 'python', 'java', 'javascript'] => (16..31),
+          ['work', 'project', 'task', 'job'] => (32..47),
+          ['tutorial', 'guide', 'learning', 'education'] => (48..63),
+          ['memory', 'storage', 'data', 'information'] => (64..79),
+          ['personal', 'private', 'individual', 'own'] => (80..95),
+          ['important', 'critical', 'key', 'essential'] => (96..111),
+          ['test', 'testing', 'spec', 'example'] => (112..127)
+        }
+        
+        semantic_clusters.each do |cluster_words, range|
+          cluster_weight = words.count { |word| cluster_words.include?(word) }
+          if cluster_weight > 0
+            range.each { |dim| embedding[dim] += cluster_weight * 0.3 }
+          end
+        end
+        
+        # Normalize to unit vector
+        normalize_vector(embedding)
+      end
+    end
+  end
+end