dspy 0.34.1 → 0.34.3

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dspy
 version: !ruby/object:Gem::Version
-  version: 0.34.1
+  version: 0.34.3
 platform: ruby
 authors:
 - Vicente Reig Rincón de Arellano
@@ -135,20 +135,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.0.0
-- !ruby/object:Gem::Dependency
-  name: informers
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
 description: The Ruby framework for programming with large language models. DSPy.rb
   brings structured LLM programming to Ruby developers. Instead of wrestling with
   prompt strings and parsing responses, you define typed signatures using idiomatic
@@ -170,8 +156,6 @@ files:
 - lib/dspy/evals.rb
 - lib/dspy/evals/version.rb
 - lib/dspy/events.rb
-- lib/dspy/events/subscriber_mixin.rb
-- lib/dspy/events/subscribers.rb
 - lib/dspy/events/types.rb
 - lib/dspy/example.rb
 - lib/dspy/ext/struct_descriptions.rb
@@ -189,14 +173,6 @@ files:
 - lib/dspy/lm/response.rb
 - lib/dspy/lm/usage.rb
 - lib/dspy/lm/vision_models.rb
-- lib/dspy/memory.rb
-- lib/dspy/memory/embedding_engine.rb
-- lib/dspy/memory/in_memory_store.rb
-- lib/dspy/memory/local_embedding_engine.rb
-- lib/dspy/memory/memory_compactor.rb
-- lib/dspy/memory/memory_manager.rb
-- lib/dspy/memory/memory_record.rb
-- lib/dspy/memory/memory_store.rb
 - lib/dspy/mixins/instruction_updatable.rb
 - lib/dspy/mixins/struct_builder.rb
 - lib/dspy/mixins/type_coercion.rb
@@ -238,7 +214,6 @@ files:
 - lib/dspy/tools.rb
 - lib/dspy/tools/base.rb
 - lib/dspy/tools/github_cli_toolset.rb
-- lib/dspy/tools/memory_toolset.rb
 - lib/dspy/tools/schema.rb
 - lib/dspy/tools/text_processing_toolset.rb
 - lib/dspy/tools/toolset.rb

data/lib/dspy/events/subscriber_mixin.rb

@@ -1,79 +0,0 @@
-# frozen_string_literal: true
-
-require 'sorbet-runtime'
-
-module DSPy
-  module Events
-    # Mixin for adding class-level event subscriptions
-    # Provides a clean way to subscribe to events at the class level
-    # instead of requiring instance-based subscriptions
-    #
-    # Usage:
-    #   class MyTracker
-    #     include DSPy::Events::SubscriberMixin
-    #     
-    #     add_subscription('llm.*') do |name, attrs|
-    #       # Handle LLM events globally for this class
-    #     end
-    #   end
-    module SubscriberMixin
-      extend T::Sig
-
-      def self.included(base)
-        base.extend(ClassMethods)
-        base.class_eval do
-          @event_subscriptions = []
-          @subscription_mutex = Mutex.new
-          
-          # Initialize subscriptions when the class is first loaded
-          @subscriptions_initialized = false
-        end
-      end
-
-      module ClassMethods
-        extend T::Sig
-
-        # Add a class-level event subscription
-        sig { params(pattern: String, block: T.proc.params(arg0: String, arg1: T::Hash[T.any(String, Symbol), T.untyped]).void).returns(String) }
-        def add_subscription(pattern, &block)
-          subscription_mutex.synchronize do
-            subscription_id = DSPy.events.subscribe(pattern, &block)
-            event_subscriptions << subscription_id
-            subscription_id
-          end
-        end
-
-        # Remove all subscriptions for this class
-        sig { void }
-        def unsubscribe_all
-          subscription_mutex.synchronize do
-            event_subscriptions.each { |id| DSPy.events.unsubscribe(id) }
-            event_subscriptions.clear
-          end
-        end
-
-        # Get list of active subscription IDs
-        sig { returns(T::Array[String]) }
-        def subscriptions
-          subscription_mutex.synchronize do
-            event_subscriptions.dup
-          end
-        end
-
-        private
-
-        # Thread-safe access to subscriptions array
-        sig { returns(T::Array[String]) }
-        def event_subscriptions
-          @event_subscriptions ||= []
-        end
-
-        # Thread-safe access to mutex
-        sig { returns(Mutex) }
-        def subscription_mutex
-          @subscription_mutex ||= Mutex.new
-        end
-      end
-    end
-  end
-end

data/lib/dspy/events/subscribers.rb

@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-module DSPy
-  module Events
-    # Base subscriber class for event-driven patterns
-    # This provides the foundation for creating custom event subscribers
-    # 
-    # Example usage:
-    #   class MySubscriber < DSPy::Events::BaseSubscriber
-    #     def subscribe
-    #       add_subscription('llm.*') do |event_name, attributes|
-    #         # Handle LLM events
-    #       end
-    #     end
-    #   end
-    #
-    #   subscriber = MySubscriber.new
-    #   # subscriber will start receiving events
-    #   subscriber.unsubscribe # Clean up when done
-    class BaseSubscriber
-      def initialize
-        @subscriptions = []
-      end
-      
-      def subscribe
-        raise NotImplementedError, "Subclasses must implement #subscribe"
-      end
-      
-      def unsubscribe
-        @subscriptions.each { |id| DSPy.events.unsubscribe(id) }
-        @subscriptions.clear
-      end
-      
-      protected
-      
-      def add_subscription(pattern, &block)
-        subscription_id = DSPy.events.subscribe(pattern, &block)
-        @subscriptions << subscription_id
-        subscription_id
-      end
-    end
-  end
-end

data/lib/dspy/memory/embedding_engine.rb

@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-
-require 'sorbet-runtime'
-
-module DSPy
-  module Memory
-    # Abstract base class for embedding engines
-    class EmbeddingEngine
-      extend T::Sig
-      extend T::Helpers
-      abstract!
-
-      # Generate embeddings for a single text
-      sig { abstract.params(text: String).returns(T::Array[Float]) }
-      def embed(text); end
-
-      # Generate embeddings for multiple texts (batch processing)
-      sig { abstract.params(texts: T::Array[String]).returns(T::Array[T::Array[Float]]) }
-      def embed_batch(texts); end
-
-      # Get the dimension of embeddings produced by this engine
-      sig { abstract.returns(Integer) }
-      def embedding_dimension; end
-
-      # Get the model name/identifier
-      sig { abstract.returns(String) }
-      def model_name; end
-
-      # Check if the engine is ready to use
-      sig { returns(T::Boolean) }
-      def ready?
-        true
-      end
-
-      # Get engine statistics
-      sig { returns(T::Hash[Symbol, T.untyped]) }
-      def stats
-        {
-          model_name: model_name,
-          embedding_dimension: embedding_dimension,
-          ready: ready?
-        }
-      end
-
-      # Normalize a vector to unit length
-      sig { params(vector: T::Array[Float]).returns(T::Array[Float]) }
-      def normalize_vector(vector)
-        magnitude = Math.sqrt(vector.sum { |x| x * x })
-        return vector if magnitude == 0.0
-        vector.map { |x| x / magnitude }
-      end
-
-      # Calculate cosine similarity between two vectors
-      sig { params(a: T::Array[Float], b: T::Array[Float]).returns(Float) }
-      def cosine_similarity(a, b)
-        return 0.0 if a.empty? || b.empty? || a.size != b.size
-        
-        dot_product = a.zip(b).sum { |x, y| x * y }
-        magnitude_a = Math.sqrt(a.sum { |x| x * x })
-        magnitude_b = Math.sqrt(b.sum { |x| x * x })
-        
-        return 0.0 if magnitude_a == 0.0 || magnitude_b == 0.0
-        
-        dot_product / (magnitude_a * magnitude_b)
-      end
-    end
-  end
-end

data/lib/dspy/memory/in_memory_store.rb

@@ -1,216 +0,0 @@
-# frozen_string_literal: true
-
-require 'sorbet-runtime'
-require_relative 'memory_store'
-
-module DSPy
-  module Memory
-    # In-memory implementation of MemoryStore for development and testing
-    class InMemoryStore < MemoryStore
-      extend T::Sig
-
-      sig { void }
-      def initialize
-        @memories = T.let({}, T::Hash[String, MemoryRecord])
-        @mutex = T.let(Mutex.new, Mutex)
-      end
-
-      sig { override.params(record: MemoryRecord).returns(T::Boolean) }
-      def store(record)
-        @mutex.synchronize do
-          @memories[record.id] = record
-          true
-        end
-      end
-
-      sig { override.params(id: String).returns(T.nilable(MemoryRecord)) }
-      def retrieve(id)
-        @mutex.synchronize do
-          record = @memories[id]
-          record&.record_access!
-          record
-        end
-      end
-
-      sig { override.params(record: MemoryRecord).returns(T::Boolean) }
-      def update(record)
-        @mutex.synchronize do
-          if @memories.key?(record.id)
-            @memories[record.id] = record
-            true
-          else
-            false
-          end
-        end
-      end
-
-      sig { override.params(id: String).returns(T::Boolean) }
-      def delete(id)
-        @mutex.synchronize do
-          @memories.delete(id) ? true : false
-        end
-      end
-
-      sig { override.params(user_id: T.nilable(String), limit: T.nilable(Integer), offset: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
-      def list(user_id: nil, limit: nil, offset: nil)
-        @mutex.synchronize do
-          records = @memories.values
-          
-          # Filter by user_id if provided
-          records = records.select { |r| r.user_id == user_id } if user_id
-          
-          # Sort by created_at (newest first)
-          records = records.sort_by(&:created_at).reverse
-          
-          # Apply offset and limit
-          records = records.drop(offset) if offset
-          records = records.take(limit) if limit
-          
-          records
-        end
-      end
-
-      sig { override.params(query: String, user_id: T.nilable(String), limit: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
-      def search(query, user_id: nil, limit: nil)
-        @mutex.synchronize do
-          regex = Regexp.new(Regexp.escape(query), Regexp::IGNORECASE)
-          
-          records = @memories.values.select do |record|
-            # Filter by user_id if provided
-            next false if user_id && record.user_id != user_id
-            
-            # Search in content and tags
-            record.content.match?(regex) || record.tags.any? { |tag| tag.match?(regex) }
-          end
-          
-          # Sort by relevance (exact matches first, then by recency)
-          records = records.sort_by do |record|
-            exact_match = record.content.downcase.include?(query.downcase) ? 0 : 1
-            [exact_match, -record.created_at.to_f]
-          end
-          
-          records = records.take(limit) if limit
-          records
-        end
-      end
-
-      sig { override.params(tags: T::Array[String], user_id: T.nilable(String), limit: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
-      def search_by_tags(tags, user_id: nil, limit: nil)
-        @mutex.synchronize do
-          records = @memories.values.select do |record|
-            # Filter by user_id if provided
-            next false if user_id && record.user_id != user_id
-            
-            # Check if record has any of the specified tags
-            tags.any? { |tag| record.has_tag?(tag) }
-          end
-          
-          # Sort by number of matching tags, then by recency
-          records = records.sort_by do |record|
-            matching_tags = tags.count { |tag| record.has_tag?(tag) }
-            [-matching_tags, -record.created_at.to_f]
-          end
-          
-          records = records.take(limit) if limit
-          records
-        end
-      end
-
-      sig { override.params(embedding: T::Array[Float], user_id: T.nilable(String), limit: T.nilable(Integer), threshold: T.nilable(Float)).returns(T::Array[MemoryRecord]) }
-      def vector_search(embedding, user_id: nil, limit: nil, threshold: nil)
-        @mutex.synchronize do
-          records_with_similarity = []
-          
-          @memories.values.each do |record|
-            # Filter by user_id if provided
-            next if user_id && record.user_id != user_id
-            
-            # Skip records without embeddings
-            next unless record.embedding
-            
-            # Calculate cosine similarity
-            similarity = cosine_similarity(embedding, record.embedding)
-            
-            # Apply threshold if provided
-            next if threshold && similarity < threshold
-            
-            records_with_similarity << [record, similarity]
-          end
-          
-          # Sort by similarity (highest first)
-          records_with_similarity.sort_by! { |_, similarity| -similarity }
-          
-          # Apply limit
-          records_with_similarity = records_with_similarity.take(limit) if limit
-          
-          # Return just the records
-          records_with_similarity.map(&:first)
-        end
-      end
-
-      sig { override.params(user_id: T.nilable(String)).returns(Integer) }
-      def count(user_id: nil)
-        @mutex.synchronize do
-          if user_id
-            @memories.values.count { |record| record.user_id == user_id }
-          else
-            @memories.size
-          end
-        end
-      end
-
-      sig { override.params(user_id: T.nilable(String)).returns(Integer) }
-      def clear(user_id: nil)
-        @mutex.synchronize do
-          if user_id
-            count = @memories.values.count { |record| record.user_id == user_id }
-            @memories.reject! { |_, record| record.user_id == user_id }
-            count
-          else
-            count = @memories.size
-            @memories.clear
-            count
-          end
-        end
-      end
-
-      sig { override.returns(T::Boolean) }
-      def supports_vector_search?
-        true
-      end
-
-      sig { override.returns(T::Hash[Symbol, T.untyped]) }
-      def stats
-        @mutex.synchronize do
-          total = @memories.size
-          with_embeddings = @memories.values.count(&:embedding)
-          users = @memories.values.map(&:user_id).compact.uniq.size
-          
-          {
-            total_memories: total,
-            memories_with_embeddings: with_embeddings,
-            unique_users: users,
-            supports_vector_search: supports_vector_search?,
-            avg_access_count: total > 0 ? @memories.values.sum(&:access_count) / total.to_f : 0
-          }
-        end
-      end
-
-      private
-
-      # Calculate cosine similarity between two vectors
-      sig { params(a: T::Array[Float], b: T::Array[Float]).returns(Float) }
-      def cosine_similarity(a, b)
-        return 0.0 if a.empty? || b.empty? || a.size != b.size
-        
-        dot_product = a.zip(b).sum { |x, y| x * y }
-        magnitude_a = Math.sqrt(a.sum { |x| x * x })
-        magnitude_b = Math.sqrt(b.sum { |x| x * x })
-        
-        return 0.0 if magnitude_a == 0.0 || magnitude_b == 0.0
-        
-        dot_product / (magnitude_a * magnitude_b)
-      end
-    end
-  end
-end

data/lib/dspy/memory/local_embedding_engine.rb

@@ -1,244 +0,0 @@
-# frozen_string_literal: true
-
-require 'sorbet-runtime'
-
-require 'informers'
-
-require_relative 'embedding_engine'
-
-module DSPy
-  module Memory
-    # Local embedding engine using ankane/informers for privacy-preserving embeddings
-    class LocalEmbeddingEngine < EmbeddingEngine
-      extend T::Sig
-
-      # Default models supported by informers
-      DEFAULT_MODEL = 'Xenova/all-MiniLM-L6-v2'
-      SUPPORTED_MODELS = [
-        'Xenova/all-MiniLM-L6-v2',
-        'Xenova/all-MiniLM-L12-v2',
-        'Xenova/multi-qa-MiniLM-L6-cos-v1',
-        'Xenova/paraphrase-MiniLM-L6-v2'
-      ].freeze
-
-      sig { returns(String) }
-      attr_reader :model_name
-
-      sig { params(model_name: String).void }
-      def initialize(model_name = DEFAULT_MODEL)
-        @model_name = model_name
-        @model = T.let(nil, T.nilable(T.untyped))
-        @embedding_dim = T.let(nil, T.nilable(Integer))
-        @ready = T.let(false, T::Boolean)
-        
-        load_model!
-      end
-
-      sig { override.params(text: String).returns(T::Array[Float]) }
-      def embed(text)
-        DSPy::Context.with_span(
-          operation: 'embedding.generate',
-          **DSPy::ObservationType::Embedding.langfuse_attributes,
-          'embedding.model' => @model_name,
-          'embedding.input' => text[0..200], # Truncate for logging
-          'embedding.input_length' => text.length
-        ) do |span|
-          ensure_ready!
-          
-          # Preprocess text
-          cleaned_text = preprocess_text(text)
-          
-          # Generate embedding
-          result = @model.call(cleaned_text)
-          
-          # Extract embedding array and normalize
-          embedding = result.first.to_a
-          normalized = normalize_vector(embedding)
-          
-          # Add embedding metadata to span
-          if span
-            span.set_attribute('embedding.dimension', normalized.length)
-            span.set_attribute('embedding.magnitude', Math.sqrt(normalized.sum { |x| x * x }))
-          end
-          
-          normalized
-        end
-      end
-
-      sig { override.params(texts: T::Array[String]).returns(T::Array[T::Array[Float]]) }
-      def embed_batch(texts)
-        ensure_ready!
-        
-        # Generate embeddings one by one (informers doesn't support true batch processing)
-        texts.map do |text|
-          embed(text)
-        end
-      end
-
-      sig { override.returns(Integer) }
-      def embedding_dimension
-        @embedding_dim || load_model_info!
-      end
-
-      sig { override.returns(String) }
-      def model_name
-        @model_name
-      end
-
-      sig { override.returns(T::Boolean) }
-      def ready?
-        @ready
-      end
-
-      sig { override.returns(T::Hash[Symbol, T.untyped]) }
-      def stats
-        {
-          model_name: @model_name,
-          embedding_dimension: embedding_dimension,
-          ready: ready?,
-          supported_models: SUPPORTED_MODELS,
-          backend: 'informers'
-        }
-      end
-
-      # Check if a model is supported
-      sig { params(model_name: String).returns(T::Boolean) }
-      def self.model_supported?(model_name)
-        SUPPORTED_MODELS.include?(model_name)
-      end
-
-      # List all supported models
-      sig { returns(T::Array[String]) }
-      def self.supported_models
-        SUPPORTED_MODELS
-      end
-
-      private
-
-      # Load the embedding model
-      sig { void }
-      def load_model!
-        begin
-          @model = Informers.pipeline('feature-extraction', @model_name)
-          @ready = true
-          load_model_info!
-        rescue => e
-          @ready = false
-          raise "Failed to load embedding model '#{@model_name}': #{e.message}"
-        end
-      end
-
-      # Load model information (dimension, etc.)
-      sig { returns(Integer) }
-      def load_model_info!
-        return @embedding_dim if @embedding_dim
-        
-        # Test with a simple string to get dimension
-        test_result = @model.call("test")
-        @embedding_dim = test_result.first.size
-      end
-
-      # Ensure the model is ready
-      sig { void }
-      def ensure_ready!
-        unless @ready
-          raise "Embedding engine not ready. Model '#{@model_name}' failed to load."
-        end
-      end
-
-      # Preprocess text for better embeddings
-      sig { params(text: String).returns(String) }
-      def preprocess_text(text)
-        # Basic text preprocessing
-        cleaned = text.strip
-        
-        # Remove excessive whitespace
-        cleaned = cleaned.gsub(/\s+/, ' ')
-        
-        # Truncate if too long (most models have token limits)
-        if cleaned.length > 8192  # Conservative limit
-          cleaned = cleaned[0..8191]
-        end
-        
-        cleaned
-      end
-    end
-
-    # Fallback embedding engine when informers is not available
-    class NoOpEmbeddingEngine < EmbeddingEngine
-      extend T::Sig
-
-      sig { override.params(text: String).returns(T::Array[Float]) }
-      def embed(text)
-        # Return a simple hash-based embedding for basic functionality
-        simple_hash_embedding(text)
-      end
-
-      sig { override.params(texts: T::Array[String]).returns(T::Array[T::Array[Float]]) }
-      def embed_batch(texts)
-        texts.map { |text| embed(text) }
-      end
-
-      sig { override.returns(Integer) }
-      def embedding_dimension
-        128  # Fixed dimension for hash-based embeddings
-      end
-
-      sig { override.returns(String) }
-      def model_name
-        'simple-hash'
-      end
-
-      sig { override.returns(T::Boolean) }
-      def ready?
-        true
-      end
-
-      private
-
-      # Generate a simple hash-based embedding that captures semantic similarity
-      sig { params(text: String).returns(T::Array[Float]) }
-      def simple_hash_embedding(text)
-        # Create a deterministic but semantically aware embedding
-        words = text.downcase.split(/\W+/).reject(&:empty?)
-        
-        # Initialize embedding vector
-        embedding = Array.new(128, 0.0)
-        
-        # Create base embedding from all words
-        words.each_with_index do |word, word_idx|
-          word_hash = word.sum(&:ord)
-          
-          # Distribute word influence across dimensions
-          (0..7).each do |i|
-            dim = (word_hash + i * 13) % 128
-            weight = Math.sin(word_hash + i) * 0.2
-            embedding[dim] += weight / Math.sqrt(words.length + 1)
-          end
-        end
-        
-        # Add semantic clusters for common words
-        semantic_clusters = {
-          ['programming', 'code', 'software', 'development'] => (0..15),
-          ['ruby', 'python', 'java', 'javascript'] => (16..31),
-          ['work', 'project', 'task', 'job'] => (32..47),
-          ['tutorial', 'guide', 'learning', 'education'] => (48..63),
-          ['memory', 'storage', 'data', 'information'] => (64..79),
-          ['personal', 'private', 'individual', 'own'] => (80..95),
-          ['important', 'critical', 'key', 'essential'] => (96..111),
-          ['test', 'testing', 'spec', 'example'] => (112..127)
-        }
-        
-        semantic_clusters.each do |cluster_words, range|
-          cluster_weight = words.count { |word| cluster_words.include?(word) }
-          if cluster_weight > 0
-            range.each { |dim| embedding[dim] += cluster_weight * 0.3 }
-          end
-        end
-        
-        # Normalize to unit vector
-        normalize_vector(embedding)
-      end
-    end
-  end
-end