dspy 0.5.1 → 0.6.0

data/lib/dspy/memory/memory_store.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module DSPy
+  module Memory
+    # Abstract base class for memory storage backends
+    class MemoryStore
+      extend T::Sig
+      extend T::Helpers
+      abstract!
+
+      # Store a memory record
+      sig { abstract.params(record: MemoryRecord).returns(T::Boolean) }
+      def store(record); end
+
+      # Retrieve a memory record by ID
+      sig { abstract.params(id: String).returns(T.nilable(MemoryRecord)) }
+      def retrieve(id); end
+
+      # Update an existing memory record
+      sig { abstract.params(record: MemoryRecord).returns(T::Boolean) }
+      def update(record); end
+
+      # Delete a memory record by ID
+      sig { abstract.params(id: String).returns(T::Boolean) }
+      def delete(id); end
+
+      # List all memory records for a user
+      sig { abstract.params(user_id: T.nilable(String), limit: T.nilable(Integer), offset: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
+      def list(user_id: nil, limit: nil, offset: nil); end
+
+      # Search memories by content (basic text search)
+      sig { abstract.params(query: String, user_id: T.nilable(String), limit: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
+      def search(query, user_id: nil, limit: nil); end
+
+      # Search memories by tags
+      sig { abstract.params(tags: T::Array[String], user_id: T.nilable(String), limit: T.nilable(Integer)).returns(T::Array[MemoryRecord]) }
+      def search_by_tags(tags, user_id: nil, limit: nil); end
+
+      # Vector similarity search (if supported by backend)
+      sig { abstract.params(embedding: T::Array[Float], user_id: T.nilable(String), limit: T.nilable(Integer), threshold: T.nilable(Float)).returns(T::Array[MemoryRecord]) }
+      def vector_search(embedding, user_id: nil, limit: nil, threshold: nil); end
+
+      # Count total memories
+      sig { abstract.params(user_id: T.nilable(String)).returns(Integer) }
+      def count(user_id: nil); end
+
+      # Clear all memories for a user (or all if user_id is nil)
+      sig { abstract.params(user_id: T.nilable(String)).returns(Integer) }
+      def clear(user_id: nil); end
+
+      # Check if the store supports vector search
+      sig { returns(T::Boolean) }
+      def supports_vector_search?
+        false
+      end
+
+      # Get store statistics
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      def stats
+        {
+          total_memories: count,
+          supports_vector_search: supports_vector_search?
+        }
+      end
+
+      # Batch operations
+      sig { params(records: T::Array[MemoryRecord]).returns(T::Array[T::Boolean]) }
+      def store_batch(records)
+        records.map { |record| store(record) }
+      end
+
+      sig { params(ids: T::Array[String]).returns(T::Array[T.nilable(MemoryRecord)]) }
+      def retrieve_batch(ids)
+        ids.map { |id| retrieve(id) }
+      end
+
+      sig { params(records: T::Array[MemoryRecord]).returns(T::Array[T::Boolean]) }
+      def update_batch(records)
+        records.map { |record| update(record) }
+      end
+
+      sig { params(ids: T::Array[String]).returns(T::Array[T::Boolean]) }
+      def delete_batch(ids)
+        ids.map { |id| delete(id) }
+      end
+    end
+  end
+end

data/lib/dspy/memory.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative 'memory/memory_record'
+require_relative 'memory/memory_store'
+require_relative 'memory/in_memory_store'
+require_relative 'memory/embedding_engine'
+require_relative 'memory/local_embedding_engine'
+require_relative 'memory/memory_compactor'
+require_relative 'memory/memory_manager'
+
+module DSPy
+  # Memory system for persistent, searchable agent memory
+  module Memory
+    class << self
+      extend T::Sig
+
+      # Configure the memory system
+      sig { returns(MemoryManager) }
+      def manager
+        @manager ||= MemoryManager.new
+      end
+
+      # Reset the memory system (useful for testing)
+      sig { void }
+      def reset!
+        @manager = nil
+      end
+    end
+  end
+end

data/lib/dspy/mixins/instrumentation_helpers.rb

@@ -65,7 +65,7 @@ module DSPy
         # Smart consolidation: skip nested events when higher-level events are being emitted
         if is_nested_context?
           # If we're in a nested context, only emit higher-level events
-          event_name.match?(/^dspy\.(chain_of_thought|react)$/)
+          event_name.match?(/^dspy\.(chain_of_thought|react|codeact)$/)
         else
           # If we're not in a nested context, emit all events normally
           true
@@ -103,11 +103,9 @@ module DSPy
         return false if caller_locations.nil?
         
         # Look for higher-level DSPy modules in the call stack
-        # We consider ChainOfThought and ReAct as higher-level modules
+        # We consider ChainOfThought, ReAct, and CodeAct as higher-level modules
         higher_level_modules = caller_locations.select do |loc|
-          loc.path.include?('chain_of_thought') || 
-          loc.path.include?('re_act') ||
-          loc.path.include?('react')
+          loc.path.match?(/(?:chain_of_thought|re_act|react|code_act)/)
         end
         
         # If we have higher-level modules in the call stack, we're in a nested context

data/lib/dspy/mixins/type_coercion.rb

@@ -25,6 +25,9 @@ module DSPy
       sig { params(value: T.untyped, prop_type: T.untyped).returns(T.untyped) }
       def coerce_value_to_type(value, prop_type)
         return value unless prop_type
+        
+        # If value is nil, return it as-is for nilable types
+        return value if value.nil?
 
         case prop_type
         when ->(type) { enum_type?(type) }

data/lib/dspy/prompt.rb

@@ -121,7 +121,7 @@ module DSPy
       
       sections << "## Input Values"
       sections << "```json"
-      sections << JSON.pretty_generate(input_values)
+      sections << JSON.pretty_generate(serialize_for_json(input_values))
       sections << "```"
       
       sections << ""
@@ -218,5 +218,52 @@ module DSPy
         output_fields: @output_schema.dig(:properties)&.keys&.length || 0
       }
     end
+
+    private
+
+    # Recursively serialize complex objects for JSON representation
+    sig { params(obj: T.untyped).returns(T.untyped) }
+    def serialize_for_json(obj)
+      case obj
+      when T::Struct
+        # Convert T::Struct to hash using to_h method if available
+        if obj.respond_to?(:to_h)
+          serialize_for_json(obj.to_h)
+        else
+          # Fallback: serialize using struct properties
+          serialize_struct_to_hash(obj)
+        end
+      when Hash
+        # Recursively serialize hash values
+        obj.transform_values { |v| serialize_for_json(v) }
+      when Array
+        # Recursively serialize array elements
+        obj.map { |item| serialize_for_json(item) }
+      when T::Enum
+        # Serialize enums to their string representation
+        obj.serialize
+      else
+        # For basic types (String, Integer, Float, Boolean, etc.), return as-is
+        obj
+      end
+    end
+
+    # Fallback method to serialize T::Struct to hash when to_h is not available
+    sig { params(struct_obj: T::Struct).returns(T::Hash[Symbol, T.untyped]) }
+    def serialize_struct_to_hash(struct_obj)
+      result = {}
+      
+      # Use struct's props method to get all properties
+      if struct_obj.class.respond_to?(:props)
+        struct_obj.class.props.each do |prop_name, _prop_info|
+          if struct_obj.respond_to?(prop_name)
+            value = struct_obj.public_send(prop_name)
+            result[prop_name] = serialize_for_json(value)
+          end
+        end
+      end
+      
+      result
+    end
   end
 end

data/lib/dspy/subscribers/logger_subscriber.rb

@@ -52,6 +52,18 @@ module DSPy
           log_react_tool_call(event)
         end
 
+        DSPy::Instrumentation.subscribe('dspy.codeact') do |event|
+          log_codeact(event)
+        end
+
+        DSPy::Instrumentation.subscribe('dspy.codeact.iteration_complete') do |event|
+          log_codeact_iteration_complete(event)
+        end
+
+        DSPy::Instrumentation.subscribe('dspy.codeact.code_execution') do |event|
+          log_codeact_code_execution(event)
+        end
+
         # Subscribe to optimization events
         DSPy::Instrumentation.subscribe('dspy.optimization.start') do |event|
           log_optimization_start(event)
@@ -236,7 +248,7 @@ module DSPy
           "status=#{status}",
           "duration_ms=#{duration}"
         ]
-        log_parts << "thought=\"#{thought&.truncate(100)}\"" if thought
+        log_parts << "thought=\"#{thought && thought.length > 100 ? thought[0..97] + '...' : thought}\"" if thought
         log_parts << "action=\"#{action}\"" if action
         log_parts << "error=\"#{payload[:error_message]}\"" if status == 'error' && payload[:error_message]
 
@@ -263,6 +275,84 @@ module DSPy
         logger.info(log_parts.join(' '))
       end
 
+      sig { params(event: T.untyped).void }
+      def log_codeact(event)
+        payload = event.payload
+        signature = payload[:signature_class]
+        duration = payload[:duration_ms]&.round(2)
+        status = payload[:status]
+        iteration_count = payload[:iteration_count]
+        code_executions = payload[:code_executions]
+        final_answer = payload[:final_answer]
+        timestamp = format_timestamp(payload)
+
+        log_parts = [
+          "event=codeact",
+          timestamp,
+          "signature=#{signature}",
+          "status=#{status}",
+          "duration_ms=#{duration}"
+        ].compact
+        log_parts << "iterations=#{iteration_count}" if iteration_count
+        log_parts << "code_executions=#{code_executions}" if code_executions
+        log_parts << "final_answer=\"#{final_answer&.truncate(100)}\"" if final_answer
+        log_parts << "error=\"#{payload[:error_message]}\"" if status == 'error' && payload[:error_message]
+
+        logger.info(log_parts.join(' '))
+      end
+
+      sig { params(event: T.untyped).void }
+      def log_codeact_iteration_complete(event)
+        payload = event.payload
+        iteration = payload[:iteration]
+        thought = payload[:thought]
+        ruby_code = payload[:ruby_code]
+        observation = payload[:observation]
+        duration = payload[:duration_ms]&.round(2)
+        status = payload[:status]
+        timestamp = format_timestamp(payload)
+
+        log_parts = [
+          "event=codeact_iteration",
+          timestamp,
+          "iteration=#{iteration}",
+          "status=#{status}",
+          "duration_ms=#{duration}"
+        ].compact
+        log_parts << "thought=\"#{thought && thought.length > 100 ? thought[0..97] + '...' : thought}\"" if thought
+        log_parts << "code=\"#{ruby_code && ruby_code.length > 100 ? ruby_code[0..97] + '...' : ruby_code}\"" if ruby_code
+        log_parts << "observation=\"#{observation && observation.length > 100 ? observation[0..97] + '...' : observation}\"" if observation
+        log_parts << "error=\"#{payload[:error_message]}\"" if status == 'error' && payload[:error_message]
+
+        logger.info(log_parts.join(' '))
+      end
+
+      sig { params(event: T.untyped).void }
+      def log_codeact_code_execution(event)
+        payload = event.payload
+        iteration = payload[:iteration]
+        ruby_code = payload[:ruby_code]
+        execution_result = payload[:execution_result]
+        execution_error = payload[:execution_error]
+        duration = payload[:duration_ms]&.round(2)
+        status = payload[:status]
+        timestamp = format_timestamp(payload)
+
+        log_parts = [
+          "event=code_execution",
+          timestamp,
+          "iteration=#{iteration}",
+          "status=#{status}",
+          "duration_ms=#{duration}"
+        ].compact
+        log_parts << "code=\"#{ruby_code && ruby_code.length > 50 ? ruby_code[0..47] + '...' : ruby_code}\"" if ruby_code
+        log_parts << "result=\"#{execution_result && execution_result.length > 100 ? execution_result[0..97] + '...' : execution_result}\"" if execution_result
+        log_parts << "execution_error=\"#{execution_error}\"" if execution_error
+        log_parts << "error=\"#{payload[:error_message]}\"" if status == 'error' && payload[:error_message]
+
+        logger.info(log_parts.join(' '))
+      end
+
       # Optimization event logging methods
       sig { params(event: T.untyped).void }
       def log_optimization_start(event)

data/lib/dspy/tools/base.rb

@@ -195,7 +195,7 @@ module DSPy
 
       # Subclasses must implement their own call method with their own signature
 
-      private
+      protected
 
       # Convert argument to the expected type based on JSON schema
       sig { params(value: T.untyped, schema: T::Hash[Symbol, T.untyped]).returns(T.untyped) }

data/lib/dspy/tools/memory_toolset.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+require_relative 'toolset'
+
+module DSPy
+  module Tools
+    # Example implementation of a memory toolset for agents
+    # Provides tools for storing, retrieving, and managing memory
+    class MemoryToolset < Toolset
+      extend T::Sig
+
+      toolset_name "memory"
+
+      # Expose methods as tools with descriptions
+      tool :store, description: "Store a key-value pair in memory with optional tags"
+      tool :retrieve, description: "Retrieve a value by key from memory"
+      tool :search, description: "Search memories by pattern in keys and/or values"
+      tool :list_keys, tool_name: "memory_list", description: "List all stored memory keys"
+      tool :update, description: "Update an existing memory value"
+      tool :delete, description: "Delete a memory by key"
+      tool :clear, description: "Clear all stored memories"
+      tool :count, description: "Get the count of stored memories"
+      tool :get_metadata, description: "Get metadata for a specific memory"
+
+      sig { void }
+      def initialize
+        @memory = T.let({}, T::Hash[String, T::Hash[Symbol, T.untyped]])
+      end
+
+      sig { params(key: String, value: String, tags: T.nilable(T::Array[String])).returns(String) }
+      def store(key:, value:, tags: nil)
+        @memory[key] = {
+          value: value,
+          tags: tags || [],
+          created_at: Time.now,
+          updated_at: Time.now,
+          access_count: 0
+        }
+        "Stored memory '#{key}' successfully"
+      end
+
+      sig { params(key: String).returns(T.nilable(String)) }
+      def retrieve(key:)
+        entry = @memory[key]
+        return nil unless entry
+
+        # Track access
+        entry[:access_count] += 1
+        entry[:last_accessed_at] = Time.now
+        entry[:value]
+      end
+
+      sig { params(pattern: String, in_keys: T::Boolean, in_values: T::Boolean).returns(T::Array[T::Hash[Symbol, String]]) }
+      def search(pattern:, in_keys: true, in_values: true)
+        results = []
+        regex = Regexp.new(pattern, Regexp::IGNORECASE)
+
+        @memory.each do |key, entry|
+          match = (in_keys && key.match?(regex)) || (in_values && entry[:value].match?(regex))
+          results << { key: key, value: entry[:value] } if match
+        end
+
+        results
+      end
+
+      sig { returns(T::Array[String]) }
+      def list_keys
+        @memory.keys.sort
+      end
+
+      sig { params(key: String, value: String).returns(String) }
+      def update(key:, value:)
+        return "Memory '#{key}' not found" unless @memory.key?(key)
+
+        @memory[key][:value] = value
+        @memory[key][:updated_at] = Time.now
+        "Updated memory '#{key}' successfully"
+      end
+
+      sig { params(key: String).returns(String) }
+      def delete(key:)
+        return "Memory '#{key}' not found" unless @memory.key?(key)
+
+        @memory.delete(key)
+        "Deleted memory '#{key}' successfully"
+      end
+
+      sig { returns(String) }
+      def clear
+        count = @memory.size
+        @memory.clear
+        "Cleared #{count} memories"
+      end
+
+      sig { returns(Integer) }
+      def count
+        @memory.size
+      end
+
+      sig { params(key: String).returns(T.nilable(T::Hash[Symbol, T.untyped])) }
+      def get_metadata(key:)
+        entry = @memory[key]
+        return nil unless entry
+
+        {
+          created_at: entry[:created_at],
+          updated_at: entry[:updated_at],
+          access_count: entry[:access_count],
+          last_accessed_at: entry[:last_accessed_at],
+          tags: entry[:tags],
+          value_length: entry[:value].length
+        }
+      end
+    end
+  end
+end

data/lib/dspy/tools/text_processing_toolset.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+require 'tempfile'
+require 'set'
+require_relative 'toolset'
+
+module DSPy
+  module Tools
+    # Text processing toolset that provides text analysis and manipulation tools
+    # Includes grep, word count, ripgrep, and other text processing utilities
+    class TextProcessingToolset < Toolset
+      extend T::Sig
+
+      toolset_name "text"
+
+      # Expose methods as tools with descriptions
+      tool :grep, description: "Search for patterns in text using grep"
+      tool :word_count, tool_name: "text_wc", description: "Count lines, words, and characters in text"
+      tool :ripgrep, tool_name: "text_rg", description: "Fast text search using ripgrep"
+      tool :extract_lines, description: "Extract specific line ranges from text"
+      tool :filter_lines, description: "Filter lines matching or not matching a pattern"
+      tool :unique_lines, description: "Get unique lines from text"
+      tool :sort_lines, description: "Sort lines in text"
+      tool :summarize_text, description: "Generate statistical summary of text content"
+
+      sig { void }
+      def initialize
+        # No persistent state needed for text processing
+      end
+
+      sig { params(text: String, pattern: String, ignore_case: T::Boolean, count_only: T::Boolean).returns(String) }
+      def grep(text:, pattern:, ignore_case: true, count_only: false)
+        # Create temporary file to use with grep
+        temp_file = Tempfile.new('text_processing')
+        temp_file.write(text)
+        temp_file.close
+
+        flags = []
+        flags << '-i' if ignore_case
+        flags << '-c' if count_only
+
+        cmd = "grep #{flags.join(' ')} '#{pattern}' '#{temp_file.path}'"
+        result = `#{cmd} 2>/dev/null`
+        
+        temp_file.unlink
+        
+        if count_only
+          "Found #{result.strip} matches for pattern '#{pattern}'"
+        elsif result.empty?
+          "No matches found for pattern '#{pattern}'"
+        else
+          result
+        end
+      rescue => e
+        "Error running grep: #{e.message}"
+      end
+
+      sig { params(text: String, lines_only: T::Boolean, words_only: T::Boolean, chars_only: T::Boolean).returns(String) }
+      def word_count(text:, lines_only: false, words_only: false, chars_only: false)
+        lines = text.lines.count
+        words = text.split(/\s+/).reject(&:empty?).count
+        chars = text.length
+
+        if lines_only
+          "Lines: #{lines}"
+        elsif words_only
+          "Words: #{words}"
+        elsif chars_only
+          "Characters: #{chars}"
+        else
+          "Lines: #{lines}, Words: #{words}, Characters: #{chars}"
+        end
+      end
+
+      sig { params(text: String, pattern: String, context: Integer).returns(String) }
+      def ripgrep(text:, pattern:, context: 0)
+        temp_file = Tempfile.new('text_processing')
+        temp_file.write(text)
+        temp_file.close
+
+        cmd = "rg"
+        cmd += " -C #{context}" if context > 0
+        cmd += " '#{pattern}' '#{temp_file.path}'"
+        
+        result = `#{cmd} 2>/dev/null`
+        
+        temp_file.unlink
+        
+        if result.empty?
+          "No matches found for pattern '#{pattern}'"
+        else
+          result
+        end
+      rescue => e
+        "Error running ripgrep: #{e.message}"
+      end
+
+      sig { params(text: String, start_line: Integer, end_line: T.nilable(Integer)).returns(String) }
+      def extract_lines(text:, start_line:, end_line: nil)
+        lines = text.lines
+        start_idx = [start_line - 1, 0].max  # Convert to 0-based, ensure >= 0
+        
+        if end_line
+          end_idx = [end_line - 1, lines.length - 1].min  # Convert to 0-based, ensure <= last line
+          extracted = lines[start_idx..end_idx]
+        else
+          extracted = lines[start_idx, 1]  # Just one line
+        end
+        
+        extracted&.join || ""
+      end
+
+      sig { params(text: String, pattern: String, invert: T::Boolean).returns(String) }
+      def filter_lines(text:, pattern:, invert: false)
+        lines = text.lines
+        regex = Regexp.new(pattern, Regexp::IGNORECASE)
+        
+        filtered = if invert
+          lines.reject { |line| line.match?(regex) }
+        else
+          lines.select { |line| line.match?(regex) }
+        end
+        
+        filtered.join
+      end
+
+      sig { params(text: String, preserve_order: T::Boolean).returns(String) }
+      def unique_lines(text:, preserve_order: true)
+        lines = text.lines.map(&:chomp)
+        
+        unique = if preserve_order
+          lines.uniq
+        else
+          lines.to_set.to_a.sort
+        end
+        
+        unique.map { |line| "#{line}\n" }.join
+      end
+
+      sig { params(text: String, reverse: T::Boolean, numeric: T::Boolean).returns(String) }
+      def sort_lines(text:, reverse: false, numeric: false)
+        lines = text.lines.map(&:chomp)
+        
+        sorted = if numeric
+          lines.sort_by { |line| line.to_f }
+        else
+          lines.sort
+        end
+        
+        sorted.reverse! if reverse
+        sorted.map { |line| "#{line}\n" }.join
+      end
+
+      sig { params(text: String).returns(String) }
+      def summarize_text(text:)
+        lines = text.lines
+        words = text.split(/\s+/).reject(&:empty?)
+        chars = text.length
+        
+        # Find most common words (simple analysis)
+        word_freq = words.each_with_object(Hash.new(0)) { |word, hash| hash[word.downcase.gsub(/[^\w]/, '')] += 1 }
+        top_words = word_freq.reject { |word, _| word.length < 3 }.sort_by { |_, count| -count }.first(5)
+        
+        # Basic text statistics
+        avg_line_length = lines.empty? ? 0 : (chars.to_f / lines.count).round(2)
+        avg_word_length = words.empty? ? 0 : (words.sum(&:length).to_f / words.count).round(2)
+        
+        summary = []
+        summary << "Text Summary:"
+        summary << "  Lines: #{lines.count}"
+        summary << "  Words: #{words.count}"
+        summary << "  Characters: #{chars}"
+        summary << "  Average line length: #{avg_line_length}"
+        summary << "  Average word length: #{avg_word_length}"
+        
+        unless top_words.empty?
+          summary << "  Most frequent words:"
+          top_words.each { |word, count| summary << "    #{word}: #{count}" }
+        end
+        
+        summary.join("\n")
+      end
+    end
+  end
+end