dspy 0.15.7 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b7160b378d76b4772dba1d2b0d22217087f062a126b86a5b8485810eaacf8e2f
-  data.tar.gz: 8f7eae40798e8f9d79a4666ff6af9771405ebf80f5e34808eb25ee6618521963
+  metadata.gz: 4ad35c319e21d14ee6ac3d27a6dece4b3eb1c9818ef7314d028b937806e6db4e
+  data.tar.gz: 3e782adb5c7daa973dfeb3d0adf26235190ff6d76dac0bf133868ea3d515fdea
 SHA512:
-  metadata.gz: 990af89217c7a7513517b21efbb347202a89232c6ba67ff6fbc1ec2a4c14b94da5225f0c3ce01b0c65db6e5c35b688e6e1daea50d7a30df2f29d5a6218917c5d
-  data.tar.gz: 8684852589ccc036b0d3efa4a9cbe70537e2912cf9ba0f862c4381da75b08a9befb6341169d1ce334fff2fa7df0819b9e15bfe711e896d473174f101fa856667
+  metadata.gz: b5ea3a0696fa843326ad959757051b3067f7f18715d4f8be2be68f7412dd41c34ce3bd7e9c541fe1d7637a3a1a36b2056152b5131147a3652b505d2f2f6fb023
+  data.tar.gz: 0f59b08312c2d562619154c7de42958dd01830e44b0139dfd9352256a84085098e42026ec319468c85d0942fd998bbb73773e714b96b58e242751d426afbb647

data/README.md

@@ -11,6 +11,46 @@ Traditional prompting is like writing code with string concatenation: it works u
 
 The result? LLM applications that actually scale and don't break when you sneeze.
 
+## Your First DSPy Program
+
+```ruby
+# Define a signature for sentiment classification
+class Classify < DSPy::Signature
+  description "Classify sentiment of a given sentence."
+
+  class Sentiment < T::Enum
+    enums do
+      Positive = new('positive')
+      Negative = new('negative')
+      Neutral = new('neutral')
+    end
+  end
+
+  input do
+    const :sentence, String
+  end
+
+  output do
+    const :sentiment, Sentiment
+    const :confidence, Float
+  end
+end
+
+# Configure DSPy with your LLM
+DSPy.configure do |c|
+  c.lm = DSPy::LM.new('openai/gpt-4o-mini', 
+                      api_key: ENV['OPENAI_API_KEY'],
+                      structured_outputs: true)  # Enable OpenAI's native JSON mode
+end
+
+# Create the predictor and run inference
+classify = DSPy::Predict.new(Classify)
+result = classify.call(sentence: "This book was super fun to read!")
+
+puts result.sentiment    # => #<Sentiment::Positive>  
+puts result.confidence   # => 0.85
+```
+
 ## What You Get
 
 **Core Building Blocks:**
@@ -95,46 +135,6 @@ sudo apt-get install cmake
 
 **Note**: The `polars-df` gem compilation can take 15-20 minutes. Pre-built binaries are available for most platforms, so compilation is only needed if a pre-built binary isn't available for your system.
 
-### Your First DSPy Program
-
-```ruby
-# Define a signature for sentiment classification
-class Classify < DSPy::Signature
-  description "Classify sentiment of a given sentence."
-
-  class Sentiment < T::Enum
-    enums do
-      Positive = new('positive')
-      Negative = new('negative')
-      Neutral = new('neutral')
-    end
-  end
-
-  input do
-    const :sentence, String
-  end
-
-  output do
-    const :sentiment, Sentiment
-    const :confidence, Float
-  end
-end
-
-# Configure DSPy with your LLM
-DSPy.configure do |c|
-  c.lm = DSPy::LM.new('openai/gpt-4o-mini', 
-                      api_key: ENV['OPENAI_API_KEY'],
-                      structured_outputs: true)  # Enable OpenAI's native JSON mode
-end
-
-# Create the predictor and run inference
-classify = DSPy::Predict.new(Classify)
-result = classify.call(sentence: "This book was super fun to read!")
-
-puts result.sentiment    # => #<Sentiment::Positive>  
-puts result.confidence   # => 0.85
-```
-
 ## Documentation
 
 📖 **[Complete Documentation Website](https://vicentereig.github.io/dspy.rb/)**

data/lib/dspy/image.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+require 'base64'
+require 'uri'
+
+module DSPy
+  class Image
+    attr_reader :url, :base64, :data, :content_type, :detail
+    
+    SUPPORTED_FORMATS = %w[image/jpeg image/png image/gif image/webp].freeze
+    MAX_SIZE_BYTES = 5 * 1024 * 1024 # 5MB limit
+    
+    # Provider capability registry
+    PROVIDER_CAPABILITIES = {
+      'openai' => {
+        sources: %w[url base64 data],
+        parameters: %w[detail]
+      },
+      'anthropic' => {
+        sources: %w[base64 data], 
+        parameters: []
+      }
+    }.freeze
+    
+    def initialize(url: nil, base64: nil, data: nil, content_type: nil, detail: nil)
+      @detail = detail # OpenAI detail level: 'low', 'high', or 'auto'
+      
+      # Validate input
+      validate_input!(url, base64, data)
+      
+      if url
+        @url = url
+        @content_type = content_type || infer_content_type_from_url(url)
+      elsif base64
+        raise ArgumentError, "content_type is required when using base64" unless content_type
+        @base64 = base64
+        @content_type = content_type
+        validate_size!(Base64.decode64(base64).bytesize)
+      elsif data
+        raise ArgumentError, "content_type is required when using data" unless content_type
+        @data = data
+        @content_type = content_type
+        validate_size!(data.size)
+      end
+      
+      validate_content_type!
+    end
+    
+    def to_openai_format
+      if url
+        format = {
+          type: 'image_url',
+          image_url: {
+            url: url
+          }
+        }
+        format[:image_url][:detail] = detail if detail
+        format
+      elsif base64
+        {
+          type: 'image_url',
+          image_url: {
+            url: "data:#{content_type};base64,#{base64}"
+          }
+        }
+      elsif data
+        {
+          type: 'image_url',
+          image_url: {
+            url: "data:#{content_type};base64,#{to_base64}"
+          }
+        }
+      end
+    end
+    
+    def to_anthropic_format
+      if url
+        # Anthropic requires base64, so we'd need to fetch the URL
+        # For now, we'll raise an error or skip
+        raise NotImplementedError, "URL fetching for Anthropic not yet implemented"
+      elsif base64
+        {
+          type: 'image',
+          source: {
+            type: 'base64',
+            media_type: content_type,
+            data: base64
+          }
+        }
+      elsif data
+        {
+          type: 'image',
+          source: {
+            type: 'base64',
+            media_type: content_type,
+            data: to_base64
+          }
+        }
+      end
+    end
+    
+    def to_base64
+      return base64 if base64
+      return Base64.strict_encode64(data.pack('C*')) if data
+      nil
+    end
+    
+    def validate!
+      validate_content_type!
+      
+      if base64
+        validate_size!(Base64.decode64(base64).bytesize)
+      elsif data
+        validate_size!(data.size)
+      end
+    end
+    
+    def validate_for_provider!(provider)
+      capabilities = PROVIDER_CAPABILITIES[provider]
+      
+      unless capabilities
+        raise DSPy::LM::IncompatibleImageFeatureError, 
+              "Unknown provider '#{provider}'. Supported providers: #{PROVIDER_CAPABILITIES.keys.join(', ')}"
+      end
+      
+      # Check source compatibility
+      current_source = if url
+                         'url'
+                       elsif base64
+                         'base64'
+                       elsif data
+                         'data'
+                       end
+      
+      unless capabilities[:sources].include?(current_source)
+        case provider
+        when 'anthropic'
+          if current_source == 'url'
+            raise DSPy::LM::IncompatibleImageFeatureError,
+                  "Anthropic doesn't support image URLs. Please provide base64 or raw data instead."
+          end
+        end
+      end
+      
+      # Check parameter compatibility
+      if detail && !capabilities[:parameters].include?('detail')
+        case provider
+        when 'anthropic'
+          raise DSPy::LM::IncompatibleImageFeatureError,
+                "Anthropic doesn't support the 'detail' parameter. This feature is OpenAI-specific."
+        end
+      end
+    end
+    
+    private
+    
+    def validate_input!(url, base64, data)
+      inputs = [url, base64, data].compact
+      
+      if inputs.empty?
+        raise ArgumentError, "Must provide either url, base64, or data"
+      elsif inputs.size > 1
+        raise ArgumentError, "Only one of url, base64, or data can be provided"
+      end
+    end
+    
+    def validate_content_type!
+      unless SUPPORTED_FORMATS.include?(content_type)
+        raise ArgumentError, "Unsupported image format: #{content_type}. Supported formats: #{SUPPORTED_FORMATS.join(', ')}"
+      end
+    end
+    
+    def validate_size!(size_bytes)
+      if size_bytes > MAX_SIZE_BYTES
+        raise ArgumentError, "Image size exceeds 5MB limit (got #{size_bytes} bytes)"
+      end
+    end
+    
+    def infer_content_type_from_url(url)
+      extension = File.extname(URI.parse(url).path).downcase
+      
+      case extension
+      when '.jpg', '.jpeg'
+        'image/jpeg'
+      when '.png'
+        'image/png'
+      when '.gif'
+        'image/gif'
+      when '.webp'
+        'image/webp'
+      else
+        'image/jpeg' # Default fallback
+      end
+    end
+  end
+end

data/lib/dspy/lm/adapter.rb

@@ -36,10 +36,25 @@ module DSPy
       # Helper method to normalize message format
       def normalize_messages(messages)
         messages.map do |msg|
-          {
-            role: msg[:role].to_s,
-            content: msg[:content].to_s
-          }
+          # Support both Message objects and hash format
+          if msg.is_a?(DSPy::LM::Message)
+            msg.to_h
+          else
+            content = msg[:content]
+            # Don't convert array content to string
+            {
+              role: msg[:role].to_s,
+              content: content.is_a?(Array) ? content : content.to_s
+            }
+          end
+        end
+      end
+      
+      # Check if messages contain images
+      def contains_images?(messages)
+        messages.any? do |msg|
+          content = msg[:content] || msg.content
+          content.is_a?(Array) && content.any? { |item| item[:type] == 'image' }
         end
       end
     end

data/lib/dspy/lm/adapters/anthropic_adapter.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'anthropic'
+require_relative '../vision_models'
 
 module DSPy
   class LM
@@ -12,14 +13,23 @@ module DSPy
       end
 
       def chat(messages:, signature: nil, **extra_params, &block)
+        normalized_messages = normalize_messages(messages)
+        
+        # Validate vision support if images are present
+        if contains_images?(normalized_messages)
+          VisionModels.validate_vision_support!('anthropic', model)
+          # Convert messages to Anthropic format with proper image handling
+          normalized_messages = format_multimodal_messages(normalized_messages)
+        end
+        
         # Anthropic requires system message to be separate from messages
-        system_message, user_messages = extract_system_message(normalize_messages(messages))
+        system_message, user_messages = extract_system_message(normalized_messages)
         
         # Check if this is a tool use request
         has_tools = extra_params.key?(:tools) && !extra_params[:tools].empty?
         
         # Apply JSON prefilling if needed for better Claude JSON compliance (but not for tool use)
-        unless has_tools
+        unless has_tools || contains_images?(normalized_messages)
           user_messages = prepare_messages_for_json(user_messages, system_message)
         end
         
@@ -111,7 +121,21 @@ module DSPy
             )
           end
         rescue => e
-          raise AdapterError, "Anthropic adapter error: #{e.message}"
+          # Check for specific image-related errors in the message
+          error_msg = e.message.to_s
+          
+          if error_msg.include?('Could not process image')
+            raise AdapterError, "Image processing failed: #{error_msg}. Ensure your image is a valid PNG, JPEG, GIF, or WebP format, properly base64-encoded, and under 5MB."
+          elsif error_msg.include?('image')
+            raise AdapterError, "Image error: #{error_msg}. Anthropic requires base64-encoded images (URLs are not supported)."
+          elsif error_msg.include?('rate')
+            raise AdapterError, "Anthropic rate limit exceeded: #{error_msg}. Please wait and try again."
+          elsif error_msg.include?('authentication') || error_msg.include?('API key')
+            raise AdapterError, "Anthropic authentication failed: #{error_msg}. Check your API key."
+          else
+            # Generic error handling
+            raise AdapterError, "Anthropic adapter error: #{e.message}"
+          end
         end
       end
 
@@ -234,6 +258,33 @@ module DSPy
 
         [system_message, user_messages]
       end
+      
+      def format_multimodal_messages(messages)
+        messages.map do |msg|
+          if msg[:content].is_a?(Array)
+            # Convert multimodal content to Anthropic format
+            formatted_content = msg[:content].map do |item|
+              case item[:type]
+              when 'text'
+                { type: 'text', text: item[:text] }
+              when 'image'
+                # Validate image compatibility before formatting
+                item[:image].validate_for_provider!('anthropic')
+                item[:image].to_anthropic_format
+              else
+                item
+              end
+            end
+            
+            {
+              role: msg[:role],
+              content: formatted_content
+            }
+          else
+            msg
+          end
+        end
+      end
     end
   end
 end

data/lib/dspy/lm/adapters/openai_adapter.rb

@@ -2,6 +2,7 @@
 
 require 'openai'
 require_relative 'openai/schema_converter'
+require_relative '../vision_models'
 
 module DSPy
   class LM
@@ -14,9 +15,18 @@ module DSPy
       end
 
       def chat(messages:, signature: nil, response_format: nil, &block)
+        normalized_messages = normalize_messages(messages)
+        
+        # Validate vision support if images are present
+        if contains_images?(normalized_messages)
+          VisionModels.validate_vision_support!('openai', model)
+          # Convert messages to OpenAI format with proper image handling
+          normalized_messages = format_multimodal_messages(normalized_messages)
+        end
+        
         request_params = {
           model: model,
-          messages: normalize_messages(messages),
+          messages: normalized_messages,
           temperature: 0.0 # DSPy default for deterministic responses
         }
 
@@ -72,7 +82,29 @@ module DSPy
             metadata: metadata
           )
         rescue => e
-          raise AdapterError, "OpenAI adapter error: #{e.message}"
+          # Check for specific error types and messages
+          error_msg = e.message.to_s
+          
+          # Try to parse error body if it looks like JSON
+          error_body = if error_msg.start_with?('{')
+                         JSON.parse(error_msg) rescue nil
+                       elsif e.respond_to?(:response) && e.response
+                         e.response[:body] rescue nil
+                       end
+          
+          # Check for specific image-related errors
+          if error_msg.include?('image_parse_error') || error_msg.include?('unsupported image')
+            raise AdapterError, "Image processing failed: #{error_msg}. Ensure your image is a valid PNG, JPEG, GIF, or WebP format and under 5MB."
+          elsif error_msg.include?('rate') && error_msg.include?('limit')
+            raise AdapterError, "OpenAI rate limit exceeded: #{error_msg}. Please wait and try again."
+          elsif error_msg.include?('authentication') || error_msg.include?('API key') || error_msg.include?('Unauthorized')
+            raise AdapterError, "OpenAI authentication failed: #{error_msg}. Check your API key."
+          elsif error_body && error_body.dig('error', 'message')
+            raise AdapterError, "OpenAI API error: #{error_body.dig('error', 'message')}"
+          else
+            # Generic error handling
+            raise AdapterError, "OpenAI adapter error: #{e.message}"
+          end
         end
       end
 
@@ -81,6 +113,33 @@ module DSPy
       def supports_structured_outputs?
         DSPy::LM::Adapters::OpenAI::SchemaConverter.supports_structured_outputs?(model)
       end
+      
+      def format_multimodal_messages(messages)
+        messages.map do |msg|
+          if msg[:content].is_a?(Array)
+            # Convert multimodal content to OpenAI format
+            formatted_content = msg[:content].map do |item|
+              case item[:type]
+              when 'text'
+                { type: 'text', text: item[:text] }
+              when 'image'
+                # Validate image compatibility before formatting
+                item[:image].validate_for_provider!('openai')
+                item[:image].to_openai_format
+              else
+                item
+              end
+            end
+            
+            {
+              role: msg[:role],
+              content: formatted_content
+            }
+          else
+            msg
+          end
+        end
+      end
     end
   end
 end

data/lib/dspy/lm/errors.rb

@@ -19,5 +19,12 @@ module DSPy
         super("API key is required but was not provided. Set it via the api_key parameter or #{env_var} environment variable.")
       end
     end
+    
+    # Raised when image features are incompatible with the target provider
+    class IncompatibleImageFeatureError < AdapterError
+      def initialize(message)
+        super(message)
+      end
+    end
   end
 end

data/lib/dspy/lm/message.rb

@@ -18,7 +18,7 @@ module DSPy
       end
       
       const :role, Role
-      const :content, String
+      const :content, T.any(String, T::Array[T::Hash[Symbol, T.untyped]])
       const :name, T.nilable(String), default: nil
       
       sig { returns(T::Hash[Symbol, T.untyped]) }
@@ -33,7 +33,64 @@ module DSPy
       
       sig { returns(String) }
       def to_s
-        name ? "#{role.serialize}(#{name}): #{content}" : "#{role.serialize}: #{content}"
+        if content.is_a?(String)
+          name ? "#{role.serialize}(#{name}): #{content}" : "#{role.serialize}: #{content}"
+        else
+          name ? "#{role.serialize}(#{name}): [multimodal content]" : "#{role.serialize}: [multimodal content]"
+        end
+      end
+      
+      sig { returns(T::Boolean) }
+      def multimodal?
+        content.is_a?(Array)
+      end
+      
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      def to_openai_format
+        formatted = { role: role.serialize }
+        
+        if content.is_a?(String)
+          formatted[:content] = content
+        else
+          # Convert multimodal content array to OpenAI format
+          formatted[:content] = content.map do |item|
+            case item[:type]
+            when 'text'
+              { type: 'text', text: item[:text] }
+            when 'image'
+              item[:image].to_openai_format
+            else
+              item
+            end
+          end
+        end
+        
+        formatted[:name] = name if name
+        formatted
+      end
+      
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      def to_anthropic_format
+        formatted = { role: role.serialize }
+        
+        if content.is_a?(String)
+          formatted[:content] = content
+        else
+          # Convert multimodal content array to Anthropic format
+          formatted[:content] = content.map do |item|
+            case item[:type]
+            when 'text'
+              { type: 'text', text: item[:text] }
+            when 'image'
+              item[:image].to_anthropic_format
+            else
+              item
+            end
+          end
+        end
+        
+        formatted[:name] = name if name
+        formatted
       end
     end
     
@@ -71,10 +128,17 @@ module DSPy
       sig { params(data: T::Hash[Symbol, T.untyped]).returns(T.nilable(Message)) }
       def self.create_from_hash(data)
         role_str = data[:role]&.to_s
-        content = data[:content]&.to_s
+        content = data[:content]
         
         return nil if role_str.nil? || content.nil?
         
+        # Handle both string and array content
+        formatted_content = if content.is_a?(Array)
+          content
+        else
+          content.to_s
+        end
+        
         # Convert string role to enum
         role = case role_str
                when 'system' then Message::Role::System
@@ -87,7 +151,7 @@ module DSPy
         
         Message.new(
           role: role,
-          content: content,
+          content: formatted_content,
           name: data[:name]&.to_s
         )
       rescue => e

data/lib/dspy/lm/message_builder.rb

@@ -41,6 +41,34 @@ module DSPy
         self
       end
       
+      sig { params(text: String, image: DSPy::Image).returns(MessageBuilder) }
+      def user_with_image(text, image)
+        content_array = [
+          { type: 'text', text: text },
+          { type: 'image', image: image }
+        ]
+        
+        @messages << Message.new(
+          role: Message::Role::User,
+          content: content_array
+        )
+        self
+      end
+      
+      sig { params(text: String, images: T::Array[DSPy::Image]).returns(MessageBuilder) }
+      def user_with_images(text, images)
+        content_array = [{ type: 'text', text: text }]
+        images.each do |image|
+          content_array << { type: 'image', image: image }
+        end
+        
+        @messages << Message.new(
+          role: Message::Role::User,
+          content: content_array
+        )
+        self
+      end
+      
       # For backward compatibility, allow conversion to hash array
       sig { returns(T::Array[T::Hash[Symbol, T.untyped]]) }
       def to_h

data/lib/dspy/lm/vision_models.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module DSPy
+  class LM
+    module VisionModels
+      # OpenAI vision-capable models
+      OPENAI_VISION_MODELS = [
+        'gpt-4-vision-preview',
+        'gpt-4-turbo',
+        'gpt-4-turbo-2024-04-09',
+        'gpt-4-turbo-preview',
+        'gpt-4o',
+        'gpt-4o-2024-05-13',
+        'gpt-4o-2024-08-06',
+        'gpt-4o-mini',
+        'gpt-4o-mini-2024-07-18'
+      ].freeze
+      
+      # Anthropic vision-capable models
+      ANTHROPIC_VISION_MODELS = [
+        'claude-3-opus-20240229',
+        'claude-3-sonnet-20240229',
+        'claude-3-haiku-20240307',
+        'claude-3-5-sonnet-20241022',
+        'claude-3-5-sonnet-20240620',
+        'claude-3-5-haiku-20241022'
+      ].freeze
+      
+      def self.supports_vision?(provider, model)
+        case provider.to_s.downcase
+        when 'openai'
+          OPENAI_VISION_MODELS.any? { |m| model.include?(m) }
+        when 'anthropic'
+          ANTHROPIC_VISION_MODELS.any? { |m| model.include?(m) }
+        else
+          false
+        end
+      end
+      
+      def self.validate_vision_support!(provider, model)
+        unless supports_vision?(provider, model)
+          raise ArgumentError, "Model #{model} does not support vision. Vision-capable models for #{provider}: #{vision_models_for(provider).join(', ')}"
+        end
+      end
+      
+      def self.vision_models_for(provider)
+        case provider.to_s.downcase
+        when 'openai'
+          OPENAI_VISION_MODELS
+        when 'anthropic'
+          ANTHROPIC_VISION_MODELS
+        else
+          []
+        end
+      end
+    end
+  end
+end

data/lib/dspy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DSPy
-  VERSION = "0.15.7"
+  VERSION = "0.16.0"
 end

data/lib/dspy.rb

@@ -124,6 +124,7 @@ require_relative 'dspy/few_shot_example'
 require_relative 'dspy/prompt'
 require_relative 'dspy/example'
 require_relative 'dspy/lm'
+require_relative 'dspy/image'
 require_relative 'dspy/strategy'
 require_relative 'dspy/prediction'
 require_relative 'dspy/predict'

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: dspy
 version: !ruby/object:Gem::Version
-  version: 0.15.7
+  version: 0.16.0
 platform: ruby
 authors:
 - Vicente Reig Rincón de Arellano
 bindir: bin
 cert_chain: []
-date: 2025-08-07 00:00:00.000000000 Z
+date: 2025-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -165,6 +165,7 @@ files:
 - lib/dspy/example.rb
 - lib/dspy/few_shot_example.rb
 - lib/dspy/field.rb
+- lib/dspy/image.rb
 - lib/dspy/instrumentation.rb
 - lib/dspy/instrumentation/event_payload_factory.rb
 - lib/dspy/instrumentation/event_payloads.rb
@@ -190,6 +191,7 @@ files:
 - lib/dspy/lm/strategy_selector.rb
 - lib/dspy/lm/structured_output_strategy.rb
 - lib/dspy/lm/usage.rb
+- lib/dspy/lm/vision_models.rb
 - lib/dspy/memory.rb
 - lib/dspy/memory/embedding_engine.rb
 - lib/dspy/memory/in_memory_store.rb