dspy 0.34.4 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d5296e0130d0550156345659e6703451bd6ca6fb9ffbac33d8584ee203f2a84
-  data.tar.gz: 339772eb768a2babbb8b700b868fc772183473b7cf4f8a6867e55945f75a3655
+  metadata.gz: 85ae2299dbddc20bfd710c68e9e8dfbeeb201b742e3fd18b768669ddc4443261
+  data.tar.gz: ba0ee2d5f637f499448e456fb58f0513aa58b5b258e754bf1b32839ea8c49b63
 SHA512:
-  metadata.gz: 827360cba1ad8d03373d40d9b1b9ce3acf966e896261b918537150568cf4635904e591c238cdea7d8e8bf45980851ce3c581f709639215c8d7e4c7e1ea78dc08
-  data.tar.gz: e8d91df3e7204ac0d0db830c5839a4cbdd9683a00be2a77dec38de156052f0a0b3956e25c8b7026fcce23880e9806fc9af023fc2a4fe830926f288f82474a85b
+  metadata.gz: 310de5ce11b23d29bd168069eae4f5ce3ac154ba91974c96b9d70e5d02a0dcb45122dc4c83f80097a409ff448a0a30d45ebec3ff0883d80b2cdd8f1215d2dfff
+  data.tar.gz: 1ce5ff1bfe900bcedabab28efba1e8352fed81c6cbf458d5b7c54e8f7eea29fa6b7ee33a62dcc73b456ede6181a85bbf055541e4fb70037ac4de05e3be2b8ce9

data/README.md

@@ -9,6 +9,7 @@
 **Build reliable LLM applications in idiomatic Ruby using composable, type-safe modules.**
 
 DSPy.rb is the Ruby port of Stanford's [DSPy](https://dspy.ai). Instead of wrestling with brittle prompt strings, you define typed signatures and let the framework handle the rest. Prompts become functions. LLM calls become predictable.
+The `1.x` line is the stable release track for production Ruby LLM applications.
 
 ```ruby
 require 'dspy'

data/lib/dspy/document.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require 'base64'
+require 'stringio'
+require 'uri'
+
+module DSPy
+  class Document
+    class RubyLLMInlineAttachment < StringIO
+      attr_reader :path
+
+      def initialize(content, path:)
+        super(content)
+        @path = path
+        binmode
+      end
+    end
+
+    private_constant :RubyLLMInlineAttachment
+
+    attr_reader :url, :base64, :data, :content_type
+
+    SUPPORTED_FORMATS = %w[application/pdf].freeze
+    MAX_SIZE_BYTES = 32 * 1024 * 1024 # 32MB limit
+
+    def initialize(url: nil, base64: nil, data: nil, content_type: nil)
+      validate_input!(url, base64, data)
+
+      if url
+        @url = url
+        @content_type = content_type || infer_content_type_from_url(url)
+      elsif base64
+        raise ArgumentError, "content_type is required when using base64" unless content_type
+
+        @base64 = base64
+        @content_type = content_type
+        validate_size!(Base64.decode64(base64).bytesize)
+      elsif data
+        raise ArgumentError, "content_type is required when using data" unless content_type
+
+        @data = data
+        @content_type = content_type
+        validate_size!(data.size)
+      end
+
+      validate_content_type!
+    end
+
+    def to_openai_format
+      raise DSPy::LM::IncompatibleDocumentFeatureError,
+            "OpenAI document inputs are not supported in this release. Use Anthropic directly or Anthropic via RubyLLM."
+    end
+
+    def to_anthropic_format
+      if url
+        {
+          type: 'document',
+          source: {
+            type: 'url',
+            url: url
+          }
+        }
+      else
+        {
+          type: 'document',
+          source: {
+            type: 'base64',
+            media_type: content_type,
+            data: to_base64
+          }
+        }
+      end
+    end
+
+    def to_gemini_format
+      raise DSPy::LM::IncompatibleDocumentFeatureError,
+            "Gemini document inputs are not supported in this release. Use Anthropic directly or Anthropic via RubyLLM."
+    end
+
+    def to_ruby_llm_attachment
+      if url
+        url
+      else
+        RubyLLMInlineAttachment.new(to_binary, path: 'document.pdf')
+      end
+    end
+
+    def to_base64
+      return base64 if base64
+      return Base64.strict_encode64(data.pack('C*')) if data
+
+      nil
+    end
+
+    def validate_for_provider!(provider)
+      case provider
+      when 'anthropic'
+        true
+      when 'openai'
+        raise DSPy::LM::IncompatibleDocumentFeatureError,
+              "OpenAI document inputs are not supported in this release. Use Anthropic directly or Anthropic via RubyLLM."
+      when 'gemini'
+        raise DSPy::LM::IncompatibleDocumentFeatureError,
+              "Gemini document inputs are not supported in this release. Use Anthropic directly or Anthropic via RubyLLM."
+      else
+        raise DSPy::LM::IncompatibleDocumentFeatureError,
+              "Unknown provider '#{provider}'. Document inputs are currently supported only for Anthropic."
+      end
+    end
+
+    private
+
+    def validate_input!(url, base64, data)
+      inputs = [url, base64, data].compact
+
+      if inputs.empty?
+        raise ArgumentError, "Must provide either url, base64, or data"
+      elsif inputs.size > 1
+        raise ArgumentError, "Only one of url, base64, or data can be provided"
+      end
+    end
+
+    def validate_content_type!
+      unless SUPPORTED_FORMATS.include?(content_type)
+        raise ArgumentError, "Unsupported document format: #{content_type}. Supported formats: #{SUPPORTED_FORMATS.join(', ')}"
+      end
+    end
+
+    def validate_size!(size_bytes)
+      if size_bytes > MAX_SIZE_BYTES
+        raise ArgumentError, "Document size exceeds 32MB limit (got #{size_bytes} bytes)"
+      end
+    end
+
+    def infer_content_type_from_url(url)
+      extension = File.extname(URI.parse(url).path).downcase
+
+      case extension
+      when '.pdf'
+        'application/pdf'
+      else
+        raise ArgumentError, "Document URL must point to a PDF (.pdf): #{url}"
+      end
+    end
+
+    def to_binary
+      return Base64.decode64(base64) if base64
+      return data.pack('C*') if data
+
+      raise ArgumentError, "Document has no binary content"
+    end
+  end
+end

data/lib/dspy/lm/adapter.rb

@@ -58,6 +58,17 @@ module DSPy
         end
       end
 
+      def contains_documents?(messages)
+        messages.any? do |msg|
+          content = msg[:content] || msg.content
+          content.is_a?(Array) && content.any? { |item| item[:type] == 'document' }
+        end
+      end
+
+      def contains_media?(messages)
+        contains_images?(messages) || contains_documents?(messages)
+      end
+
       # Format multimodal messages for a specific provider
       # @param messages [Array<Hash>] Array of message hashes
       # @param provider_name [String] Provider name for image validation and formatting
@@ -71,6 +82,8 @@ module DSPy
                 { type: 'text', text: item[:text] }
               when 'image'
                 format_image_for_provider(item[:image], provider_name)
+              when 'document'
+                format_document_for_provider(item[:document], provider_name)
               else
                 item
               end
@@ -96,6 +109,16 @@ module DSPy
           { type: 'image', image: image }
         end
       end
+
+      def format_document_for_provider(document, provider_name)
+        document.validate_for_provider!(provider_name)
+        format_method = "to_#{provider_name}_format"
+        if document.respond_to?(format_method)
+          document.send(format_method)
+        else
+          { type: 'document', document: document }
+        end
+      end
     end
   end
 end

data/lib/dspy/lm/errors.rb

@@ -7,8 +7,6 @@ module DSPy
     class UnsupportedProviderError < Error; end
     class ConfigurationError < Error; end
     class MissingAdapterError < Error; end
-    class UnsupportedVersionError < Error; end
-    class MissingOfficialSDKError < Error; end
     
     # Raised when API key is missing or invalid
     class MissingAPIKeyError < Error
@@ -29,5 +27,12 @@ module DSPy
         super(message)
       end
     end
+
+    # Raised when document features are incompatible with the target provider
+    class IncompatibleDocumentFeatureError < AdapterError
+      def initialize(message)
+        super(message)
+      end
+    end
   end
 end

data/lib/dspy/lm/json_strategy.rb

@@ -136,59 +136,96 @@ module DSPy
       def extract_json_from_content(content)
         return content if content.nil? || content.empty?
 
-        # Fix Anthropic Beta API bug with optional fields producing invalid JSON
-        # When some output fields are optional and not returned, Anthropic's structured outputs
-        # can produce trailing comma+brace: {"field1": {...},} instead of {"field1": {...}}
-        # This workaround removes the invalid trailing syntax before JSON parsing
-        if content =~ /,\s*\}\s*$/
-          content = content.sub(/,(\s*\}\s*)$/, '\1')
-        end
-
         # Try 1: Check for ```json code block (with or without preceding text)
         if content.include?('```json')
           json_match = content.match(/```json\s*\n(.*?)\n```/m)
-          return json_match[1].strip if json_match
+          if json_match
+            normalized = normalize_json_candidate(json_match[1].strip)
+            return normalized if valid_json?(normalized)
+          end
         end
 
         # Try 2: Check for generic ``` code block
         if content.include?('```')
           code_match = content.match(/```\s*\n(.*?)\n```/m)
           if code_match
-            potential_json = code_match[1].strip
-            # Verify it's JSON
-            begin
-              JSON.parse(potential_json)
-              return potential_json
-            rescue JSON::ParserError
-              # Not valid JSON, continue
-            end
+            potential_json = normalize_json_candidate(code_match[1].strip)
+            return potential_json if valid_json?(potential_json)
           end
         end
 
         # Try 3: Try parsing entire content as JSON
-        begin
-          JSON.parse(content)
-          return content
-        rescue JSON::ParserError
-          # Not pure JSON, try extracting
-        end
+        normalized_content = normalize_json_candidate(content)
+        return normalized_content if valid_json?(normalized_content)
 
         # Try 4: Look for JSON object pattern in text (greedy match for nested objects)
         json_pattern = /\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\}/m
         json_match = content.match(json_pattern)
         if json_match
-          potential_json = json_match[0]
-          begin
-            JSON.parse(potential_json)
-            return potential_json
-          rescue JSON::ParserError
-            # Not valid JSON
-          end
+          potential_json = normalize_json_candidate(json_match[0])
+          return potential_json if valid_json?(potential_json)
         end
 
         # Return content as-is if no JSON found
         content
       end
+
+      sig { params(content: String).returns(String) }
+      def normalize_json_candidate(content)
+        escape_control_characters_in_strings(remove_trailing_object_commas(content))
+      end
+
+      sig { params(content: String).returns(String) }
+      def remove_trailing_object_commas(content)
+        content.sub(/,(\s*\}\s*)$/, '\1')
+      end
+
+      sig { params(content: String).returns(T::Boolean) }
+      def valid_json?(content)
+        JSON.parse(content)
+        true
+      rescue JSON::ParserError
+        false
+      end
+
+      sig { params(content: String).returns(String) }
+      def escape_control_characters_in_strings(content)
+        escaped = +""
+        in_string = false
+        escaping = false
+
+        content.each_char do |char|
+          if in_string
+            if escaping
+              escaped << char
+              escaping = false
+              next
+            end
+
+            case char
+            when '\\'
+              escaped << char
+              escaping = true
+            when '"'
+              escaped << char
+              in_string = false
+            when "\n"
+              escaped << '\n'
+            when "\r"
+              escaped << '\r'
+            when "\t"
+              escaped << '\t'
+            else
+              escaped << (char.ord < 0x20 ? "" : char)
+            end
+          else
+            escaped << char
+            in_string = true if char == '"'
+          end
+        end
+
+        escaped
+      end
     end
   end
 end

data/lib/dspy/lm/message.rb

@@ -59,6 +59,8 @@ module DSPy
               { type: 'text', text: item[:text] }
             when 'image'
               item[:image].to_openai_format
+            when 'document'
+              item[:document].to_openai_format
             else
               item
             end
@@ -83,6 +85,8 @@ module DSPy
               { type: 'text', text: item[:text] }
             when 'image'
               item[:image].to_anthropic_format
+            when 'document'
+              item[:document].to_anthropic_format
             else
               item
             end
@@ -160,4 +164,4 @@ module DSPy
       end
     end
   end
-end
+end

data/lib/dspy/lm/message_builder.rb

@@ -68,6 +68,20 @@ module DSPy
         )
         self
       end
+
+      sig { params(text: String, document: DSPy::Document).returns(MessageBuilder) }
+      def user_with_document(text, document)
+        content_array = [
+          { type: 'text', text: text },
+          { type: 'document', document: document }
+        ]
+
+        @messages << Message.new(
+          role: Message::Role::User,
+          content: content_array
+        )
+        self
+      end
       
       # For backward compatibility, allow conversion to hash array
       sig { returns(T::Array[T::Hash[Symbol, T.untyped]]) }
@@ -76,4 +90,4 @@ module DSPy
       end
     end
   end
-end
+end

data/lib/dspy/lm.rb

@@ -161,16 +161,78 @@ module DSPy
         )
       end
 
-      # Add user message
-      user_prompt = prompt.render_user_prompt(input_values)
-      messages << Message.new(
-        role: Message::Role::User,
-        content: user_prompt
-      )
+      document_inputs = extract_document_inputs(input_values)
+
+      if document_inputs.empty?
+        user_prompt = prompt.render_user_prompt(input_values)
+        messages << Message.new(
+          role: Message::Role::User,
+          content: user_prompt
+        )
+      else
+        validate_document_predict_support!(input_values, document_inputs)
+
+        placeholder_inputs = input_values.transform_values do |value|
+          value.is_a?(DSPy::Document) ? "[attached pdf document]" : value
+        end
+
+        user_prompt = prompt.render_user_prompt(placeholder_inputs)
+        content_array = [
+          { type: 'text', text: user_prompt },
+          { type: 'document', document: document_inputs.first.last }
+        ]
+
+        messages << Message.new(
+          role: Message::Role::User,
+          content: content_array
+        )
+      end
 
       messages
     end
 
+    def extract_document_inputs(input_values)
+      input_values.each_with_object([]) do |(key, value), inputs|
+        if value.is_a?(DSPy::Document)
+          inputs << [key, value]
+        elsif nested_document_input?(value)
+          raise DSPy::LM::IncompatibleDocumentFeatureError,
+                "Only one top-level DSPy::Document input is currently supported in Predict."
+        end
+      end
+    end
+
+    def nested_document_input?(value)
+      case value
+      when T::Struct
+        value.class.props.any? { |name, _| nested_document_input?(value.public_send(name)) }
+      when Array
+        value.any? { |item| nested_document_input?(item) }
+      when Hash
+        value.values.any? { |item| nested_document_input?(item) }
+      else
+        value.is_a?(DSPy::Document)
+      end
+    end
+
+    def validate_document_predict_support!(input_values, document_inputs)
+      if document_inputs.length > 1
+        raise DSPy::LM::IncompatibleDocumentFeatureError,
+              "Only one top-level DSPy::Document input is currently supported in Predict."
+      end
+
+      if input_values.values.any? { |value| value.is_a?(DSPy::Image) }
+        raise DSPy::LM::IncompatibleDocumentFeatureError,
+              "Predict does not support mixing DSPy::Document and DSPy::Image inputs in this release."
+      end
+
+      return if provider == 'anthropic'
+      return if adapter.class.name.include?('RubyLLMAdapter') && adapter.provider == 'anthropic'
+
+      raise DSPy::LM::IncompatibleDocumentFeatureError,
+            "Document inputs are currently supported only for Anthropic models and Anthropic via RubyLLM."
+    end
+
     def will_use_structured_outputs?(signature_class, data_format: nil)
       return false unless signature_class
       return false if data_format == :toon

data/lib/dspy/ruby_llm/lm/adapters/ruby_llm_adapter.rb

@@ -5,9 +5,6 @@ require 'ruby_llm'
 require 'dspy/lm/adapter'
 require 'dspy/lm/vision_models'
 
-require 'dspy/ruby_llm/guardrails'
-DSPy::RubyLLM::Guardrails.ensure_ruby_llm_installed!
-
 module DSPy
   module RubyLLM
     module LM
@@ -49,6 +46,8 @@ module DSPy
           def chat(messages:, signature: nil, &block)
             normalized_messages = normalize_messages(messages)
 
+            validate_document_support!(normalized_messages)
+
             # Validate vision support if images are present
             if contains_images?(normalized_messages)
               validate_vision_support!
@@ -255,6 +254,9 @@ module DSPy
                   elsif item[:image_url]
                     attachments << item[:image_url][:url]
                   end
+                when 'document'
+                  document = item[:document]
+                  attachments << document.to_ruby_llm_attachment if document
                 end
               end
               content = text_parts.join("\n")
@@ -263,6 +265,14 @@ module DSPy
             [content.to_s, attachments]
           end
 
+          def validate_document_support!(messages)
+            return unless contains_documents?(messages)
+            return if provider == 'anthropic'
+
+            raise DSPy::LM::IncompatibleDocumentFeatureError,
+                  "RubyLLM document inputs are currently supported only when the underlying provider is Anthropic."
+          end
+
           def map_response(ruby_llm_response)
             DSPy::LM::Response.new(
               content: ruby_llm_response.content.to_s,

data/lib/dspy/ruby_llm/version.rb

@@ -2,6 +2,6 @@
 
 module DSPy
   module RubyLLM
-    VERSION = '0.1.0'
+    VERSION = '0.1.1'
   end
 end

data/lib/dspy/ruby_llm.rb

@@ -2,7 +2,4 @@
 
 require 'dspy/ruby_llm/version'
 
-require 'dspy/ruby_llm/guardrails'
-DSPy::RubyLLM::Guardrails.ensure_ruby_llm_installed!
-
 require 'dspy/ruby_llm/lm/adapters/ruby_llm_adapter'

data/lib/dspy/support/openai_sdk_warning.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module DSPy
+  module Support
+    module OpenAISDKWarning
+      WARNING_MESSAGE = <<~WARNING.freeze
+        WARNING: ruby-openai gem detected. This may cause conflicts with DSPy's OpenAI integration.
+
+        DSPy uses the official 'openai' gem. The community 'ruby-openai' gem uses the same
+        OpenAI namespace and will cause conflicts.
+
+        To fix this, remove 'ruby-openai' from your Gemfile and use the official gem instead:
+        - Remove: gem 'ruby-openai'
+        - Keep: gem 'openai' (official SDK that DSPy uses)
+
+        The official gem provides better compatibility and is actively maintained by OpenAI.
+      WARNING
+
+      def self.warn_if_community_gem_loaded!
+        return if @warned
+        return unless community_gem_loaded?
+
+        Kernel.warn WARNING_MESSAGE
+        @warned = true
+      end
+
+      def self.community_gem_loaded?
+        defined?(::OpenAI) && defined?(::OpenAI::Client) && !defined?(::OpenAI::Internal)
+      end
+    end
+  end
+end

data/lib/dspy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DSPy
-  VERSION = "0.34.4"
+  VERSION = "1.0.0"
 end

data/lib/dspy.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 require_relative 'dspy/support/warning_filters'
+require_relative 'dspy/support/openai_sdk_warning'
 require 'sorbet-runtime'
 require 'dry-configurable'
 require 'dry/logger'
@@ -267,6 +268,7 @@ require_relative 'dspy/prompt'
 require_relative 'dspy/example'
 require_relative 'dspy/lm'
 require_relative 'dspy/image'
+require_relative 'dspy/document'
 require_relative 'dspy/prediction'
 require_relative 'dspy/predict'
 require_relative 'dspy/chain_of_thought'
@@ -309,19 +311,4 @@ DSPy::Observability.configure!
 
 # LoggerSubscriber will be lazy-initialized when first accessed
 
-# Detect potential gem conflicts and warn users
-# DSPy uses the official openai gem, warn if ruby-openai (community version) is detected
-if defined?(OpenAI) && defined?(OpenAI::Client) && !defined?(OpenAI::Internal)
-  warn <<~WARNING
-    WARNING: ruby-openai gem detected. This may cause conflicts with DSPy's OpenAI integration.
-    
-    DSPy uses the official 'openai' gem. The community 'ruby-openai' gem uses the same
-    OpenAI namespace and will cause conflicts.
-    
-    To fix this, remove 'ruby-openai' from your Gemfile and use the official gem instead:
-    - Remove: gem 'ruby-openai'
-    - Keep: gem 'openai' (official SDK that DSPy uses)
-    
-    The official gem provides better compatibility and is actively maintained by OpenAI.
-  WARNING
-end
+DSPy::Support::OpenAISDKWarning.warn_if_community_gem_loaded!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dspy
 version: !ruby/object:Gem::Version
-  version: 0.34.4
+  version: 1.0.0
 platform: ruby
 authors:
 - Vicente Reig Rincón de Arellano
@@ -157,6 +157,7 @@ files:
 - lib/dspy/callbacks.rb
 - lib/dspy/chain_of_thought.rb
 - lib/dspy/context.rb
+- lib/dspy/document.rb
 - lib/dspy/error_formatter.rb
 - lib/dspy/errors.rb
 - lib/dspy/evals.rb
@@ -194,7 +195,6 @@ files:
 - lib/dspy/registry/registry_manager.rb
 - lib/dspy/registry/signature_registry.rb
 - lib/dspy/ruby_llm.rb
-- lib/dspy/ruby_llm/guardrails.rb
 - lib/dspy/ruby_llm/lm/adapters/ruby_llm_adapter.rb
 - lib/dspy/ruby_llm/version.rb
 - lib/dspy/schema.rb
@@ -210,6 +210,7 @@ files:
 - lib/dspy/storage/program_storage.rb
 - lib/dspy/storage/storage_manager.rb
 - lib/dspy/structured_outputs_prompt.rb
+- lib/dspy/support/openai_sdk_warning.rb
 - lib/dspy/support/warning_filters.rb
 - lib/dspy/teleprompt/bootstrap_strategy.rb
 - lib/dspy/teleprompt/data_handler.rb

data/lib/dspy/ruby_llm/guardrails.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-require 'dspy/lm/errors'
-
-module DSPy
-  module RubyLLM
-    class Guardrails
-      SUPPORTED_RUBY_LLM_VERSIONS = "~> 1.3".freeze
-
-      def self.ensure_ruby_llm_installed!
-        require 'ruby_llm'
-
-        spec = Gem.loaded_specs["ruby_llm"]
-        unless spec && Gem::Requirement.new(SUPPORTED_RUBY_LLM_VERSIONS).satisfied_by?(spec.version)
-          msg = <<~MSG
-            DSPy requires the `ruby_llm` gem #{SUPPORTED_RUBY_LLM_VERSIONS}.
-            Please install or upgrade it with `bundle add ruby_llm --version "#{SUPPORTED_RUBY_LLM_VERSIONS}"`.
-          MSG
-          raise DSPy::LM::UnsupportedVersionError, msg
-        end
-      end
-    end
-  end
-end