dspy 0.28.1 → 0.28.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b377060443eeb9c3c5d975e76750d6c519d1b93cf6f20dc6ad20bcda08d1ca4
-  data.tar.gz: 4580845b3fd9991b531c8c2bd809595cbd3328da57a23e58307cdbe52e3822bc
+  metadata.gz: f1cc0ac1e2e1dc27f6255b11ca70ff0ad0eb37b5ae50ff38b97ae7d35b7b69b8
+  data.tar.gz: 2399987757b4f037080632e646714e785328fbe3c1fd39138fe750336cdd7710
 SHA512:
-  metadata.gz: 444a5e08364b2e996bf49d230cb9a94a1930e26d2ea796cd0104ea4888b45ade4099cec502b92c08752631adff47b1d9d1897da9209e9faabbb28fb6761935b0
-  data.tar.gz: c1b3a83482c861923304c463d6f0b1c9042fbc31da920526ec18ed0f5148b4408a9023d312eded9263ca2d706e4779b78526e28dc50a17458cbbac9afa56b024
+  metadata.gz: ea48762186d3de89a005e8eac27f57ca90b294c4ab096ec1c7985d06396216d719ca3a2144406d1f43af472a560670e502329a785d33b8923b5c9c4c0f83dfd1
+  data.tar.gz: 98fee1468f2692a0f7cc87622722f945dbc344fb299e71935ef592bc1d59e68e48e1390208981b6263998627dacb38dc325727cc191f81e1db79fff57c1537a4

data/lib/dspy/callbacks.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module DSPy
+  # Provides Rails-style callback hooks for DSPy modules
+  #
+  # @example Define callbacks in base class
+  #   class DSPy::Module
+  #     include DSPy::Callbacks
+  #
+  #     create_before_callback :forward
+  #     create_after_callback :forward
+  #     create_around_callback :forward
+  #   end
+  #
+  # @example Use callbacks in subclasses
+  #   class MyAgent < DSPy::Module
+  #     before :setup_context
+  #     after :log_metrics
+  #     around :manage_memory
+  #
+  #     private
+  #
+  #     def setup_context
+  #       @start_time = Time.now
+  #     end
+  #
+  #     def log_metrics
+  #       puts "Duration: #{Time.now - @start_time}"
+  #     end
+  #
+  #     def manage_memory
+  #       load_context
+  #       yield
+  #       save_context
+  #     end
+  #   end
+  module Callbacks
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      # Creates a before callback hook for the specified method
+      #
+      # @param method_name [Symbol] the method to add callback support to
+      def create_before_callback(method_name)
+        mark_method_has_callbacks(method_name)
+        ensure_callback_method_defined(:before, method_name)
+        wrap_method_with_callbacks(method_name)
+      end
+
+      # Creates an after callback hook for the specified method
+      #
+      # @param method_name [Symbol] the method to add callback support to
+      def create_after_callback(method_name)
+        mark_method_has_callbacks(method_name)
+        ensure_callback_method_defined(:after, method_name)
+        wrap_method_with_callbacks(method_name)
+      end
+
+      # Creates an around callback hook for the specified method
+      #
+      # @param method_name [Symbol] the method to add callback support to
+      def create_around_callback(method_name)
+        mark_method_has_callbacks(method_name)
+        ensure_callback_method_defined(:around, method_name)
+        wrap_method_with_callbacks(method_name)
+      end
+
+      private
+
+      # Ensures the callback registration method exists
+      def ensure_callback_method_defined(type, target_method_name)
+        return if singleton_class.method_defined?(type)
+
+        define_singleton_method(type) do |callback_method|
+          register_callback(type, target_method_name, callback_method)
+        end
+      end
+
+      # Registers a callback for execution
+      def register_callback(type, method_name, callback_method)
+        own_callbacks_for(method_name)[type] ||= []
+        own_callbacks_for(method_name)[type] << callback_method
+      end
+
+      # Returns own callbacks (not including parent)
+      def own_callbacks_for(method_name)
+        @callbacks ||= {}
+        @callbacks[method_name] ||= {}
+      end
+
+      # Marks that a method has callback support (even if no callbacks registered yet)
+      def mark_method_has_callbacks(method_name)
+        own_callbacks_for(method_name)
+      end
+
+      # Returns the callback registry for a method
+      # Includes callbacks from parent classes
+      def callbacks_for(method_name)
+        own_callbacks = own_callbacks_for(method_name)
+
+        # Merge parent callbacks if this is a subclass
+        if superclass.respond_to?(:callbacks_for, true)
+          parent_callbacks = superclass.send(:callbacks_for, method_name)
+
+          # Merge each callback type, with own callbacks coming after parent callbacks
+          merged_callbacks = {}
+          [:before, :after, :around].each do |type|
+            parent_list = parent_callbacks[type] || []
+            own_list = own_callbacks[type] || []
+            merged_callbacks[type] = parent_list + own_list if parent_list.any? || own_list.any?
+          end
+
+          merged_callbacks
+        else
+          own_callbacks
+        end
+      end
+
+      # Wraps a method with callback execution logic
+      def wrap_method_with_callbacks(method_name)
+        return if method_wrapped?(method_name)
+
+        # Defer wrapping if method doesn't exist yet
+        return unless method_defined?(method_name)
+
+        # Mark as wrapped BEFORE define_method to prevent infinite recursion
+        mark_method_wrapped(method_name)
+
+        original_method = instance_method(method_name)
+
+        define_method(method_name) do |*args, **kwargs, &block|
+          # Execute before callbacks
+          run_callbacks(:before, method_name)
+
+          # Execute around callbacks or original method
+          result = if self.class.send(:has_around_callbacks?, method_name)
+            execute_with_around_callbacks(method_name, original_method, *args, **kwargs, &block)
+          else
+            original_method.bind(self).call(*args, **kwargs, &block)
+          end
+
+          # Execute after callbacks
+          run_callbacks(:after, method_name)
+
+          result
+        end
+      end
+
+      # Checks if method has around callbacks
+      def has_around_callbacks?(method_name)
+        callbacks_for(method_name)[:around]&.any?
+      end
+
+      # Hook into method_added to wrap methods when they're defined
+      def method_added(method_name)
+        super
+
+        # Check if this method or any parent has callback support (even if no callbacks registered yet)
+        has_callback_support = method_has_callback_support?(method_name)
+
+        return unless has_callback_support
+        return if method_wrapped?(method_name)
+
+        wrap_method_with_callbacks(method_name)
+      end
+
+      # Checks if a method has callback support in this class or parents
+      def method_has_callback_support?(method_name)
+        # Check own callbacks registry
+        return true if @callbacks&.key?(method_name)
+
+        # Check parent class
+        if superclass.respond_to?(:method_has_callback_support?, true)
+          superclass.send(:method_has_callback_support?, method_name)
+        else
+          false
+        end
+      end
+
+      # Marks a method as wrapped
+      def mark_method_wrapped(method_name)
+        @wrapped_methods ||= []
+        @wrapped_methods << method_name
+      end
+
+      # Checks if method is already wrapped
+      def method_wrapped?(method_name)
+        @wrapped_methods&.include?(method_name)
+      end
+    end
+
+    private
+
+    # Executes callbacks of a specific type
+    def run_callbacks(type, method_name)
+      callbacks = self.class.send(:callbacks_for, method_name)[type]
+      return unless callbacks
+
+      callbacks.each do |callback_method|
+        send(callback_method)
+      end
+    end
+
+    # Executes method with around callbacks
+    def execute_with_around_callbacks(method_name, original_method, *args, **kwargs, &block)
+      callbacks = self.class.send(:callbacks_for, method_name)[:around]
+
+      # Build callback chain from innermost (original method) to outermost
+      chain = callbacks.reverse.inject(
+        -> { original_method.bind(self).call(*args, **kwargs, &block) }
+      ) do |inner, callback_method|
+        -> { send(callback_method) { inner.call } }
+      end
+
+      chain.call
+    end
+  end
+end

data/lib/dspy/chain_of_thought.rb

@@ -46,7 +46,8 @@ module DSPy
         input_schema: @signature_class.input_json_schema,
         output_schema: @signature_class.output_json_schema,
         few_shot_examples: new_prompt.few_shot_examples,
-        signature_class_name: @signature_class.name
+        signature_class_name: @signature_class.name,
+        schema_format: new_prompt.schema_format
       )
       
       instance.instance_variable_set(:@prompt, enhanced_prompt)

data/lib/dspy/lm.rb

@@ -31,15 +31,16 @@ require_relative 'structured_outputs_prompt'
 module DSPy
   class LM
     extend T::Sig
-    attr_reader :model_id, :api_key, :model, :provider, :adapter
+    attr_reader :model_id, :api_key, :model, :provider, :adapter, :schema_format
 
-    def initialize(model_id, api_key: nil, **options)
+    def initialize(model_id, api_key: nil, schema_format: :json, **options)
       @model_id = model_id
       @api_key = api_key
-      
+      @schema_format = schema_format
+
       # Parse provider and model from model_id
       @provider, @model = parse_model_id(model_id)
-      
+
       # Create appropriate adapter with options
       @adapter = AdapterFactory.create(model_id, api_key: api_key, **options)
     end

data/lib/dspy/module.rb

@@ -3,16 +3,23 @@
 require 'sorbet-runtime'
 require 'dry-configurable'
 require_relative 'context'
+require_relative 'callbacks'
 
 module DSPy
   class Module
     extend T::Sig
     extend T::Generic
     include Dry::Configurable
+    include DSPy::Callbacks
 
     # Per-instance LM configuration
     setting :lm, default: nil
 
+    # Define callback hooks for forward method
+    create_before_callback :forward
+    create_after_callback :forward
+    create_around_callback :forward
+
     # The main forward method that users will call is generic and type parameterized
     sig do
       type_parameters(:I, :O)
@@ -72,5 +79,31 @@ module DSPy
     def lm
       config.lm || DSPy.current_lm
     end
+
+    # Save the module state to a JSON file
+    # Lightweight serialization for intermediate optimization trials
+    #
+    # @param path [String] Path to save the module state (JSON format)
+    sig { params(path: String).void }
+    def save(path)
+      require 'json'
+      require 'fileutils'
+
+      # Ensure parent directory exists
+      dir = File.dirname(path)
+      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+
+      # Serialize module to JSON
+      File.write(path, JSON.pretty_generate(to_h))
+    end
+
+    # Default serialization method - subclasses can override
+    sig { returns(T::Hash[Symbol, T.untyped]) }
+    def to_h
+      {
+        class_name: self.class.name,
+        state: {}
+      }
+    end
   end
 end

data/lib/dspy/predict.rb

@@ -53,11 +53,18 @@ module DSPy
     sig { returns(Prompt) }
     attr_reader :prompt
 
+    # Mutable demos attribute for MIPROv2 compatibility
+    sig { returns(T.nilable(T::Array[FewShotExample])) }
+    attr_accessor :demos
+
     sig { params(signature_class: T.class_of(Signature)).void }
     def initialize(signature_class)
       super()
       @signature_class = signature_class
+
+      # Prompt will read schema_format from config automatically
       @prompt = Prompt.from_signature(signature_class)
+      @demos = nil
     end
 
     # Reconstruct program from serialized hash

data/lib/dspy/prompt.rb

@@ -22,21 +22,39 @@ module DSPy
     sig { returns(T.nilable(String)) }
     attr_reader :signature_class_name
 
+    # Returns the effective schema format
+    # Precedence: instance variable (if not :json default) > config.lm > :json
+    sig { returns(Symbol) }
+    def schema_format
+      # If @schema_format was explicitly set to something other than :json, respect it
+      return @schema_format if @schema_format && @schema_format != :json
+
+      # Otherwise, read from config if available
+      DSPy.config.lm&.schema_format || @schema_format || :json
+    end
+
+    sig { returns(T.nilable(T.class_of(Signature))) }
+    attr_reader :signature_class
+
     sig do
       params(
         instruction: String,
         input_schema: T::Hash[Symbol, T.untyped],
         output_schema: T::Hash[Symbol, T.untyped],
         few_shot_examples: T::Array[FewShotExample],
-        signature_class_name: T.nilable(String)
+        signature_class_name: T.nilable(String),
+        schema_format: Symbol,
+        signature_class: T.nilable(T.class_of(Signature))
       ).void
     end
-    def initialize(instruction:, input_schema:, output_schema:, few_shot_examples: [], signature_class_name: nil)
+    def initialize(instruction:, input_schema:, output_schema:, few_shot_examples: [], signature_class_name: nil, schema_format: :json, signature_class: nil)
       @instruction = instruction
       @few_shot_examples = few_shot_examples.freeze
       @input_schema = input_schema.freeze
       @output_schema = output_schema.freeze
       @signature_class_name = signature_class_name
+      @schema_format = schema_format
+      @signature_class = signature_class
     end
 
     # Immutable update methods for optimization
@@ -47,7 +65,9 @@ module DSPy
         input_schema: @input_schema,
         output_schema: @output_schema,
         few_shot_examples: @few_shot_examples,
-        signature_class_name: @signature_class_name
+        signature_class_name: @signature_class_name,
+        schema_format: @schema_format,
+        signature_class: @signature_class
       )
     end
 
@@ -58,7 +78,9 @@ module DSPy
         input_schema: @input_schema,
         output_schema: @output_schema,
         few_shot_examples: new_examples,
-        signature_class_name: @signature_class_name
+        signature_class_name: @signature_class_name,
+        schema_format: @schema_format,
+        signature_class: @signature_class
       )
     end
 
@@ -72,16 +94,29 @@ module DSPy
     sig { returns(String) }
     def render_system_prompt
       sections = []
-      
-      sections << "Your input schema fields are:"
-      sections << "```json"
-      sections << JSON.pretty_generate(@input_schema)
-      sections << "```"
-      
-      sections << "Your output schema fields are:"
-      sections << "```json"
-      sections << JSON.pretty_generate(@output_schema)
-      sections << "```"
+
+      case schema_format
+      when :baml
+        sections << "Your input schema fields are:"
+        sections << "```baml"
+        sections << render_baml_schema(@input_schema, :input)
+        sections << "```"
+
+        sections << "Your output schema fields are:"
+        sections << "```baml"
+        sections << render_baml_schema(@output_schema, :output)
+        sections << "```"
+      else  # :json (default)
+        sections << "Your input schema fields are:"
+        sections << "```json"
+        sections << JSON.pretty_generate(@input_schema)
+        sections << "```"
+
+        sections << "Your output schema fields are:"
+        sections << "```json"
+        sections << JSON.pretty_generate(@output_schema)
+        sections << "```"
+      end
       
       sections << ""
       sections << "All interactions will be structured in the following way, with the appropriate values filled in."
@@ -148,32 +183,36 @@ module DSPy
         few_shot_examples: @few_shot_examples.map(&:to_h),
         input_schema: @input_schema,
         output_schema: @output_schema,
-        signature_class_name: @signature_class_name
+        signature_class_name: @signature_class_name,
+        schema_format: @schema_format
       }
     end
 
     sig { params(hash: T::Hash[Symbol, T.untyped]).returns(Prompt) }
     def self.from_h(hash)
       examples = (hash[:few_shot_examples] || []).map { |ex| FewShotExample.from_h(ex) }
-      
+
       new(
         instruction: hash[:instruction] || "",
         input_schema: hash[:input_schema] || {},
         output_schema: hash[:output_schema] || {},
         few_shot_examples: examples,
-        signature_class_name: hash[:signature_class_name]
+        signature_class_name: hash[:signature_class_name],
+        schema_format: hash[:schema_format] || :json
       )
     end
 
     # Create prompt from signature class
-    sig { params(signature_class: T.class_of(Signature)).returns(Prompt) }
-    def self.from_signature(signature_class)
+    sig { params(signature_class: T.class_of(Signature), schema_format: Symbol).returns(Prompt) }
+    def self.from_signature(signature_class, schema_format: :json)
       new(
         instruction: signature_class.description || "Complete this task.",
         input_schema: signature_class.input_json_schema,
         output_schema: signature_class.output_json_schema,
         few_shot_examples: [],
-        signature_class_name: signature_class.name
+        signature_class_name: signature_class.name,
+        schema_format: schema_format,
+        signature_class: signature_class
       )
     end
 
@@ -221,6 +260,37 @@ module DSPy
 
     private
 
+    # Render BAML schema for input or output
+    sig { params(schema: T::Hash[Symbol, T.untyped], type: Symbol).returns(String) }
+    def render_baml_schema(schema, type)
+      # If we have a signature_class, use sorbet-baml's to_baml method with custom name
+      if @signature_class
+        begin
+          require 'sorbet_baml'
+
+          struct_class = type == :input ? @signature_class.input_struct_class : @signature_class.output_struct_class
+          if struct_class
+            # Generate a proper class name from signature class name
+            base_name = @signature_class_name || @signature_class.name || "Schema"
+            class_name = type == :input ? "#{base_name}Input" : "#{base_name}Output"
+
+            # Get raw BAML and replace the ugly class name
+            raw_baml = struct_class.to_baml
+            # Replace the class definition line with a proper name
+            return raw_baml.sub(/^class #<Class:0x[0-9a-f]+>/, "class #{class_name}")
+          end
+        rescue LoadError
+          # Fall back to manual BAML generation if sorbet_baml is not available
+        end
+      end
+
+      # Fallback: generate BAML manually from schema
+      # This is a simple implementation that handles basic types
+      # For production use, sorbet-baml should be available
+      "# BAML schema generation requires sorbet-baml gem\n" \
+      "# Please install: gem install sorbet-baml"
+    end
+
     # Recursively serialize complex objects for JSON representation
     sig { params(obj: T.untyped).returns(T.untyped) }
     def serialize_for_json(obj)