dspy 0.3.1 → 0.4.0

data/lib/dspy/teleprompt/utils.rb

@@ -0,0 +1,380 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+require_relative '../instrumentation'
+require_relative '../evaluate'
+require_relative '../example'
+require_relative 'data_handler'
+
+module DSPy
+  module Teleprompt
+    # Bootstrap utilities for MIPROv2 optimization
+    # Handles few-shot example generation and candidate program evaluation
+    module Utils
+      extend T::Sig
+
+      # Configuration for bootstrap operations
+      class BootstrapConfig
+        extend T::Sig
+
+        sig { returns(Integer) }
+        attr_accessor :max_bootstrapped_examples
+
+        sig { returns(Integer) }
+        attr_accessor :max_labeled_examples
+
+        sig { returns(Integer) }
+        attr_accessor :num_candidate_sets
+
+        sig { returns(Integer) }
+        attr_accessor :max_errors
+
+        sig { returns(Integer) }
+        attr_accessor :num_threads
+
+        sig { returns(Float) }
+        attr_accessor :success_threshold
+
+        sig { returns(Integer) }
+        attr_accessor :minibatch_size
+
+        sig { void }
+        def initialize
+          @max_bootstrapped_examples = 4
+          @max_labeled_examples = 16
+          @num_candidate_sets = 10
+          @max_errors = 5
+          @num_threads = 1
+          @success_threshold = 0.8
+          @minibatch_size = 50
+        end
+      end
+
+      # Result of bootstrap operation
+      class BootstrapResult
+        extend T::Sig
+
+        sig { returns(T::Array[T::Array[DSPy::Example]]) }
+        attr_reader :candidate_sets
+
+        sig { returns(T::Array[DSPy::Example]) }
+        attr_reader :successful_examples
+
+        sig { returns(T::Array[DSPy::Example]) }
+        attr_reader :failed_examples
+
+        sig { returns(T::Hash[Symbol, T.untyped]) }
+        attr_reader :statistics
+
+        sig do
+          params(
+            candidate_sets: T::Array[T::Array[DSPy::Example]],
+            successful_examples: T::Array[DSPy::Example],
+            failed_examples: T::Array[DSPy::Example],
+            statistics: T::Hash[Symbol, T.untyped]
+          ).void
+        end
+        def initialize(candidate_sets:, successful_examples:, failed_examples:, statistics:)
+          @candidate_sets = candidate_sets.freeze
+          @successful_examples = successful_examples.freeze
+          @failed_examples = failed_examples.freeze
+          @statistics = statistics.freeze
+        end
+
+        sig { returns(Float) }
+        def success_rate
+          total = @successful_examples.size + @failed_examples.size
+          return 0.0 if total == 0
+          @successful_examples.size.to_f / total.to_f
+        end
+
+        sig { returns(Integer) }
+        def total_examples
+          @successful_examples.size + @failed_examples.size
+        end
+      end
+
+      # Create multiple candidate sets of few-shot examples through bootstrapping
+      sig do
+        params(
+          program: T.untyped,
+          trainset: T::Array[T.untyped],
+          config: BootstrapConfig,
+          metric: T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))
+        ).returns(BootstrapResult)
+      end
+      def self.create_n_fewshot_demo_sets(program, trainset, config: BootstrapConfig.new, metric: nil)
+        Instrumentation.instrument('dspy.optimization.bootstrap_start', {
+          trainset_size: trainset.size,
+          max_bootstrapped_examples: config.max_bootstrapped_examples,
+          num_candidate_sets: config.num_candidate_sets
+        }) do
+          # Convert to typed examples if needed
+          typed_examples = ensure_typed_examples(trainset)
+          
+          # Generate successful examples through bootstrap
+          successful_examples, failed_examples = generate_successful_examples(
+            program, 
+            typed_examples, 
+            config,
+            metric
+          )
+
+          # Create candidate sets from successful examples
+          candidate_sets = create_candidate_sets(successful_examples, config)
+
+          # Gather statistics
+          statistics = {
+            total_trainset: trainset.size,
+            successful_count: successful_examples.size,
+            failed_count: failed_examples.size,
+            success_rate: successful_examples.size.to_f / (successful_examples.size + failed_examples.size),
+            candidate_sets_created: candidate_sets.size,
+            average_set_size: candidate_sets.empty? ? 0 : candidate_sets.map(&:size).sum.to_f / candidate_sets.size
+          }
+
+          emit_bootstrap_complete_event(statistics)
+
+          BootstrapResult.new(
+            candidate_sets: candidate_sets,
+            successful_examples: successful_examples,
+            failed_examples: failed_examples,
+            statistics: statistics
+          )
+        end
+      end
+
+      # Evaluate a candidate program on examples with proper error handling
+      sig do
+        params(
+          program: T.untyped,
+          examples: T::Array[T.untyped],
+          config: BootstrapConfig,
+          metric: T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))
+        ).returns(DSPy::Evaluate::BatchEvaluationResult)
+      end
+      def self.eval_candidate_program(program, examples, config: BootstrapConfig.new, metric: nil)
+        # Use minibatch evaluation for large datasets
+        if examples.size > config.minibatch_size
+          eval_candidate_program_minibatch(program, examples, config, metric)
+        else
+          eval_candidate_program_full(program, examples, config, metric)
+        end
+      end
+
+      # Minibatch evaluation for large datasets
+      sig do
+        params(
+          program: T.untyped,
+          examples: T::Array[T.untyped],
+          config: BootstrapConfig,
+          metric: T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))
+        ).returns(DSPy::Evaluate::BatchEvaluationResult)
+      end
+      def self.eval_candidate_program_minibatch(program, examples, config, metric)
+        Instrumentation.instrument('dspy.optimization.minibatch_evaluation', {
+          total_examples: examples.size,
+          minibatch_size: config.minibatch_size,
+          num_batches: (examples.size.to_f / config.minibatch_size).ceil
+        }) do
+          # Randomly sample a minibatch for evaluation
+          sample_size = [config.minibatch_size, examples.size].min
+          sampled_examples = examples.sample(sample_size)
+          
+          eval_candidate_program_full(program, sampled_examples, config, metric)
+        end
+      end
+
+      # Full evaluation on all examples
+      sig do
+        params(
+          program: T.untyped,
+          examples: T::Array[T.untyped],
+          config: BootstrapConfig,
+          metric: T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))
+        ).returns(DSPy::Evaluate::BatchEvaluationResult)
+      end
+      def self.eval_candidate_program_full(program, examples, config, metric)
+        # Create evaluator with proper configuration
+        evaluator = DSPy::Evaluate.new(
+          program,
+          metric: metric || default_metric_for_examples(examples),
+          num_threads: config.num_threads,
+          max_errors: config.max_errors
+        )
+
+        # Run evaluation
+        evaluator.evaluate(examples, display_progress: false)
+      end
+
+      private
+
+      # Convert various example formats to typed examples
+      sig { params(examples: T::Array[T.untyped]).returns(T::Array[DSPy::Example]) }
+      def self.ensure_typed_examples(examples)
+        return examples if examples.all? { |ex| ex.is_a?(DSPy::Example) }
+        
+        raise ArgumentError, "All examples must be DSPy::Example instances. Legacy format support has been removed. Please convert your examples to use the structured format with :input and :expected keys."
+      end
+
+      # Generate successful examples through program execution
+      sig do
+        params(
+          program: T.untyped,
+          examples: T::Array[DSPy::Example],
+          config: BootstrapConfig,
+          metric: T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))
+        ).returns([T::Array[DSPy::Example], T::Array[DSPy::Example]])
+      end
+      def self.generate_successful_examples(program, examples, config, metric)
+        successful = []
+        failed = []
+        error_count = 0
+
+        # Use DataHandler for efficient shuffling
+        data_handler = DataHandler.new(examples)
+        shuffled_examples = data_handler.shuffle(random_state: 42)
+
+        shuffled_examples.each_with_index do |example, index|
+          break if successful.size >= config.max_labeled_examples
+          break if error_count >= config.max_errors
+
+          begin
+            # Run program on example input
+            prediction = program.call(**example.input_values)
+            
+            # Check if prediction matches expected output
+            if metric
+              success = metric.call(example, prediction.to_h)
+            else
+              success = example.matches_prediction?(prediction.to_h)
+            end
+
+            if success
+              # Create a new example with the successful prediction as reasoning/context
+              successful_example = create_successful_bootstrap_example(example, prediction)
+              successful << successful_example
+              
+              emit_bootstrap_example_event(index, true, nil)
+            else
+              failed << example
+              emit_bootstrap_example_event(index, false, "Prediction did not match expected output")
+            end
+
+          rescue => error
+            error_count += 1
+            failed << example
+            emit_bootstrap_example_event(index, false, error.message)
+            
+            # Log error but continue processing
+            DSPy.logger.warn("Bootstrap error on example #{index}: #{error.message}")
+            
+            # Stop if too many errors
+            if error_count >= config.max_errors
+              DSPy.logger.error("Too many bootstrap errors (#{error_count}), stopping early")
+              break
+            end
+          end
+        end
+
+        [successful, failed]
+      end
+
+      # Create candidate sets from successful examples using efficient data handling
+      sig do
+        params(
+          successful_examples: T::Array[DSPy::Example],
+          config: BootstrapConfig
+        ).returns(T::Array[T::Array[DSPy::Example]])
+      end
+      def self.create_candidate_sets(successful_examples, config)
+        return [] if successful_examples.empty?
+
+        # Use DataHandler for efficient sampling
+        data_handler = DataHandler.new(successful_examples)
+        set_size = [config.max_bootstrapped_examples, successful_examples.size].min
+
+        # Create candidate sets efficiently
+        candidate_sets = data_handler.create_candidate_sets(
+          config.num_candidate_sets,
+          set_size,
+          random_state: 42  # For reproducible results
+        )
+
+        candidate_sets
+      end
+
+      # Create a bootstrap example that includes the successful prediction
+      sig do
+        params(
+          original_example: DSPy::Example,
+          prediction: T.untyped
+        ).returns(DSPy::Example)
+      end
+      def self.create_successful_bootstrap_example(original_example, prediction)
+        # Convert prediction to FewShotExample format
+        DSPy::Example.new(
+          signature_class: original_example.signature_class,
+          input: original_example.input_values,
+          expected: prediction.to_h,
+          id: "bootstrap_#{original_example.id || SecureRandom.uuid}",
+          metadata: {
+            source: "bootstrap",
+            original_expected: original_example.expected_values,
+            bootstrap_timestamp: Time.now.iso8601
+          }
+        )
+      end
+
+
+      # Create default metric for examples
+      sig { params(examples: T::Array[T.untyped]).returns(T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))) }
+      def self.default_metric_for_examples(examples)
+        if examples.first.is_a?(DSPy::Example)
+          proc { |example, prediction| example.matches_prediction?(prediction) }
+        else
+          nil
+        end
+      end
+
+      # Emit bootstrap completion event
+      sig { params(statistics: T::Hash[Symbol, T.untyped]).void }
+      def self.emit_bootstrap_complete_event(statistics)
+        Instrumentation.emit('dspy.optimization.bootstrap_complete', {
+          successful_count: statistics[:successful_count],
+          failed_count: statistics[:failed_count],
+          success_rate: statistics[:success_rate],
+          candidate_sets_created: statistics[:candidate_sets_created],
+          average_set_size: statistics[:average_set_size]
+        })
+      end
+
+      # Emit individual bootstrap example event
+      sig { params(index: Integer, success: T::Boolean, error: T.nilable(String)).void }
+      def self.emit_bootstrap_example_event(index, success, error)
+        Instrumentation.emit('dspy.optimization.bootstrap_example', {
+          example_index: index,
+          success: success,
+          error: error,
+          timestamp: Time.now.iso8601
+        })
+      end
+
+      # Infer signature class from examples
+      sig { params(examples: T::Array[T.untyped]).returns(T.nilable(T.class_of(Signature))) }
+      def self.infer_signature_class(examples)
+        return nil if examples.empty?
+
+        first_example = examples.first
+        
+        if first_example.is_a?(DSPy::Example)
+          first_example.signature_class
+        elsif first_example.is_a?(Hash) && first_example[:signature_class]
+          first_example[:signature_class]
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/dspy/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DSPy
+  VERSION = "0.4.0"
+end

data/lib/dspy.rb

@@ -3,6 +3,8 @@ require 'sorbet-runtime'
 require 'dry-configurable'
 require 'dry/logger'
 
+require_relative 'dspy/version'
+
 module DSPy
   extend Dry::Configurable
   setting :lm
@@ -16,12 +18,26 @@ end
 require_relative 'dspy/module'
 require_relative 'dspy/field'
 require_relative 'dspy/signature'
+require_relative 'dspy/few_shot_example'
+require_relative 'dspy/prompt'
+require_relative 'dspy/example'
 require_relative 'dspy/lm'
 require_relative 'dspy/predict'
 require_relative 'dspy/chain_of_thought'
 require_relative 'dspy/re_act'
+require_relative 'dspy/evaluate'
+require_relative 'dspy/teleprompt/teleprompter'
+require_relative 'dspy/teleprompt/utils'
+require_relative 'dspy/teleprompt/data_handler'
+require_relative 'dspy/propose/grounded_proposer'
+require_relative 'dspy/teleprompt/simple_optimizer'
+require_relative 'dspy/teleprompt/mipro_v2'
 require_relative 'dspy/subscribers/logger_subscriber'
 require_relative 'dspy/tools'
 require_relative 'dspy/instrumentation'
+require_relative 'dspy/storage/program_storage'
+require_relative 'dspy/storage/storage_manager'
+require_relative 'dspy/registry/signature_registry'
+require_relative 'dspy/registry/registry_manager'
 
 # LoggerSubscriber will be lazy-initialized when first accessed

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: dspy
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Vicente Reig Rincón de Arellano
 bindir: bin
 cert_chain: []
-date: 2025-06-27 00:00:00.000000000 Z
+date: 2025-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -94,47 +94,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.1.0
 - !ruby/object:Gem::Dependency
-  name: ruby_llm
+  name: sorbet-runtime
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.5'
 - !ruby/object:Gem::Dependency
-  name: sorbet-runtime
+  name: sorbet-schema
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
-  name: sorbet-schema
+  name: polars-df
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: 0.20.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: 0.20.0
 description: A Ruby implementation of DSPy, a framework for programming with large
   language models
 email:
@@ -146,6 +146,9 @@ files:
 - README.md
 - lib/dspy.rb
 - lib/dspy/chain_of_thought.rb
+- lib/dspy/evaluate.rb
+- lib/dspy/example.rb
+- lib/dspy/few_shot_example.rb
 - lib/dspy/field.rb
 - lib/dspy/instrumentation.rb
 - lib/dspy/instrumentation/token_tracker.rb
@@ -154,17 +157,31 @@ files:
 - lib/dspy/lm/adapter_factory.rb
 - lib/dspy/lm/adapters/anthropic_adapter.rb
 - lib/dspy/lm/adapters/openai_adapter.rb
-- lib/dspy/lm/adapters/ruby_llm_adapter.rb
 - lib/dspy/lm/errors.rb
 - lib/dspy/lm/response.rb
 - lib/dspy/module.rb
 - lib/dspy/predict.rb
+- lib/dspy/prompt.rb
+- lib/dspy/propose/grounded_proposer.rb
 - lib/dspy/re_act.rb
+- lib/dspy/registry/registry_manager.rb
+- lib/dspy/registry/signature_registry.rb
 - lib/dspy/schema_adapters.rb
 - lib/dspy/signature.rb
+- lib/dspy/storage/program_storage.rb
+- lib/dspy/storage/storage_manager.rb
+- lib/dspy/subscribers/langfuse_subscriber.rb
 - lib/dspy/subscribers/logger_subscriber.rb
+- lib/dspy/subscribers/newrelic_subscriber.rb
+- lib/dspy/subscribers/otel_subscriber.rb
+- lib/dspy/teleprompt/data_handler.rb
+- lib/dspy/teleprompt/mipro_v2.rb
+- lib/dspy/teleprompt/simple_optimizer.rb
+- lib/dspy/teleprompt/teleprompter.rb
+- lib/dspy/teleprompt/utils.rb
 - lib/dspy/tools.rb
 - lib/dspy/tools/base.rb
+- lib/dspy/version.rb
 homepage: https://github.com/vicentereig/dspy.rb
 licenses:
 - MIT

data/lib/dspy/lm/adapters/ruby_llm_adapter.rb

@@ -1,81 +0,0 @@
-# frozen_string_literal: true
-
-begin
-  require 'ruby_llm'
-rescue LoadError
-  # ruby_llm is optional for backward compatibility
-end
-
-module DSPy
-  class LM
-    class RubyLLMAdapter < Adapter
-      def initialize(model:, api_key:)
-        super
-        
-        unless defined?(RubyLLM)
-          raise ConfigurationError, 
-                "ruby_llm gem is required for RubyLLMAdapter. " \
-                "Add 'gem \"ruby_llm\"' to your Gemfile."
-        end
-
-        configure_ruby_llm
-      end
-
-      def chat(messages:, &block)
-        begin
-          chat = RubyLLM.chat(model: model)
-          
-          # Add messages to chat
-          messages.each do |msg|
-            chat.add_message(role: msg[:role].to_sym, content: msg[:content])
-          end
-
-          # Get the last user message for ask method
-          last_user_message = messages.reverse.find { |msg| msg[:role] == 'user' }
-          
-          if last_user_message
-            # Remove the last user message since ask() will add it
-            chat.messages.pop if chat.messages.last&.content == last_user_message[:content]
-            chat.ask(last_user_message[:content], &block)
-          else
-            raise AdapterError, "No user message found in conversation"
-          end
-
-          content = chat.messages.last&.content || ""
-
-          Response.new(
-            content: content,
-            usage: nil, # ruby_llm doesn't provide usage info
-            metadata: {
-              provider: 'ruby_llm',
-              model: model,
-              message_count: chat.messages.length
-            }
-          )
-        rescue => e
-          raise AdapterError, "RubyLLM adapter error: #{e.message}"
-        end
-      end
-
-      private
-
-      def configure_ruby_llm
-        # Determine provider from model for configuration
-        if model.include?('gpt') || model.include?('openai')
-          RubyLLM.configure do |config|
-            config.openai_api_key = api_key
-          end
-        elsif model.include?('claude') || model.include?('anthropic')
-          RubyLLM.configure do |config|
-            config.anthropic_api_key = api_key
-          end
-        else
-          # Default to OpenAI configuration
-          RubyLLM.configure do |config|
-            config.openai_api_key = api_key
-          end
-        end
-      end
-    end
-  end
-end