dspy 0.3.1 → 0.5.0

data/lib/dspy/teleprompt/utils.rb

@@ -0,0 +1,380 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+require_relative '../instrumentation'
+require_relative '../evaluate'
+require_relative '../example'
+require_relative 'data_handler'
+
+module DSPy
+  module Teleprompt
+    # Bootstrap utilities for MIPROv2 optimization
+    # Handles few-shot example generation and candidate program evaluation
+    module Utils
+      extend T::Sig
+
+      # Configuration for bootstrap operations
+      class BootstrapConfig
+        extend T::Sig
+
+        sig { returns(Integer) }
+        attr_accessor :max_bootstrapped_examples
+
+        sig { returns(Integer) }
+        attr_accessor :max_labeled_examples
+
+        sig { returns(Integer) }
+        attr_accessor :num_candidate_sets
+
+        sig { returns(Integer) }
+        attr_accessor :max_errors
+
+        sig { returns(Integer) }
+        attr_accessor :num_threads
+
+        sig { returns(Float) }
+        attr_accessor :success_threshold
+
+        sig { returns(Integer) }
+        attr_accessor :minibatch_size
+
+        sig { void }
+        def initialize
+          @max_bootstrapped_examples = 4
+          @max_labeled_examples = 16
+          @num_candidate_sets = 10
+          @max_errors = 5
+          @num_threads = 1
+          @success_threshold = 0.8
+          @minibatch_size = 50
+        end
+      end
+
+      # Result of bootstrap operation
+      class BootstrapResult
+        extend T::Sig
+
+        sig { returns(T::Array[T::Array[DSPy::Example]]) }
+        attr_reader :candidate_sets
+
+        sig { returns(T::Array[DSPy::Example]) }
+        attr_reader :successful_examples
+
+        sig { returns(T::Array[DSPy::Example]) }
+        attr_reader :failed_examples
+
+        sig { returns(T::Hash[Symbol, T.untyped]) }
+        attr_reader :statistics
+
+        sig do
+          params(
+            candidate_sets: T::Array[T::Array[DSPy::Example]],
+            successful_examples: T::Array[DSPy::Example],
+            failed_examples: T::Array[DSPy::Example],
+            statistics: T::Hash[Symbol, T.untyped]
+          ).void
+        end
+        def initialize(candidate_sets:, successful_examples:, failed_examples:, statistics:)
+          @candidate_sets = candidate_sets.freeze
+          @successful_examples = successful_examples.freeze
+          @failed_examples = failed_examples.freeze
+          @statistics = statistics.freeze
+        end
+
+        sig { returns(Float) }
+        def success_rate
+          total = @successful_examples.size + @failed_examples.size
+          return 0.0 if total == 0
+          @successful_examples.size.to_f / total.to_f
+        end
+
+        sig { returns(Integer) }
+        def total_examples
+          @successful_examples.size + @failed_examples.size
+        end
+      end
+
+      # Create multiple candidate sets of few-shot examples through bootstrapping
+      sig do
+        params(
+          program: T.untyped,
+          trainset: T::Array[T.untyped],
+          config: BootstrapConfig,
+          metric: T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))
+        ).returns(BootstrapResult)
+      end
+      def self.create_n_fewshot_demo_sets(program, trainset, config: BootstrapConfig.new, metric: nil)
+        Instrumentation.instrument('dspy.optimization.bootstrap_start', {
+          trainset_size: trainset.size,
+          max_bootstrapped_examples: config.max_bootstrapped_examples,
+          num_candidate_sets: config.num_candidate_sets
+        }) do
+          # Convert to typed examples if needed
+          typed_examples = ensure_typed_examples(trainset)
+          
+          # Generate successful examples through bootstrap
+          successful_examples, failed_examples = generate_successful_examples(
+            program, 
+            typed_examples, 
+            config,
+            metric
+          )
+
+          # Create candidate sets from successful examples
+          candidate_sets = create_candidate_sets(successful_examples, config)
+
+          # Gather statistics
+          statistics = {
+            total_trainset: trainset.size,
+            successful_count: successful_examples.size,
+            failed_count: failed_examples.size,
+            success_rate: successful_examples.size.to_f / (successful_examples.size + failed_examples.size),
+            candidate_sets_created: candidate_sets.size,
+            average_set_size: candidate_sets.empty? ? 0 : candidate_sets.map(&:size).sum.to_f / candidate_sets.size
+          }
+
+          emit_bootstrap_complete_event(statistics)
+
+          BootstrapResult.new(
+            candidate_sets: candidate_sets,
+            successful_examples: successful_examples,
+            failed_examples: failed_examples,
+            statistics: statistics
+          )
+        end
+      end
+
+      # Evaluate a candidate program on examples with proper error handling
+      sig do
+        params(
+          program: T.untyped,
+          examples: T::Array[T.untyped],
+          config: BootstrapConfig,
+          metric: T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))
+        ).returns(DSPy::Evaluate::BatchEvaluationResult)
+      end
+      def self.eval_candidate_program(program, examples, config: BootstrapConfig.new, metric: nil)
+        # Use minibatch evaluation for large datasets
+        if examples.size > config.minibatch_size
+          eval_candidate_program_minibatch(program, examples, config, metric)
+        else
+          eval_candidate_program_full(program, examples, config, metric)
+        end
+      end
+
+      # Minibatch evaluation for large datasets
+      sig do
+        params(
+          program: T.untyped,
+          examples: T::Array[T.untyped],
+          config: BootstrapConfig,
+          metric: T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))
+        ).returns(DSPy::Evaluate::BatchEvaluationResult)
+      end
+      def self.eval_candidate_program_minibatch(program, examples, config, metric)
+        Instrumentation.instrument('dspy.optimization.minibatch_evaluation', {
+          total_examples: examples.size,
+          minibatch_size: config.minibatch_size,
+          num_batches: (examples.size.to_f / config.minibatch_size).ceil
+        }) do
+          # Randomly sample a minibatch for evaluation
+          sample_size = [config.minibatch_size, examples.size].min
+          sampled_examples = examples.sample(sample_size)
+          
+          eval_candidate_program_full(program, sampled_examples, config, metric)
+        end
+      end
+
+      # Full evaluation on all examples
+      sig do
+        params(
+          program: T.untyped,
+          examples: T::Array[T.untyped],
+          config: BootstrapConfig,
+          metric: T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))
+        ).returns(DSPy::Evaluate::BatchEvaluationResult)
+      end
+      def self.eval_candidate_program_full(program, examples, config, metric)
+        # Create evaluator with proper configuration
+        evaluator = DSPy::Evaluate.new(
+          program,
+          metric: metric || default_metric_for_examples(examples),
+          num_threads: config.num_threads,
+          max_errors: config.max_errors
+        )
+
+        # Run evaluation
+        evaluator.evaluate(examples, display_progress: false)
+      end
+
+      private
+
+      # Convert various example formats to typed examples
+      sig { params(examples: T::Array[T.untyped]).returns(T::Array[DSPy::Example]) }
+      def self.ensure_typed_examples(examples)
+        return examples if examples.all? { |ex| ex.is_a?(DSPy::Example) }
+        
+        raise ArgumentError, "All examples must be DSPy::Example instances. Legacy format support has been removed. Please convert your examples to use the structured format with :input and :expected keys."
+      end
+
+      # Generate successful examples through program execution
+      sig do
+        params(
+          program: T.untyped,
+          examples: T::Array[DSPy::Example],
+          config: BootstrapConfig,
+          metric: T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))
+        ).returns([T::Array[DSPy::Example], T::Array[DSPy::Example]])
+      end
+      def self.generate_successful_examples(program, examples, config, metric)
+        successful = []
+        failed = []
+        error_count = 0
+
+        # Use DataHandler for efficient shuffling
+        data_handler = DataHandler.new(examples)
+        shuffled_examples = data_handler.shuffle(random_state: 42)
+
+        shuffled_examples.each_with_index do |example, index|
+          break if successful.size >= config.max_labeled_examples
+          break if error_count >= config.max_errors
+
+          begin
+            # Run program on example input
+            prediction = program.call(**example.input_values)
+            
+            # Check if prediction matches expected output
+            if metric
+              success = metric.call(example, prediction.to_h)
+            else
+              success = example.matches_prediction?(prediction.to_h)
+            end
+
+            if success
+              # Create a new example with the successful prediction as reasoning/context
+              successful_example = create_successful_bootstrap_example(example, prediction)
+              successful << successful_example
+              
+              emit_bootstrap_example_event(index, true, nil)
+            else
+              failed << example
+              emit_bootstrap_example_event(index, false, "Prediction did not match expected output")
+            end
+
+          rescue => error
+            error_count += 1
+            failed << example
+            emit_bootstrap_example_event(index, false, error.message)
+            
+            # Log error but continue processing
+            DSPy.logger.warn("Bootstrap error on example #{index}: #{error.message}")
+            
+            # Stop if too many errors
+            if error_count >= config.max_errors
+              DSPy.logger.error("Too many bootstrap errors (#{error_count}), stopping early")
+              break
+            end
+          end
+        end
+
+        [successful, failed]
+      end
+
+      # Create candidate sets from successful examples using efficient data handling
+      sig do
+        params(
+          successful_examples: T::Array[DSPy::Example],
+          config: BootstrapConfig
+        ).returns(T::Array[T::Array[DSPy::Example]])
+      end
+      def self.create_candidate_sets(successful_examples, config)
+        return [] if successful_examples.empty?
+
+        # Use DataHandler for efficient sampling
+        data_handler = DataHandler.new(successful_examples)
+        set_size = [config.max_bootstrapped_examples, successful_examples.size].min
+
+        # Create candidate sets efficiently
+        candidate_sets = data_handler.create_candidate_sets(
+          config.num_candidate_sets,
+          set_size,
+          random_state: 42  # For reproducible results
+        )
+
+        candidate_sets
+      end
+
+      # Create a bootstrap example that includes the successful prediction
+      sig do
+        params(
+          original_example: DSPy::Example,
+          prediction: T.untyped
+        ).returns(DSPy::Example)
+      end
+      def self.create_successful_bootstrap_example(original_example, prediction)
+        # Convert prediction to FewShotExample format
+        DSPy::Example.new(
+          signature_class: original_example.signature_class,
+          input: original_example.input_values,
+          expected: prediction.to_h,
+          id: "bootstrap_#{original_example.id || SecureRandom.uuid}",
+          metadata: {
+            source: "bootstrap",
+            original_expected: original_example.expected_values,
+            bootstrap_timestamp: Time.now.iso8601
+          }
+        )
+      end
+
+
+      # Create default metric for examples
+      sig { params(examples: T::Array[T.untyped]).returns(T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))) }
+      def self.default_metric_for_examples(examples)
+        if examples.first.is_a?(DSPy::Example)
+          proc { |example, prediction| example.matches_prediction?(prediction) }
+        else
+          nil
+        end
+      end
+
+      # Emit bootstrap completion event
+      sig { params(statistics: T::Hash[Symbol, T.untyped]).void }
+      def self.emit_bootstrap_complete_event(statistics)
+        Instrumentation.emit('dspy.optimization.bootstrap_complete', {
+          successful_count: statistics[:successful_count],
+          failed_count: statistics[:failed_count],
+          success_rate: statistics[:success_rate],
+          candidate_sets_created: statistics[:candidate_sets_created],
+          average_set_size: statistics[:average_set_size]
+        })
+      end
+
+      # Emit individual bootstrap example event
+      sig { params(index: Integer, success: T::Boolean, error: T.nilable(String)).void }
+      def self.emit_bootstrap_example_event(index, success, error)
+        Instrumentation.emit('dspy.optimization.bootstrap_example', {
+          example_index: index,
+          success: success,
+          error: error,
+          timestamp: Time.now.iso8601
+        })
+      end
+
+      # Infer signature class from examples
+      sig { params(examples: T::Array[T.untyped]).returns(T.nilable(T.class_of(Signature))) }
+      def self.infer_signature_class(examples)
+        return nil if examples.empty?
+
+        first_example = examples.first
+        
+        if first_example.is_a?(DSPy::Example)
+          first_example.signature_class
+        elsif first_example.is_a?(Hash) && first_example[:signature_class]
+          first_example[:signature_class]
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/dspy/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DSPy
+  VERSION = "0.5.0"
+end

data/lib/dspy.rb

@@ -2,26 +2,131 @@
 require 'sorbet-runtime'
 require 'dry-configurable'
 require 'dry/logger'
+require 'securerandom'
+
+require_relative 'dspy/version'
 
 module DSPy
   extend Dry::Configurable
+
+  # Timestamp format options for instrumentation events
+  class TimestampFormat < T::Enum
+    enums do
+      ISO8601 = new('iso8601')
+      RFC3339_NANO = new('rfc3339_nano')
+      UNIX_NANO = new('unix_nano')
+    end
+  end
+  
   setting :lm
   setting :logger, default: Dry.Logger(:dspy, formatter: :string)
+  
+  # Nested instrumentation configuration using proper dry-configurable syntax
+  setting :instrumentation do
+    # Core settings
+    setting :enabled, default: false
+    setting :subscribers, default: []
+    setting :sampling_rate, default: 1.0
+    setting :trace_level, default: :standard
+    setting :async_processing, default: false
+    setting :buffer_size, default: 1000
+    setting :flush_interval, default: 30
+    setting :error_reporting, default: false
+    setting :error_service, default: nil
+    setting :sampling_rules, default: {}
+    setting :timestamp_format, default: TimestampFormat::ISO8601
+
+    # Nested correlation ID configuration
+    setting :correlation_id do
+      setting :enabled, default: false
+      setting :header, default: 'X-Correlation-ID'
+      setting :generator, default: -> { SecureRandom.uuid }
+    end
+
+    # Nested logger configuration
+    setting :logger do
+      setting :level, default: :info
+      setting :include_payloads, default: true
+      setting :correlation_id, default: true
+      setting :sampling, default: {}
+      setting :sampling_conditions, default: {}
+    end
+
+    # Nested OpenTelemetry configuration
+    setting :otel do
+      setting :tracer_name, default: 'dspy-ruby'
+      setting :service_name, default: 'dspy-application'
+      setting :service_version, default: DSPy::VERSION
+      setting :endpoint, default: -> { ENV['OTEL_EXPORTER_OTLP_ENDPOINT'] }
+    end
+
+    # Nested New Relic configuration
+    setting :newrelic do
+      setting :app_name, default: 'DSPy Application'
+      setting :license_key, default: -> { ENV['NEW_RELIC_LICENSE_KEY'] }
+      setting :custom_attributes, default: {}
+    end
+
+    # Nested Langfuse configuration
+    setting :langfuse do
+      setting :public_key, default: -> { ENV['LANGFUSE_PUBLIC_KEY'] }
+      setting :secret_key, default: -> { ENV['LANGFUSE_SECRET_KEY'] }
+      setting :host, default: -> { ENV['LANGFUSE_HOST'] }
+      setting :track_tokens, default: true
+      setting :track_costs, default: true
+      setting :track_prompts, default: true
+    end
+  end
 
   def self.logger
     config.logger
   end
+
+  # Validation methods for instrumentation configuration
+  def self.validate_instrumentation!
+    config = self.config.instrumentation
+    
+    raise ArgumentError, "Sampling rate must be between 0.0 and 1.0" unless config.sampling_rate.between?(0.0, 1.0)
+    raise ArgumentError, "Buffer size must be positive" unless config.buffer_size > 0
+    raise ArgumentError, "Flush interval must be positive" unless config.flush_interval > 0
+    raise ArgumentError, "Invalid trace level" unless [:minimal, :standard, :detailed].include?(config.trace_level)
+    raise ArgumentError, "Invalid timestamp format" unless config.timestamp_format.is_a?(TimestampFormat)
+    
+    if config.enabled && config.subscribers.empty?
+      raise ArgumentError, "Must specify at least one subscriber when instrumentation is enabled"
+    end
+    
+    # Validate subscribers are valid symbols
+    invalid_subscribers = config.subscribers - [:logger, :otel, :newrelic, :langfuse]
+    unless invalid_subscribers.empty?
+      raise ArgumentError, "Invalid subscribers: #{invalid_subscribers.join(', ')}"
+    end
+  end
 end
 
 require_relative 'dspy/module'
 require_relative 'dspy/field'
 require_relative 'dspy/signature'
+require_relative 'dspy/few_shot_example'
+require_relative 'dspy/prompt'
+require_relative 'dspy/example'
 require_relative 'dspy/lm'
 require_relative 'dspy/predict'
 require_relative 'dspy/chain_of_thought'
 require_relative 'dspy/re_act'
+require_relative 'dspy/evaluate'
+require_relative 'dspy/teleprompt/teleprompter'
+require_relative 'dspy/teleprompt/utils'
+require_relative 'dspy/teleprompt/data_handler'
+require_relative 'dspy/propose/grounded_proposer'
+require_relative 'dspy/teleprompt/simple_optimizer'
+require_relative 'dspy/teleprompt/mipro_v2'
 require_relative 'dspy/subscribers/logger_subscriber'
 require_relative 'dspy/tools'
 require_relative 'dspy/instrumentation'
+require_relative 'dspy/storage/program_storage'
+require_relative 'dspy/storage/storage_manager'
+require_relative 'dspy/registry/signature_registry'
+require_relative 'dspy/registry/registry_manager'
 
 # LoggerSubscriber will be lazy-initialized when first accessed

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: dspy
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Vicente Reig Rincón de Arellano
 bindir: bin
 cert_chain: []
-date: 2025-06-27 00:00:00.000000000 Z
+date: 2025-07-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -94,47 +94,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.1.0
 - !ruby/object:Gem::Dependency
-  name: ruby_llm
+  name: sorbet-runtime
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.5'
 - !ruby/object:Gem::Dependency
-  name: sorbet-runtime
+  name: sorbet-schema
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
-  name: sorbet-schema
+  name: polars-df
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: 0.20.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: 0.20.0
 description: A Ruby implementation of DSPy, a framework for programming with large
   language models
 email:
@@ -146,6 +146,9 @@ files:
 - README.md
 - lib/dspy.rb
 - lib/dspy/chain_of_thought.rb
+- lib/dspy/evaluate.rb
+- lib/dspy/example.rb
+- lib/dspy/few_shot_example.rb
 - lib/dspy/field.rb
 - lib/dspy/instrumentation.rb
 - lib/dspy/instrumentation/token_tracker.rb
@@ -154,17 +157,34 @@ files:
 - lib/dspy/lm/adapter_factory.rb
 - lib/dspy/lm/adapters/anthropic_adapter.rb
 - lib/dspy/lm/adapters/openai_adapter.rb
-- lib/dspy/lm/adapters/ruby_llm_adapter.rb
 - lib/dspy/lm/errors.rb
 - lib/dspy/lm/response.rb
+- lib/dspy/mixins/instrumentation_helpers.rb
+- lib/dspy/mixins/struct_builder.rb
+- lib/dspy/mixins/type_coercion.rb
 - lib/dspy/module.rb
 - lib/dspy/predict.rb
+- lib/dspy/prompt.rb
+- lib/dspy/propose/grounded_proposer.rb
 - lib/dspy/re_act.rb
+- lib/dspy/registry/registry_manager.rb
+- lib/dspy/registry/signature_registry.rb
 - lib/dspy/schema_adapters.rb
 - lib/dspy/signature.rb
+- lib/dspy/storage/program_storage.rb
+- lib/dspy/storage/storage_manager.rb
+- lib/dspy/subscribers/langfuse_subscriber.rb
 - lib/dspy/subscribers/logger_subscriber.rb
+- lib/dspy/subscribers/newrelic_subscriber.rb
+- lib/dspy/subscribers/otel_subscriber.rb
+- lib/dspy/teleprompt/data_handler.rb
+- lib/dspy/teleprompt/mipro_v2.rb
+- lib/dspy/teleprompt/simple_optimizer.rb
+- lib/dspy/teleprompt/teleprompter.rb
+- lib/dspy/teleprompt/utils.rb
 - lib/dspy/tools.rb
 - lib/dspy/tools/base.rb
+- lib/dspy/version.rb
 homepage: https://github.com/vicentereig/dspy.rb
 licenses:
 - MIT