dspy 0.25.1 → 0.26.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55addf122534bacff753f272a385ddb035e66322ed63ecc5bc27ce3a2bd4ea03
-  data.tar.gz: b73d12f9f560dcaf60fdfac9640bf6a422bb0b912912bf805dfb183058e94f81
+  metadata.gz: e243b7278275462baea2f493270166a1ae4b5419d4f072a769e4ba4b0f65e3e0
+  data.tar.gz: 13bcbcf4ee67c08f19ad619bc8118e39ca9a02045c5a18b3a664fb112724fa87
 SHA512:
-  metadata.gz: 9ffff85304ccbf2b72143e878e134637b97db118ac7b69e91597664d24e1a4fad63685219844d75c91c5e859dc7080a39052ac5baa68df520f1fb6731f35aece
-  data.tar.gz: f845270b9fbe9ff81fbed8519517cf174add60eb629d26702d8411077347a4dd1b9206bbf6b1fd7b4b4b7be260b35f48e2dfaf3951ae9d0e87b6d1aad7ea6b3e
+  metadata.gz: 687385021bf9391b22ae51a3f7c05880bec9691347a4e8ecac9175b7e81190c9f63cb0670a94e7324a045d748346cc91b6f7e174808607eaf0d02b8a0a117992
+  data.tar.gz: 3212712d53aca34cbc475503396d4fbeb7b8c11632b673782e7cd2dc2e6fdb22a3ad67688d92b60bd72e845f7b598b446e94fd3f5c6efd5e87f212bb1be14b9e

data/README.md

@@ -73,7 +73,7 @@ puts result.confidence   # => 0.85
 - **Prompt Objects** - Manipulate prompts as first-class objects instead of strings
 - **Typed Examples** - Type-safe training data with automatic validation
 - **Evaluation Framework** - Advanced metrics beyond simple accuracy with error-resilient pipelines
-- **MIPROv2 Optimization** - Automatic prompt optimization with storage and persistence
+- **MIPROv2 Optimization** - Advanced Bayesian optimization with Gaussian Processes, multiple optimization strategies, and storage persistence
 - **GEPA Optimization** - Genetic-Pareto optimization for multi-objective prompt improvement
 
 **Production Features:**
@@ -128,7 +128,7 @@ For LLMs and AI assistants working with DSPy.rb:
 ### Optimization
 - **[Evaluation Framework](docs/src/optimization/evaluation.md)** - Advanced metrics beyond simple accuracy
 - **[Prompt Optimization](docs/src/optimization/prompt-optimization.md)** - Manipulate prompts as objects
-- **[MIPROv2 Optimizer](docs/src/optimization/miprov2.md)** - Automatic optimization algorithms
+- **[MIPROv2 Optimizer](docs/src/optimization/miprov2.md)** - Advanced Bayesian optimization with Gaussian Processes
 - **[GEPA Optimizer](docs/src/optimization/gepa.md)** - Genetic-Pareto optimization for multi-objective prompt optimization
 
 ### Production Features
@@ -157,30 +157,6 @@ Then run:
 bundle install
 ```
 
-#### System Dependencies for Ubuntu/Pop!_OS
-
-If you need to compile the `polars-df` dependency from source (used for data processing in evaluations), install these system packages:
-
-```bash
-# Update package list
-sudo apt-get update
-
-# Install Ruby development files (if not already installed)
-sudo apt-get install ruby-full ruby-dev
-
-# Install essential build tools
-sudo apt-get install build-essential
-
-# Install Rust and Cargo (required for polars-df compilation)
-curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
-source $HOME/.cargo/env
-
-# Install CMake (often needed for Rust projects)
-sudo apt-get install cmake
-```
-
-**Note**: The `polars-df` gem compilation can take 15-20 minutes. Pre-built binaries are available for most platforms, so compilation is only needed if a pre-built binary isn't available for your system.
-
 ## Recent Achievements
 
 DSPy.rb has rapidly evolved from experimental to production-ready:
@@ -190,7 +166,7 @@ DSPy.rb has rapidly evolved from experimental to production-ready:
 - ✅ **Type-Safe Strategy Configuration** - Provider-optimized automatic strategy selection  
 - ✅ **Core Module System** - Predict, ChainOfThought, ReAct, CodeAct with type safety
 - ✅ **Production Observability** - OpenTelemetry, New Relic, and Langfuse integration
-- ✅ **Optimization Framework** - MIPROv2 algorithm with storage & persistence
+- ✅ **Advanced Optimization** - MIPROv2 with Bayesian optimization, Gaussian Processes, and multiple strategies
 
 ### Recent Advances  
 - ✅ **Enhanced Langfuse Integration (v0.25.0)** - Comprehensive OpenTelemetry span reporting with proper input/output, hierarchical nesting, accurate timing, and observation types

data/lib/dspy/optimizers/gaussian_process.rb

@@ -0,0 +1,141 @@
+# typed: strict
+# frozen_string_literal: true
+
+require 'numo/narray'
+require 'sorbet-runtime'
+
+module DSPy
+  module Optimizers
+    # Pure Ruby Gaussian Process implementation for Bayesian optimization
+    # No external LAPACK/BLAS dependencies required
+    class GaussianProcess
+      extend T::Sig
+
+      sig { params(length_scale: Float, signal_variance: Float, noise_variance: Float).void }
+      def initialize(length_scale: 1.0, signal_variance: 1.0, noise_variance: 1e-6)
+        @length_scale = length_scale
+        @signal_variance = signal_variance
+        @noise_variance = noise_variance
+        @fitted = T.let(false, T::Boolean)
+      end
+
+      sig { params(x1: T::Array[T::Array[Float]], x2: T::Array[T::Array[Float]]).returns(Numo::DFloat) }
+      def rbf_kernel(x1, x2)
+        # Convert to Numo arrays
+        x1_array = Numo::DFloat[*x1]
+        x2_array = Numo::DFloat[*x2]
+        
+        # Compute squared Euclidean distances manually
+        n1, n2 = x1_array.shape[0], x2_array.shape[0]
+        sqdist = Numo::DFloat.zeros(n1, n2)
+        
+        (0...n1).each do |i|
+          (0...n2).each do |j|
+            diff = x1_array[i, true] - x2_array[j, true]
+            sqdist[i, j] = (diff ** 2).sum
+          end
+        end
+        
+        # RBF kernel: σ² * exp(-0.5 * d² / ℓ²)
+        @signal_variance * Numo::NMath.exp(-0.5 * sqdist / (@length_scale ** 2))
+      end
+
+      sig { params(x_train: T::Array[T::Array[Float]], y_train: T::Array[Float]).void }
+      def fit(x_train, y_train)
+        @x_train = x_train
+        @y_train = Numo::DFloat[*y_train]
+        
+        # Compute kernel matrix
+        k_matrix = rbf_kernel(x_train, x_train)
+        
+        # Add noise to diagonal for numerical stability
+        n = k_matrix.shape[0]
+        (0...n).each { |i| k_matrix[i, i] += @noise_variance }
+        
+        # Store inverted kernel matrix using simple LU decomposition
+        @k_inv = matrix_inverse(k_matrix)
+        @alpha = @k_inv.dot(@y_train)
+        
+        @fitted = true
+      end
+
+      sig { params(x_test: T::Array[T::Array[Float]], return_std: T::Boolean).returns(T.any(Numo::DFloat, [Numo::DFloat, Numo::DFloat])) }
+      def predict(x_test, return_std: false)
+        raise "Gaussian Process not fitted" unless @fitted
+        
+        # Kernel between training and test points
+        k_star = rbf_kernel(T.must(@x_train), x_test)
+        
+        # Predictive mean
+        mean = k_star.transpose.dot(@alpha)
+        
+        return mean unless return_std
+        
+        # Predictive variance (simplified for small matrices)
+        k_star_star = rbf_kernel(x_test, x_test)
+        var_matrix = k_star_star - k_star.transpose.dot(@k_inv).dot(k_star)
+        var = var_matrix.diagonal
+        
+        # Ensure positive variance (element-wise maximum)
+        var = var.map { |v| [v, 1e-12].max }
+        std = Numo::NMath.sqrt(var)
+        
+        [mean, std]
+      end
+
+      private
+
+      sig { returns(T.nilable(T::Array[T::Array[Float]])) }
+      attr_reader :x_train
+
+      sig { returns(T.nilable(Numo::DFloat)) }
+      attr_reader :y_train, :k_inv, :alpha
+
+      # Simple matrix inversion using Gauss-Jordan elimination
+      # Only suitable for small matrices (< 100x100)
+      sig { params(matrix: Numo::DFloat).returns(Numo::DFloat) }
+      def matrix_inverse(matrix)
+        n = matrix.shape[0]
+        raise "Matrix must be square" unless matrix.shape[0] == matrix.shape[1]
+        
+        # Create augmented matrix [A|I]
+        augmented = Numo::DFloat.zeros(n, 2*n)
+        augmented[true, 0...n] = matrix.copy
+        (0...n).each { |i| augmented[i, n+i] = 1.0 }
+        
+        # Gauss-Jordan elimination
+        (0...n).each do |i|
+          # Find pivot
+          max_row = i
+          (i+1...n).each do |k|
+            if augmented[k, i].abs > augmented[max_row, i].abs
+              max_row = k
+            end
+          end
+          
+          # Swap rows if needed
+          if max_row != i
+            temp = augmented[i, true].copy
+            augmented[i, true] = augmented[max_row, true]
+            augmented[max_row, true] = temp
+          end
+          
+          # Make diagonal element 1
+          pivot = augmented[i, i]
+          raise "Matrix is singular" if pivot.abs < 1e-12
+          augmented[i, true] /= pivot
+          
+          # Eliminate column
+          (0...n).each do |j|
+            next if i == j
+            factor = augmented[j, i]
+            augmented[j, true] -= factor * augmented[i, true]
+          end
+        end
+        
+        # Extract inverse matrix
+        augmented[true, n...2*n]
+      end
+    end
+  end
+end

data/lib/dspy/teleprompt/mipro_v2.rb

@@ -5,14 +5,34 @@ require 'sorbet-runtime'
 require_relative 'teleprompter'
 require_relative 'utils'
 require_relative '../propose/grounded_proposer'
+require_relative '../optimizers/gaussian_process'
 
 module DSPy
   module Teleprompt
+    # Enum for candidate configuration types
+    class CandidateType < T::Enum
+      enums do
+        Baseline = new("baseline")
+        InstructionOnly = new("instruction_only")
+        FewShotOnly = new("few_shot_only")
+        Combined = new("combined")
+      end
+    end
+
+    # Enum for optimization strategies
+    class OptimizationStrategy < T::Enum
+      enums do
+        Greedy = new("greedy")
+        Adaptive = new("adaptive") 
+        Bayesian = new("bayesian")
+      end
+    end
     # MIPROv2: Multi-prompt Instruction Proposal with Retrieval Optimization
     # State-of-the-art prompt optimization combining bootstrap sampling, 
     # instruction generation, and Bayesian optimization
     class MIPROv2 < Teleprompter
       extend T::Sig
+      include Dry::Configurable
 
       # Auto-configuration modes for different optimization needs
       module AutoMode
@@ -25,15 +45,17 @@ module DSPy
           ).returns(MIPROv2)
         end
         def self.light(metric: nil, **kwargs)
-          config = MIPROv2Config.new
-          config.num_trials = 6
-          config.num_instruction_candidates = 3
-          config.max_bootstrapped_examples = 2
-          config.max_labeled_examples = 8
-          config.bootstrap_sets = 3
-          config.optimization_strategy = "greedy"
-          config.early_stopping_patience = 2
-          MIPROv2.new(metric: metric, config: config, **kwargs)
+          optimizer = MIPROv2.new(metric: metric, **kwargs)
+          optimizer.configure do |config|
+            config.num_trials = 6
+            config.num_instruction_candidates = 3
+            config.max_bootstrapped_examples = 2
+            config.max_labeled_examples = 8
+            config.bootstrap_sets = 3
+            config.optimization_strategy = :greedy
+            config.early_stopping_patience = 2
+          end
+          optimizer
         end
 
         sig do
@@ -43,15 +65,17 @@ module DSPy
           ).returns(MIPROv2)
         end
         def self.medium(metric: nil, **kwargs)
-          config = MIPROv2Config.new
-          config.num_trials = 12
-          config.num_instruction_candidates = 5
-          config.max_bootstrapped_examples = 4
-          config.max_labeled_examples = 16
-          config.bootstrap_sets = 5
-          config.optimization_strategy = "adaptive"
-          config.early_stopping_patience = 3
-          MIPROv2.new(metric: metric, config: config, **kwargs)
+          optimizer = MIPROv2.new(metric: metric, **kwargs)
+          optimizer.configure do |config|
+            config.num_trials = 12
+            config.num_instruction_candidates = 5
+            config.max_bootstrapped_examples = 4
+            config.max_labeled_examples = 16
+            config.bootstrap_sets = 5
+            config.optimization_strategy = :adaptive
+            config.early_stopping_patience = 3
+          end
+          optimizer
         end
 
         sig do
@@ -61,137 +85,102 @@ module DSPy
           ).returns(MIPROv2)
         end
         def self.heavy(metric: nil, **kwargs)
-          config = MIPROv2Config.new
-          config.num_trials = 18
-          config.num_instruction_candidates = 8
-          config.max_bootstrapped_examples = 6
-          config.max_labeled_examples = 24
-          config.bootstrap_sets = 8
-          config.optimization_strategy = "bayesian"
-          config.early_stopping_patience = 5
-          MIPROv2.new(metric: metric, config: config, **kwargs)
+          optimizer = MIPROv2.new(metric: metric, **kwargs)
+          optimizer.configure do |config|
+            config.num_trials = 18
+            config.num_instruction_candidates = 8
+            config.max_bootstrapped_examples = 6
+            config.max_labeled_examples = 24
+            config.bootstrap_sets = 8
+            config.optimization_strategy = :bayesian
+            config.early_stopping_patience = 5
+          end
+          optimizer
         end
       end
 
-      # Configuration for MIPROv2 optimization
-      class MIPROv2Config < Config
-        extend T::Sig
-
-        sig { returns(Integer) }
-        attr_accessor :num_trials
-
-        sig { returns(Integer) }
-        attr_accessor :num_instruction_candidates
-
-        sig { returns(Integer) }
-        attr_accessor :bootstrap_sets
-
-        sig { returns(String) }
-        attr_accessor :optimization_strategy
-
-        sig { returns(Float) }
-        attr_accessor :init_temperature
-
-        sig { returns(Float) }
-        attr_accessor :final_temperature
-
-        sig { returns(Integer) }
-        attr_accessor :early_stopping_patience
-
-        sig { returns(T::Boolean) }
-        attr_accessor :use_bayesian_optimization
-
-        sig { returns(T::Boolean) }
-        attr_accessor :track_diversity
-
-        sig { returns(DSPy::Propose::GroundedProposer::Config) }
-        attr_accessor :proposer_config
-
-        sig { void }
-        def initialize
-          super
-          @num_trials = 12
-          @num_instruction_candidates = 5
-          @bootstrap_sets = 5
-          @optimization_strategy = "adaptive" # greedy, adaptive, bayesian
-          @init_temperature = 1.0
-          @final_temperature = 0.1
-          @early_stopping_patience = 3
-          @use_bayesian_optimization = true
-          @track_diversity = true
-          @proposer_config = DSPy::Propose::GroundedProposer::Config.new
+      # Dry-configurable settings for MIPROv2
+      setting :num_trials, default: 12
+      setting :num_instruction_candidates, default: 5
+      setting :bootstrap_sets, default: 5
+      setting :max_bootstrapped_examples, default: 4
+      setting :max_labeled_examples, default: 16
+      setting :optimization_strategy, default: OptimizationStrategy::Adaptive, constructor: ->(value) {
+        # Coerce symbols to enum values
+        case value
+        when :greedy then OptimizationStrategy::Greedy
+        when :adaptive then OptimizationStrategy::Adaptive
+        when :bayesian then OptimizationStrategy::Bayesian
+        when OptimizationStrategy then value
+        when nil then OptimizationStrategy::Adaptive
+        else
+          raise ArgumentError, "Invalid optimization strategy: #{value}. Must be one of :greedy, :adaptive, :bayesian"
         end
-
-        sig { returns(T::Hash[Symbol, T.untyped]) }
-        def to_h
-          super.merge({
-            num_trials: @num_trials,
-            num_instruction_candidates: @num_instruction_candidates,
-            bootstrap_sets: @bootstrap_sets,
-            optimization_strategy: @optimization_strategy,
-            init_temperature: @init_temperature,
-            final_temperature: @final_temperature,
-            early_stopping_patience: @early_stopping_patience,
-            use_bayesian_optimization: @use_bayesian_optimization,
-            track_diversity: @track_diversity
-          })
+      }
+      setting :init_temperature, default: 1.0
+      setting :final_temperature, default: 0.1
+      setting :early_stopping_patience, default: 3
+      setting :use_bayesian_optimization, default: true
+      setting :track_diversity, default: true
+      setting :max_errors, default: 3
+      setting :num_threads, default: 1
+
+      # Class-level configuration method - sets defaults for new instances
+      def self.configure(&block)
+        if block_given?
+          # Store configuration in a class variable for new instances
+          @default_config_block = block
         end
       end
 
-      # Candidate configuration for optimization trials
-      class CandidateConfig
-        extend T::Sig
-
-        sig { returns(String) }
-        attr_reader :instruction
-
-        sig { returns(T::Array[T.untyped]) }
-        attr_reader :few_shot_examples
-
-        sig { returns(T::Hash[Symbol, T.untyped]) }
-        attr_reader :metadata
+      # Get the default configuration block
+      def self.default_config_block
+        @default_config_block
+      end
 
-        sig { returns(String) }
-        attr_reader :config_id
 
-        sig do
-          params(
-            instruction: String,
-            few_shot_examples: T::Array[T.untyped],
-            metadata: T::Hash[Symbol, T.untyped]
-          ).void
-        end
-        def initialize(instruction:, few_shot_examples:, metadata: {})
-          @instruction = instruction
-          @few_shot_examples = few_shot_examples
-          @metadata = metadata.freeze
-          @config_id = generate_config_id
+      # Simple data structure for evaluated candidate configurations (immutable)
+      EvaluatedCandidate = Data.define(
+        :instruction,
+        :few_shot_examples,
+        :type,
+        :metadata,
+        :config_id
+      ) do
+        extend T::Sig
+        
+        # Generate a config ID based on content
+        sig { params(instruction: String, few_shot_examples: T::Array[T.untyped], type: CandidateType, metadata: T::Hash[Symbol, T.untyped]).returns(EvaluatedCandidate) }
+        def self.create(instruction:, few_shot_examples: [], type: CandidateType::Baseline, metadata: {})
+          content = "#{instruction}_#{few_shot_examples.size}_#{type.serialize}_#{metadata.hash}"
+          config_id = Digest::SHA256.hexdigest(content)[0, 12]
+          
+          new(
+            instruction: instruction.freeze,
+            few_shot_examples: few_shot_examples.freeze,
+            type: type,
+            metadata: metadata.freeze,
+            config_id: config_id
+          )
         end
 
         sig { returns(T::Hash[Symbol, T.untyped]) }
         def to_h
           {
-            instruction: @instruction,
-            few_shot_examples: @few_shot_examples.size,
-            metadata: @metadata,
-            config_id: @config_id
+            instruction: instruction,
+            few_shot_examples: few_shot_examples.size,
+            type: type.serialize,
+            metadata: metadata,
+            config_id: config_id
           }
         end
-
-        private
-
-        sig { returns(String) }
-        def generate_config_id
-          content = "#{@instruction}_#{@few_shot_examples.size}_#{@metadata.hash}"
-          Digest::SHA256.hexdigest(content)[0, 12]
-        end
       end
 
       # Result of MIPROv2 optimization
       class MIPROv2Result < OptimizationResult
         extend T::Sig
 
-        sig { returns(T::Array[CandidateConfig]) }
+        sig { returns(T::Array[EvaluatedCandidate]) }
         attr_reader :evaluated_candidates
 
         sig { returns(T::Hash[Symbol, T.untyped]) }
@@ -211,7 +200,7 @@ module DSPy
             optimized_program: T.untyped,
             scores: T::Hash[Symbol, T.untyped],
             history: T::Hash[Symbol, T.untyped],
-            evaluated_candidates: T::Array[CandidateConfig],
+            evaluated_candidates: T::Array[EvaluatedCandidate],
             optimization_trace: T::Hash[Symbol, T.untyped],
             bootstrap_statistics: T::Hash[Symbol, T.untyped],
             proposal_statistics: T::Hash[Symbol, T.untyped],
@@ -255,17 +244,25 @@ module DSPy
       sig { returns(T.nilable(DSPy::Propose::GroundedProposer)) }
       attr_reader :proposer
 
-      sig do
-        params(
-          metric: T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T.untyped)),
-          config: T.nilable(MIPROv2Config)
-        ).void
-      end
-      def initialize(metric: nil, config: nil)
-        @mipro_config = config || MIPROv2Config.new
-        super(metric: metric, config: @mipro_config)
+      # Override dry-configurable's initialize to add our parameter validation
+      def initialize(metric: nil, **kwargs)
+        # Reject old config parameter pattern
+        if kwargs.key?(:config)
+          raise ArgumentError, "config parameter is no longer supported. Use .configure blocks instead."
+        end
         
-        @proposer = DSPy::Propose::GroundedProposer.new(config: @mipro_config.proposer_config)
+        # Let dry-configurable handle its initialization
+        super(**kwargs)
+        
+        # Apply class-level configuration if it exists
+        if self.class.default_config_block
+          configure(&self.class.default_config_block)
+        end
+        
+        @metric = metric
+        
+        # Initialize proposer with a basic config for now (will be updated later)  
+        @proposer = DSPy::Propose::GroundedProposer.new(config: DSPy::Propose::GroundedProposer::Config.new)
         @optimization_trace = []
         @evaluated_candidates = []
       end
@@ -284,8 +281,8 @@ module DSPy
         instrument_step('miprov2_compile', {
           trainset_size: trainset.size,
           valset_size: valset&.size || 0,
-          num_trials: @mipro_config.num_trials,
-          optimization_strategy: @mipro_config.optimization_strategy,
+          num_trials: config.num_trials,
+          optimization_strategy: config.optimization_strategy,
           mode: infer_auto_mode
         }) do
           # Convert examples to typed format
@@ -345,11 +342,11 @@ module DSPy
       sig { params(program: T.untyped, trainset: T::Array[DSPy::Example]).returns(Utils::BootstrapResult) }
       def phase_1_bootstrap(program, trainset)
         bootstrap_config = Utils::BootstrapConfig.new
-        bootstrap_config.max_bootstrapped_examples = @mipro_config.max_bootstrapped_examples
-        bootstrap_config.max_labeled_examples = @mipro_config.max_labeled_examples
-        bootstrap_config.num_candidate_sets = @mipro_config.bootstrap_sets
-        bootstrap_config.max_errors = @mipro_config.max_errors
-        bootstrap_config.num_threads = @mipro_config.num_threads
+        bootstrap_config.max_bootstrapped_examples = config.max_bootstrapped_examples
+        bootstrap_config.max_labeled_examples = config.max_labeled_examples
+        bootstrap_config.num_candidate_sets = config.bootstrap_sets
+        bootstrap_config.max_errors = config.max_errors
+        bootstrap_config.num_threads = config.num_threads
 
         Utils.create_n_fewshot_demo_sets(program, trainset, config: bootstrap_config, metric: @metric)
       end
@@ -374,7 +371,7 @@ module DSPy
         raise ArgumentError, "Cannot extract signature class from program" unless signature_class
 
         # Configure proposer for this optimization run
-        @mipro_config.proposer_config.num_instruction_candidates = @mipro_config.num_instruction_candidates
+        @proposer.config.num_instruction_candidates = config.num_instruction_candidates
 
         @proposer.propose_instructions(
           signature_class,
@@ -407,7 +404,7 @@ module DSPy
         best_program = nil
         best_evaluation_result = nil
         
-        @mipro_config.num_trials.times do |trial_idx|
+        config.num_trials.times do |trial_idx|
           trials_completed = trial_idx + 1
           
           # Select next candidate based on optimization strategy
@@ -476,33 +473,39 @@ module DSPy
         params(
           proposal_result: DSPy::Propose::GroundedProposer::ProposalResult,
           bootstrap_result: Utils::BootstrapResult
-        ).returns(T::Array[CandidateConfig])
+        ).returns(T::Array[EvaluatedCandidate])
       end
       def generate_candidate_configurations(proposal_result, bootstrap_result)
         candidates = []
         
         # Base configuration (no modifications)
-        candidates << CandidateConfig.new(
+        candidates << EvaluatedCandidate.new(
           instruction: "",
           few_shot_examples: [],
-          metadata: { type: "baseline" }
+          type: CandidateType::Baseline,
+          metadata: {},
+          config_id: SecureRandom.hex(6)
         )
         
         # Instruction-only candidates
         proposal_result.candidate_instructions.each_with_index do |instruction, idx|
-          candidates << CandidateConfig.new(
+          candidates << EvaluatedCandidate.new(
             instruction: instruction,
             few_shot_examples: [],
-            metadata: { type: "instruction_only", proposal_rank: idx }
+            type: CandidateType::InstructionOnly,
+            metadata: { proposal_rank: idx },
+            config_id: SecureRandom.hex(6)
           )
         end
         
         # Few-shot only candidates
         bootstrap_result.candidate_sets.each_with_index do |candidate_set, idx|
-          candidates << CandidateConfig.new(
+          candidates << EvaluatedCandidate.new(
             instruction: "",
             few_shot_examples: candidate_set,
-            metadata: { type: "few_shot_only", bootstrap_rank: idx }
+            type: CandidateType::FewShotOnly,
+            metadata: { bootstrap_rank: idx },
+            config_id: SecureRandom.hex(6)
           )
         end
         
@@ -512,14 +515,15 @@ module DSPy
         
         top_instructions.each_with_index do |instruction, i_idx|
           top_bootstrap_sets.each_with_index do |candidate_set, b_idx|
-            candidates << CandidateConfig.new(
+            candidates << EvaluatedCandidate.new(
               instruction: instruction,
               few_shot_examples: candidate_set,
+              type: CandidateType::Combined,
               metadata: { 
-                type: "combined", 
                 instruction_rank: i_idx, 
                 bootstrap_rank: b_idx 
-              }
+              },
+              config_id: SecureRandom.hex(6)
             )
           end
         end
@@ -528,13 +532,13 @@ module DSPy
       end
 
       # Initialize optimization state for candidate selection
-      sig { params(candidates: T::Array[CandidateConfig]).returns(T::Hash[Symbol, T.untyped]) }
+      sig { params(candidates: T::Array[EvaluatedCandidate]).returns(T::Hash[Symbol, T.untyped]) }
       def initialize_optimization_state(candidates)
         {
           candidates: candidates,
           scores: {},
           exploration_counts: Hash.new(0),
-          temperature: @mipro_config.init_temperature,
+          temperature: config.init_temperature,
           best_score_history: [],
           diversity_scores: {},
           no_improvement_count: 0
@@ -544,18 +548,18 @@ module DSPy
       # Select next candidate based on optimization strategy
       sig do
         params(
-          candidates: T::Array[CandidateConfig],
+          candidates: T::Array[EvaluatedCandidate],
           state: T::Hash[Symbol, T.untyped],
           trial_idx: Integer
-        ).returns(CandidateConfig)
+        ).returns(EvaluatedCandidate)
       end
       def select_next_candidate(candidates, state, trial_idx)
-        case @mipro_config.optimization_strategy
-        when "greedy"
+        case config.optimization_strategy
+        when OptimizationStrategy::Greedy
           select_candidate_greedy(candidates, state)
-        when "adaptive"
+        when OptimizationStrategy::Adaptive
           select_candidate_adaptive(candidates, state, trial_idx)
-        when "bayesian"
+        when OptimizationStrategy::Bayesian
           select_candidate_bayesian(candidates, state, trial_idx)
         else
           candidates.sample # Random fallback
@@ -563,7 +567,7 @@ module DSPy
       end
 
       # Greedy candidate selection (exploit best known configurations)
-      sig { params(candidates: T::Array[CandidateConfig], state: T::Hash[Symbol, T.untyped]).returns(CandidateConfig) }
+      sig { params(candidates: T::Array[EvaluatedCandidate], state: T::Hash[Symbol, T.untyped]).returns(EvaluatedCandidate) }
       def select_candidate_greedy(candidates, state)
         # Prioritize unexplored candidates, then highest scoring
         unexplored = candidates.reject { |c| state[:scores].key?(c.config_id) }
@@ -577,15 +581,15 @@ module DSPy
       # Adaptive candidate selection (balance exploration and exploitation)
       sig do
         params(
-          candidates: T::Array[CandidateConfig],
+          candidates: T::Array[EvaluatedCandidate],
           state: T::Hash[Symbol, T.untyped],
           trial_idx: Integer
-        ).returns(CandidateConfig)
+        ).returns(EvaluatedCandidate)
       end
       def select_candidate_adaptive(candidates, state, trial_idx)
         # Update temperature based on progress
-        progress = trial_idx.to_f / @mipro_config.num_trials
-        state[:temperature] = @mipro_config.init_temperature * (1 - progress) + @mipro_config.final_temperature * progress
+        progress = trial_idx.to_f / config.num_trials
+        state[:temperature] = config.init_temperature * (1 - progress) + config.final_temperature * progress
         
         # Calculate selection scores combining exploitation and exploration
         candidate_scores = candidates.map do |candidate|
@@ -618,22 +622,86 @@ module DSPy
       # Bayesian candidate selection (use probabilistic model)
       sig do
         params(
-          candidates: T::Array[CandidateConfig],
+          candidates: T::Array[EvaluatedCandidate],
           state: T::Hash[Symbol, T.untyped],
           trial_idx: Integer
-        ).returns(CandidateConfig)
+        ).returns(EvaluatedCandidate)
       end
       def select_candidate_bayesian(candidates, state, trial_idx)
-        # For now, use adaptive selection with Bayesian-inspired exploration
-        # In a full implementation, this would use Gaussian processes or similar
-        select_candidate_adaptive(candidates, state, trial_idx)
+        # Need at least 3 observations to fit GP, otherwise fall back to adaptive
+        return select_candidate_adaptive(candidates, state, trial_idx) if state[:scores].size < 3
+        
+        # Get scored candidates for training the GP
+        scored_candidates = candidates.select { |c| state[:scores].key?(c.config_id) }
+        return select_candidate_adaptive(candidates, state, trial_idx) if scored_candidates.size < 3
+        
+        begin
+          # Encode candidates as numerical features
+          all_candidate_features = encode_candidates_for_gp(candidates)
+          scored_features = encode_candidates_for_gp(scored_candidates)
+          scored_targets = scored_candidates.map { |c| state[:scores][c.config_id].to_f }
+          
+          # Train Gaussian Process
+          gp = DSPy::Optimizers::GaussianProcess.new(
+            length_scale: 1.0,
+            signal_variance: 1.0,
+            noise_variance: 0.01
+          )
+          gp.fit(scored_features, scored_targets)
+          
+          # Predict mean and uncertainty for all candidates
+          means, stds = gp.predict(all_candidate_features, return_std: true)
+          
+          # Upper Confidence Bound (UCB) acquisition function
+          kappa = 2.0 * Math.sqrt(Math.log(trial_idx + 1))  # Exploration parameter
+          acquisition_scores = means.to_a.zip(stds.to_a).map { |m, s| m + kappa * s }
+          
+          # Select candidate with highest acquisition score
+          best_idx = acquisition_scores.each_with_index.max_by { |score, _| score }[1]
+          candidates[best_idx]
+          
+        rescue => e
+          # If GP fails for any reason, fall back to adaptive selection
+          DSPy.logger.warn("Bayesian optimization failed: #{e.message}. Falling back to adaptive selection.")
+          select_candidate_adaptive(candidates, state, trial_idx)
+        end
+      end
+      
+      private
+
+      
+      # Encode candidates as numerical features for Gaussian Process
+      sig { params(candidates: T::Array[EvaluatedCandidate]).returns(T::Array[T::Array[Float]]) }
+      def encode_candidates_for_gp(candidates)
+        # Simple encoding: use hash of config as features
+        # In practice, this could be more sophisticated (e.g., instruction embeddings)
+        candidates.map do |candidate|
+          # Create deterministic numerical features from the candidate config
+          config_hash = candidate.config_id.hash.abs
+          
+          # Extract multiple features to create a feature vector
+          features = []
+          features << (config_hash % 1000).to_f / 1000.0  # Feature 1: hash mod 1000, normalized
+          features << ((config_hash / 1000) % 1000).to_f / 1000.0  # Feature 2: different part of hash
+          features << ((config_hash / 1_000_000) % 1000).to_f / 1000.0  # Feature 3: high bits
+          
+          # Add instruction length if available
+          instruction = candidate.instruction
+          if instruction && !instruction.empty?
+            features << [instruction.length.to_f / 100.0, 2.0].min  # Instruction length, capped at 200 chars
+          else
+            features << 0.5  # Default value
+          end
+          
+          features
+        end
       end
 
       # Evaluate a candidate configuration
       sig do
         params(
           program: T.untyped,
-          candidate: CandidateConfig,
+          candidate: EvaluatedCandidate,
           evaluation_set: T::Array[DSPy::Example]
         ).returns([Float, T.untyped, DSPy::Evaluate::BatchEvaluationResult])
       end
@@ -651,7 +719,7 @@ module DSPy
       end
 
       # Apply candidate configuration to program
-      sig { params(program: T.untyped, candidate: CandidateConfig).returns(T.untyped) }
+      sig { params(program: T.untyped, candidate: EvaluatedCandidate).returns(T.untyped) }
       def apply_candidate_configuration(program, candidate)
         modified_program = program
         
@@ -679,7 +747,7 @@ module DSPy
       sig do
         params(
           state: T::Hash[Symbol, T.untyped],
-          candidate: CandidateConfig,
+          candidate: EvaluatedCandidate,
           score: Float
         ).void
       end
@@ -689,7 +757,7 @@ module DSPy
         state[:best_score_history] << score
         
         # Track diversity if enabled
-        if @mipro_config.track_diversity
+        if config.track_diversity
           state[:diversity_scores][candidate.config_id] = calculate_diversity_score(candidate)
         end
         
@@ -705,14 +773,14 @@ module DSPy
       sig { params(state: T::Hash[Symbol, T.untyped], trial_idx: Integer).returns(T::Boolean) }
       def should_early_stop?(state, trial_idx)
         # Don't stop too early
-        return false if trial_idx < @mipro_config.early_stopping_patience
+        return false if trial_idx < config.early_stopping_patience
         
         # Stop if no improvement for patience trials
-        state[:no_improvement_count] >= @mipro_config.early_stopping_patience
+        state[:no_improvement_count] >= config.early_stopping_patience
       end
 
       # Calculate diversity score for candidate
-      sig { params(candidate: CandidateConfig).returns(Float) }
+      sig { params(candidate: EvaluatedCandidate).returns(Float) }
       def calculate_diversity_score(candidate)
         # Simple diversity metric based on instruction length and few-shot count
         instruction_diversity = candidate.instruction.length / 200.0
@@ -739,8 +807,8 @@ module DSPy
         
         history = {
           total_trials: optimization_result[:trials_completed],
-          optimization_strategy: @mipro_config.optimization_strategy,
-          early_stopped: optimization_result[:trials_completed] < @mipro_config.num_trials,
+          optimization_strategy: config.optimization_strategy,
+          early_stopped: optimization_result[:trials_completed] < config.num_trials,
           score_history: optimization_result[:optimization_state][:best_score_history]
         }
         
@@ -749,7 +817,7 @@ module DSPy
           auto_mode: infer_auto_mode,
           best_instruction: best_candidate&.instruction || "",
           best_few_shot_count: best_candidate&.few_shot_examples&.size || 0,
-          best_candidate_type: best_candidate&.metadata&.fetch(:type, "unknown"),
+          best_candidate_type: best_candidate&.type&.serialize || "unknown",
           optimization_timestamp: Time.now.iso8601
         }
         
@@ -820,7 +888,7 @@ module DSPy
       # Infer auto mode based on configuration
       sig { returns(String) }
       def infer_auto_mode
-        case @mipro_config.num_trials
+        case config.num_trials
         when 0..6 then "light"
         when 7..12 then "medium"
         else "heavy"

data/lib/dspy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DSPy
-  VERSION = "0.25.1"
+  VERSION = "0.26.1"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: dspy
 version: !ruby/object:Gem::Version
-  version: 0.25.1
+  version: 0.26.1
 platform: ruby
 authors:
 - Vicente Reig Rincón de Arellano
 bindir: bin
 cert_chain: []
-date: 2025-09-08 00:00:00.000000000 Z
+date: 2025-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -122,19 +122,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.3'
 - !ruby/object:Gem::Dependency
-  name: polars-df
+  name: numo-narray
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.20.0
+        version: '0.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.20.0
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: informers
   requirement: !ruby/object:Gem::Requirement
@@ -239,6 +239,7 @@ files:
 - lib/dspy/module.rb
 - lib/dspy/observability.rb
 - lib/dspy/observability/async_span_processor.rb
+- lib/dspy/optimizers/gaussian_process.rb
 - lib/dspy/predict.rb
 - lib/dspy/prediction.rb
 - lib/dspy/prompt.rb