dspy 0.29.0 → 0.29.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 747119ce407283e4d8ed5f01014262f24a94418ad2cbef4305a28b21cb58c8bc
-  data.tar.gz: 3693faccd1fca98015864fd4404491b619b2aa600ab83a78dd3fc7d9e3342ef1
+  metadata.gz: 42a6bfa4a1e0fe7b7dd38e3a2fe017959365f566248319e1deec6cadcd06efc3
+  data.tar.gz: ffceb724c96e7803ec7531bf52c12f502266e21d4ab9bcf534a8005e59835883
 SHA512:
-  metadata.gz: 7fecac3bc3389e11bdb2328234455cbbbcbf6c7544518cbf94082e75a3f0bde489339b70f91dfd31a43bbd87b17451bfb5a78387c90f5b669aefa09e8af73500
-  data.tar.gz: 01feee252179dd66016a8633658631dacc2262f5598ee07d4f78ec2947ebbb57cd15b19b433b8ea9a87b7120fbf5afa984fd929677de0fcfaf7a4368e5b29d85
+  metadata.gz: 2d81f0ab86954523f8b3511f3105f8005b9709f0a46fb08f0403dcac841c70efcfbff90dcdedccaf322554280f62bdbab6a36be34d0e787f01734d8ee2219bf6
+  data.tar.gz: 31d048ad3d7493be0c52a24729d033ce58a891039c180945aa80d54afb7a529b6e420ccfc49fdab32ef00803483ecbb63dd2c2bd9d15f569c44debfdfd6dea0c

data/README.md

@@ -111,7 +111,7 @@ end
 - **Prompt Objects** - Manipulate prompts as first-class objects instead of strings
 - **Typed Examples** - Type-safe training data with automatic validation
 - **Evaluation Framework** - Advanced metrics beyond simple accuracy with error-resilient pipelines
-- **MIPROv2 Optimization** - Advanced Bayesian optimization with Gaussian Processes, multiple optimization strategies, and storage persistence
+- **MIPROv2 Optimization** - Advanced Bayesian optimization with Gaussian Processes, multiple optimization strategies, auto-config presets, and storage persistence
 
 **Production Features:**
 - **Reliable JSON Extraction** - Native structured outputs for OpenAI and Gemini, Anthropic tool-based extraction, and automatic strategy selection with fallback

data/lib/dspy/teleprompt/mipro_v2.rb

@@ -2,9 +2,11 @@
 
 require 'digest'
 require 'time'
+require 'json'
 require 'concurrent-ruby'
 require 'sorbet-runtime'
 require 'securerandom'
+require 'set'
 require_relative 'teleprompter'
 require_relative 'utils'
 require_relative '../propose/grounded_proposer'
@@ -30,6 +32,58 @@ module DSPy
         Bayesian = new("bayesian")
       end
     end
+
+    class AutoPreset < T::Enum
+      enums do
+        None = new("none")
+        Light = new("light")
+        Medium = new("medium")
+        Heavy = new("heavy")
+      end
+    end
+
+    AUTO_PRESET_SETTINGS = {
+      AutoPreset::None => {},
+      AutoPreset::Light => {
+        candidate_budget: 6,
+        instruction_candidates: 3,
+        instruction_candidates_when_fewshot: 3,
+        bootstrap_sets: 3,
+        max_bootstrapped_examples: 2,
+        max_labeled_examples: 8,
+        optimization_strategy: OptimizationStrategy::Greedy,
+        early_stopping_patience: 2,
+        valset_target_size: 100,
+        minibatch_size: nil
+      },
+      AutoPreset::Medium => {
+        candidate_budget: 12,
+        instruction_candidates: 5,
+        instruction_candidates_when_fewshot: 5,
+        bootstrap_sets: 5,
+        max_bootstrapped_examples: 4,
+        max_labeled_examples: 16,
+        optimization_strategy: OptimizationStrategy::Adaptive,
+        early_stopping_patience: 3,
+        valset_target_size: 300,
+        minibatch_size: nil
+      },
+      AutoPreset::Heavy => {
+        candidate_budget: 18,
+        instruction_candidates: 8,
+        instruction_candidates_when_fewshot: 8,
+        bootstrap_sets: 8,
+        max_bootstrapped_examples: 6,
+        max_labeled_examples: 24,
+        optimization_strategy: OptimizationStrategy::Bayesian,
+        early_stopping_patience: 5,
+        valset_target_size: 1000,
+        minibatch_size: nil
+      }
+    }.freeze
+
+    DEFAULT_AUTO_SEED = 42
+
     # MIPROv2: Multi-prompt Instruction Proposal with Retrieval Optimization
     # State-of-the-art prompt optimization combining bootstrap sampling, 
     # instruction generation, and Bayesian optimization
@@ -50,13 +104,7 @@ module DSPy
         def self.light(metric: nil, **kwargs)
           optimizer = MIPROv2.new(metric: metric, **kwargs)
           optimizer.configure do |config|
-            config.num_trials = 6
-            config.num_instruction_candidates = 3
-            config.max_bootstrapped_examples = 2
-            config.max_labeled_examples = 8
-            config.bootstrap_sets = 3
-            config.optimization_strategy = :greedy
-            config.early_stopping_patience = 2
+            MIPROv2.apply_auto_defaults(config, AutoPreset::Light)
           end
           optimizer
         end
@@ -70,13 +118,7 @@ module DSPy
         def self.medium(metric: nil, **kwargs)
           optimizer = MIPROv2.new(metric: metric, **kwargs)
           optimizer.configure do |config|
-            config.num_trials = 12
-            config.num_instruction_candidates = 5
-            config.max_bootstrapped_examples = 4
-            config.max_labeled_examples = 16
-            config.bootstrap_sets = 5
-            config.optimization_strategy = :adaptive
-            config.early_stopping_patience = 3
+            MIPROv2.apply_auto_defaults(config, AutoPreset::Medium)
           end
           optimizer
         end
@@ -90,19 +132,33 @@ module DSPy
         def self.heavy(metric: nil, **kwargs)
           optimizer = MIPROv2.new(metric: metric, **kwargs)
           optimizer.configure do |config|
-            config.num_trials = 18
-            config.num_instruction_candidates = 8
-            config.max_bootstrapped_examples = 6
-            config.max_labeled_examples = 24
-            config.bootstrap_sets = 8
-            config.optimization_strategy = :bayesian
-            config.early_stopping_patience = 5
+            MIPROv2.apply_auto_defaults(config, AutoPreset::Heavy)
           end
           optimizer
         end
       end
 
       # Dry-configurable settings for MIPROv2
+      setting :auto_preset, default: AutoPreset::None, constructor: ->(value) {
+        case value
+        when AutoPreset
+          value
+        when String, Symbol
+          begin
+            AutoPreset.deserialize(value.to_s.downcase)
+          rescue ArgumentError
+            raise ArgumentError, "Invalid auto preset: #{value}. Must be one of :none, :light, :medium, :heavy"
+          end
+        when nil
+          AutoPreset::None
+        else
+          raise ArgumentError, "Invalid auto preset: #{value.inspect}"
+        end
+      }
+      setting :auto_seed, default: DEFAULT_AUTO_SEED, constructor: ->(value) {
+        value.nil? ? DEFAULT_AUTO_SEED : Integer(value)
+      }
+      setting :valset_target_size, default: nil
       setting :num_trials, default: 12
       setting :num_instruction_candidates, default: 5
       setting :bootstrap_sets, default: 5
@@ -142,6 +198,26 @@ module DSPy
         @default_config_block
       end
 
+      class << self
+        extend T::Sig
+
+        sig { params(config: T.untyped, preset: AutoPreset).void }
+        def apply_auto_defaults(config, preset)
+          settings = AUTO_PRESET_SETTINGS.fetch(preset) { {} }
+
+          config.auto_preset = preset
+          config.num_trials = settings[:candidate_budget] if settings[:candidate_budget]
+          config.num_instruction_candidates = settings[:instruction_candidates] if settings[:instruction_candidates]
+          config.bootstrap_sets = settings[:bootstrap_sets] if settings[:bootstrap_sets]
+          config.max_bootstrapped_examples = settings[:max_bootstrapped_examples] if settings.key?(:max_bootstrapped_examples)
+          config.max_labeled_examples = settings[:max_labeled_examples] if settings.key?(:max_labeled_examples)
+          config.optimization_strategy = settings[:optimization_strategy] if settings[:optimization_strategy]
+          config.early_stopping_patience = settings[:early_stopping_patience] if settings[:early_stopping_patience]
+          config.minibatch_size = settings[:minibatch_size] if settings.key?(:minibatch_size)
+          config.valset_target_size = settings[:valset_target_size] if settings[:valset_target_size]
+        end
+      end
+
 
       # Simple data structure for evaluated candidate configurations (immutable)
       EvaluatedCandidate = Data.define(
@@ -294,6 +370,13 @@ module DSPy
           typed_trainset = ensure_typed_examples(trainset)
           typed_valset = valset ? ensure_typed_examples(valset) : nil
 
+          if auto_preset_active?
+            typed_trainset, typed_valset = prepare_datasets_for_auto(typed_trainset, typed_valset)
+            typed_valset = apply_auto_preset!(program, typed_valset)
+          else
+            typed_valset = limit_validation_set(typed_valset, config.valset_target_size)
+          end
+
           # Use validation set if available, otherwise use part of training set
           evaluation_set = typed_valset || typed_trainset.take([typed_trainset.size / 3, 10].max)
 
@@ -345,6 +428,105 @@ module DSPy
 
       private
 
+      sig { returns(T::Boolean) }
+      def auto_preset_active?
+        config.auto_preset != AutoPreset::None
+      end
+
+      sig { params(trainset: T::Array[DSPy::Example], valset: T.nilable(T::Array[DSPy::Example])).returns([T::Array[DSPy::Example], T::Array[DSPy::Example]]) }
+      def prepare_datasets_for_auto(trainset, valset)
+        settings = auto_settings_for(config.auto_preset)
+        target_size = settings[:valset_target_size]
+        config.valset_target_size = target_size
+
+        if valset && valset.any?
+          [trainset, limit_validation_set(valset, target_size)]
+        else
+          raise ArgumentError, "Training set must contain at least 2 examples when auto presets are enabled" if trainset.size < 2
+
+          shuffled = trainset.shuffle(random: Random.new(config.auto_seed))
+          default_val_size = [
+            [(trainset.size * 0.8).ceil, 1].max,
+            trainset.size - 1
+          ].min
+
+          desired_val_size = target_size ? [default_val_size, target_size].min : default_val_size
+          desired_val_size = [[desired_val_size, 1].max, trainset.size - 1].min
+
+          validation_examples = shuffled.take(desired_val_size)
+          training_examples = shuffled.drop(desired_val_size)
+
+          [training_examples, limit_validation_set(validation_examples, target_size)]
+        end
+      end
+
+      sig { params(program: T.untyped, valset: T::Array[DSPy::Example]).returns(T::Array[DSPy::Example]) }
+      def apply_auto_preset!(program, valset)
+        settings = auto_settings_for(config.auto_preset)
+        zeroshot = zero_shot_for_settings?(settings)
+        candidate_budget = settings[:candidate_budget]
+
+        if candidate_budget && candidate_budget.positive?
+          config.num_trials = compute_trials_from_candidate_budget(program, candidate_budget, zeroshot)
+          instruction_candidates = if zeroshot
+            candidate_budget
+          else
+            settings[:instruction_candidates_when_fewshot] || (candidate_budget / 2.0).ceil
+          end
+          config.num_instruction_candidates = [instruction_candidates, 1].max
+        end
+
+        config.bootstrap_sets = settings[:bootstrap_sets] if settings[:bootstrap_sets]
+        config.max_bootstrapped_examples = settings[:max_bootstrapped_examples] if settings.key?(:max_bootstrapped_examples)
+        config.max_labeled_examples = settings[:max_labeled_examples] if settings.key?(:max_labeled_examples)
+        config.optimization_strategy = settings[:optimization_strategy] if settings[:optimization_strategy]
+        config.early_stopping_patience = settings[:early_stopping_patience] if settings[:early_stopping_patience]
+        config.minibatch_size = settings[:minibatch_size] if settings.key?(:minibatch_size)
+
+        config.valset_target_size = settings[:valset_target_size]
+        limit_validation_set(valset, config.valset_target_size)
+      end
+
+      sig { params(valset: T.nilable(T::Array[DSPy::Example]), target_size: T.nilable(Integer)).returns(T.nilable(T::Array[DSPy::Example])) }
+      def limit_validation_set(valset, target_size)
+        return valset unless valset && target_size && target_size.positive?
+        return valset if valset.size <= target_size
+
+        valset.shuffle(random: Random.new(config.auto_seed)).take(target_size)
+      end
+
+      sig { params(program: T.untyped, num_candidates: Integer, zeroshot: T::Boolean).returns(Integer) }
+      def compute_trials_from_candidate_budget(program, num_candidates, zeroshot)
+        predictor_count =
+          if program.respond_to?(:predictors)
+            Array(program.predictors).size
+          else
+            1
+          end
+
+        predictor_count = 1 if predictor_count.zero?
+        variable_count = zeroshot ? predictor_count : predictor_count * 2
+        log_term = Math.log2([num_candidates, 2].max)
+
+        [
+          (2 * variable_count * log_term).ceil,
+          (1.5 * num_candidates).ceil
+        ].max
+      end
+
+      sig { params(settings: T::Hash[Symbol, T.untyped]).returns(T::Boolean) }
+      def zero_shot_for_settings?(settings)
+        settings.fetch(:max_bootstrapped_examples, 0).to_i.zero? &&
+          settings.fetch(:max_labeled_examples, 0).to_i.zero?
+      end
+
+      sig { params(preset: AutoPreset).returns(T::Hash[Symbol, T.untyped]) }
+      def auto_settings_for(preset)
+        AUTO_PRESET_SETTINGS.fetch(preset) do
+          raise ArgumentError, "Unknown auto preset: #{preset.inspect}"
+        end
+      end
+
       # Phase 1: Bootstrap few-shot examples from training data
       # Returns a hash mapping predictor indices to arrays of demo sets
       sig { params(program: T.untyped, trainset: T::Array[DSPy::Example]).returns(T::Hash[Integer, T::Array[T::Array[DSPy::FewShotExample]]]) }
@@ -546,6 +728,21 @@ module DSPy
       end
       def generate_candidate_configurations(proposal_result, demo_candidates)
         candidates = []
+        seen_signatures = Set.new
+
+        add_candidate = lambda do |instruction:, few_shot_examples:, type:, metadata:, config_id:|
+          signature = candidate_signature(type, instruction, metadata, few_shot_examples)
+          next if seen_signatures.include?(signature)
+
+          seen_signatures << signature
+          candidates << EvaluatedCandidate.new(
+            instruction: instruction,
+            few_shot_examples: few_shot_examples,
+            type: type,
+            metadata: metadata,
+            config_id: config_id
+          )
+        end
 
         predictor_instruction_map = if proposal_result.respond_to?(:predictor_instructions) && proposal_result.predictor_instructions.any?
           proposal_result.predictor_instructions
@@ -557,7 +754,7 @@ module DSPy
         demo_maps = build_demo_maps(demo_candidates)
 
         # Base configuration (no modifications)
-        candidates << EvaluatedCandidate.new(
+        add_candidate.call(
           instruction: "",
           few_shot_examples: [],
           type: CandidateType::Baseline,
@@ -570,7 +767,7 @@ module DSPy
 
         instruction_maps.each_with_index do |instruction_map, combo_idx|
           primary_instruction = instruction_map[0] || instruction_map.values.first || ""
-          candidates << EvaluatedCandidate.new(
+          add_candidate.call(
             instruction: primary_instruction,
             few_shot_examples: [],
             type: CandidateType::InstructionOnly,
@@ -587,7 +784,7 @@ module DSPy
           next if demo_map.empty?
 
           flattened_examples = demo_map.values.flatten
-          candidates << EvaluatedCandidate.new(
+          add_candidate.call(
             instruction: "",
             few_shot_examples: flattened_examples,
             type: CandidateType::FewShotOnly,
@@ -607,7 +804,7 @@ module DSPy
             next if demo_map.empty?
 
             flattened_examples = demo_map.values.flatten
-            candidates << EvaluatedCandidate.new(
+            add_candidate.call(
               instruction: primary_instruction,
               few_shot_examples: flattened_examples,
               type: CandidateType::Combined,
@@ -687,6 +884,55 @@ module DSPy
         end
       end
 
+      sig do
+        params(
+          type: CandidateType,
+          instruction: String,
+          metadata: T::Hash[Symbol, T.untyped],
+          few_shot_examples: T::Array[T.untyped]
+        ).returns(String)
+      end
+      def candidate_signature(type, instruction, metadata, few_shot_examples)
+        JSON.generate(
+          type: type.serialize,
+          instruction: instruction,
+          instructions_map: normalize_instruction_map(metadata[:instructions_map] || {}),
+          demos_map: normalize_demo_map(metadata[:demos_map] || {}),
+          few_shot_examples: few_shot_examples.map { |example| serialize_few_shot_example(example) }
+        )
+      end
+
+      sig { params(map: T::Hash[Integer, T.untyped]).returns(T::Hash[Integer, String]) }
+      def normalize_instruction_map(map)
+        map.sort_by { |index, _| index }.each_with_object({}) do |(index, value), memo|
+          memo[index] = value.to_s
+        end
+      end
+
+      sig { params(map: T::Hash[Integer, T::Array[T.untyped]]).returns(T::Hash[Integer, T::Array[T.untyped]]) }
+      def normalize_demo_map(map)
+        map.sort_by { |index, _| index }.each_with_object({}) do |(index, demos), memo|
+          memo[index] = Array(demos).map { |demo| serialize_few_shot_example(demo) }
+        end
+      end
+
+      sig { params(example: T.untyped).returns(T.untyped) }
+      def serialize_few_shot_example(example)
+        case example
+        when DSPy::FewShotExample
+          deep_dup(example.to_h)
+        when DSPy::Example
+          {
+            input: deep_dup(example.input_values),
+            expected: deep_dup(example.expected_values)
+          }
+        when Hash
+          deep_dup(example)
+        else
+          example
+        end
+      end
+
       sig { params(examples: T::Array[T.untyped]).returns(T::Array[DSPy::FewShotExample]) }
       def normalize_few_shot_examples(examples)
         examples.map do |example|
@@ -1412,10 +1658,13 @@ module DSPy
       # Infer auto mode based on configuration
       sig { returns(String) }
       def infer_auto_mode
+        return config.auto_preset.serialize unless config.auto_preset == AutoPreset::None
+
         case config.num_trials
         when 0..6 then "light"
         when 7..12 then "medium"
-        else "heavy"
+        when 13..Float::INFINITY then "heavy"
+        else "manual"
         end
       end
     end

data/lib/dspy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DSPy
-  VERSION = "0.29.0"
+  VERSION = "0.29.1"
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: dspy
 version: !ruby/object:Gem::Version
-  version: 0.29.0
+  version: 0.29.1
 platform: ruby
 authors:
 - Vicente Reig Rincón de Arellano
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2025-10-19 00:00:00.000000000 Z
+date: 2025-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-configurable
@@ -321,7 +320,6 @@ homepage: https://github.com/vicentereig/dspy.rb
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -336,8 +334,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key: 
+rubygems_version: 3.6.5
 specification_version: 4
 summary: The Ruby framework for programming—rather than prompting—language models.
 test_files: []