dspy 0.28.0 → 0.28.2

data/lib/dspy/teleprompt/mipro_v2.rb

@@ -294,18 +294,18 @@ module DSPy
 
           # Phase 1: Bootstrap few-shot examples
           emit_event('phase_start', { phase: 1, name: 'bootstrap' })
-          bootstrap_result = phase_1_bootstrap(program, typed_trainset)
-          emit_event('phase_complete', { 
-            phase: 1, 
-            success_rate: bootstrap_result.statistics[:success_rate],
-            candidate_sets: bootstrap_result.candidate_sets.size
+          demo_candidates = phase_1_bootstrap(program, typed_trainset)
+          emit_event('phase_complete', {
+            phase: 1,
+            num_predictors: demo_candidates.keys.size,
+            demo_sets_per_predictor: demo_candidates[0]&.size || 0
           })
 
           # Phase 2: Generate instruction candidates
           emit_event('phase_start', { phase: 2, name: 'instruction_proposal' })
-          proposal_result = phase_2_propose_instructions(program, typed_trainset, bootstrap_result)
-          emit_event('phase_complete', { 
-            phase: 2, 
+          proposal_result = phase_2_propose_instructions(program, typed_trainset, demo_candidates)
+          emit_event('phase_complete', {
+            phase: 2,
             num_candidates: proposal_result.num_candidates,
             best_instruction_preview: proposal_result.best_instruction[0, 50]
           })
@@ -316,7 +316,7 @@ module DSPy
             program,
             evaluation_set,
             proposal_result,
-            bootstrap_result
+            demo_candidates
           )
           emit_event('phase_complete', { 
             phase: 3, 
@@ -327,7 +327,7 @@ module DSPy
           # Build final result
           final_result = build_miprov2_result(
             optimization_result,
-            bootstrap_result,
+            demo_candidates,
             proposal_result
           )
 
@@ -339,16 +339,17 @@ module DSPy
       private
 
       # Phase 1: Bootstrap few-shot examples from training data
-      sig { params(program: T.untyped, trainset: T::Array[DSPy::Example]).returns(Utils::BootstrapResult) }
+      # Returns a hash mapping predictor indices to arrays of demo sets
+      sig { params(program: T.untyped, trainset: T::Array[DSPy::Example]).returns(T::Hash[Integer, T::Array[T::Array[DSPy::FewShotExample]]]) }
       def phase_1_bootstrap(program, trainset)
-        bootstrap_config = Utils::BootstrapConfig.new
-        bootstrap_config.max_bootstrapped_examples = config.max_bootstrapped_examples
-        bootstrap_config.max_labeled_examples = config.max_labeled_examples
-        bootstrap_config.num_candidate_sets = config.bootstrap_sets
-        bootstrap_config.max_errors = config.max_errors
-        bootstrap_config.num_threads = config.num_threads
-
-        Utils.create_n_fewshot_demo_sets(program, trainset, config: bootstrap_config, metric: @metric)
+        Utils.create_n_fewshot_demo_sets(
+          program,
+          config.bootstrap_sets,  # num_candidate_sets
+          trainset,
+          max_bootstrapped_demos: config.max_bootstrapped_examples,
+          max_labeled_demos: config.max_labeled_examples,
+          metric: @metric
+        )
       end
 
       # Phase 2: Generate instruction candidates using grounded proposer
@@ -356,22 +357,31 @@ module DSPy
         params(
           program: T.untyped,
           trainset: T::Array[DSPy::Example],
-          bootstrap_result: Utils::BootstrapResult
+          demo_candidates: T::Hash[Integer, T::Array[T::Array[DSPy::FewShotExample]]]
         ).returns(DSPy::Propose::GroundedProposer::ProposalResult)
       end
-      def phase_2_propose_instructions(program, trainset, bootstrap_result)
+      def phase_2_propose_instructions(program, trainset, demo_candidates)
         # Get current instruction if available
         current_instruction = extract_current_instruction(program)
-        
+
         # Use few-shot examples from bootstrap if available
-        few_shot_examples = bootstrap_result.successful_examples.take(5)
+        # Flatten demo sets from first predictor and take first 5 examples
+        few_shot_examples = demo_candidates[0]&.flatten&.take(5) || []
 
         # Get signature class from program
         signature_class = extract_signature_class(program)
         raise ArgumentError, "Cannot extract signature class from program" unless signature_class
 
-        # Configure proposer for this optimization run
-        @proposer.config.num_instruction_candidates = config.num_instruction_candidates
+        # Re-initialize proposer with program and trainset for awareness features
+        # This enables program_aware and use_dataset_summary flags to work correctly
+        proposer_config = DSPy::Propose::GroundedProposer::Config.new
+        proposer_config.num_instruction_candidates = config.num_instruction_candidates
+
+        @proposer = DSPy::Propose::GroundedProposer.new(
+          config: proposer_config,
+          program: program,
+          trainset: trainset
+        )
 
         @proposer.propose_instructions(
           signature_class,
@@ -387,12 +397,12 @@ module DSPy
           program: T.untyped,
           evaluation_set: T::Array[DSPy::Example],
           proposal_result: DSPy::Propose::GroundedProposer::ProposalResult,
-          bootstrap_result: Utils::BootstrapResult
+          demo_candidates: T::Hash[Integer, T::Array[T::Array[DSPy::FewShotExample]]]
         ).returns(T::Hash[Symbol, T.untyped])
       end
-      def phase_3_optimize(program, evaluation_set, proposal_result, bootstrap_result)
+      def phase_3_optimize(program, evaluation_set, proposal_result, demo_candidates)
         # Generate candidate configurations
-        candidates = generate_candidate_configurations(proposal_result, bootstrap_result)
+        candidates = generate_candidate_configurations(proposal_result, demo_candidates)
         
         # Initialize optimization state
         optimization_state = initialize_optimization_state(candidates)
@@ -468,16 +478,16 @@ module DSPy
         }
       end
 
-      # Generate candidate configurations from proposals and bootstrap results
+      # Generate candidate configurations from proposals and demo candidates
       sig do
         params(
           proposal_result: DSPy::Propose::GroundedProposer::ProposalResult,
-          bootstrap_result: Utils::BootstrapResult
+          demo_candidates: T::Hash[Integer, T::Array[T::Array[DSPy::FewShotExample]]]
         ).returns(T::Array[EvaluatedCandidate])
       end
-      def generate_candidate_configurations(proposal_result, bootstrap_result)
+      def generate_candidate_configurations(proposal_result, demo_candidates)
         candidates = []
-        
+
         # Base configuration (no modifications)
         candidates << EvaluatedCandidate.new(
           instruction: "",
@@ -486,7 +496,7 @@ module DSPy
           metadata: {},
           config_id: SecureRandom.hex(6)
         )
-        
+
         # Instruction-only candidates
         proposal_result.candidate_instructions.each_with_index do |instruction, idx|
           candidates << EvaluatedCandidate.new(
@@ -497,12 +507,14 @@ module DSPy
             config_id: SecureRandom.hex(6)
           )
         end
-        
+
         # Few-shot only candidates
-        bootstrap_result.candidate_sets.each_with_index do |candidate_set, idx|
+        # Extract demo sets from first predictor (predictor index 0)
+        demo_sets = demo_candidates[0] || []
+        demo_sets.each_with_index do |demo_set, idx|
           candidates << EvaluatedCandidate.new(
             instruction: "",
-            few_shot_examples: candidate_set,
+            few_shot_examples: demo_set,
             type: CandidateType::FewShotOnly,
             metadata: { bootstrap_rank: idx },
             config_id: SecureRandom.hex(6)
@@ -511,7 +523,7 @@ module DSPy
         
         # Combined candidates (instruction + few-shot)
         top_instructions = proposal_result.candidate_instructions.take(3)
-        top_bootstrap_sets = bootstrap_result.candidate_sets.take(3)
+        top_bootstrap_sets = demo_sets.take(3)
         
         top_instructions.each_with_index do |instruction, i_idx|
           top_bootstrap_sets.each_with_index do |candidate_set, b_idx|
@@ -685,10 +697,10 @@ module DSPy
           features << ((config_hash / 1000) % 1000).to_f / 1000.0  # Feature 2: different part of hash
           features << ((config_hash / 1_000_000) % 1000).to_f / 1000.0  # Feature 3: high bits
           
-          # Add instruction length if available
+          # Add instruction length if available (Python-compatible: no cap)
           instruction = candidate.instruction
           if instruction && !instruction.empty?
-            features << [instruction.length.to_f / 100.0, 2.0].min  # Instruction length, capped at 200 chars
+            features << instruction.length.to_f / 100.0  # Instruction length, uncapped
           else
             features << 0.5  # Default value
           end
@@ -731,11 +743,17 @@ module DSPy
         # Apply few-shot examples if provided
         if candidate.few_shot_examples.any? && program.respond_to?(:with_examples)
           few_shot_examples = candidate.few_shot_examples.map do |example|
-            DSPy::FewShotExample.new(
-              input: example.input_values,
-              output: example.expected_values,
-              reasoning: extract_reasoning_from_example(example)
-            )
+            # If already a FewShotExample, use it directly
+            if example.is_a?(DSPy::FewShotExample)
+              example
+            else
+              # Convert from DSPy::Example
+              DSPy::FewShotExample.new(
+                input: example.input_values,
+                output: example.expected_values,
+                reasoning: extract_reasoning_from_example(example)
+              )
+            end
           end
           modified_program = modified_program.with_examples(few_shot_examples)
         end
@@ -779,39 +797,38 @@ module DSPy
         state[:no_improvement_count] >= config.early_stopping_patience
       end
 
-      # Calculate diversity score for candidate
+      # Calculate diversity score for candidate (Python-compatible: only few-shot count)
       sig { params(candidate: EvaluatedCandidate).returns(Float) }
       def calculate_diversity_score(candidate)
-        # Simple diversity metric based on instruction length and few-shot count
-        instruction_diversity = candidate.instruction.length / 200.0
+        # Python DSPy doesn't use instruction length for diversity, only few-shot count
         few_shot_diversity = candidate.few_shot_examples.size / 10.0
-        
-        [instruction_diversity + few_shot_diversity, 1.0].min
+
+        [few_shot_diversity, 1.0].min
       end
 
       # Build final MIPROv2 result
       sig do
         params(
           optimization_result: T::Hash[Symbol, T.untyped],
-          bootstrap_result: Utils::BootstrapResult,
+          demo_candidates: T::Hash[Integer, T::Array[T::Array[DSPy::FewShotExample]]],
           proposal_result: DSPy::Propose::GroundedProposer::ProposalResult
         ).returns(MIPROv2Result)
       end
-      def build_miprov2_result(optimization_result, bootstrap_result, proposal_result)
+      def build_miprov2_result(optimization_result, demo_candidates, proposal_result)
         best_candidate = optimization_result[:best_candidate]
         best_program = optimization_result[:best_program]
         best_score = optimization_result[:best_score]
         best_evaluation_result = optimization_result[:best_evaluation_result]
-        
+
         scores = { pass_rate: best_score }
-        
+
         history = {
           total_trials: optimization_result[:trials_completed],
           optimization_strategy: config.optimization_strategy,
           early_stopped: optimization_result[:trials_completed] < config.num_trials,
           score_history: optimization_result[:optimization_state][:best_score_history]
         }
-        
+
         metadata = {
           optimizer: "MIPROv2",
           auto_mode: infer_auto_mode,
@@ -820,7 +837,15 @@ module DSPy
           best_candidate_type: best_candidate&.type&.serialize || "unknown",
           optimization_timestamp: Time.now.iso8601
         }
-        
+
+        # Create bootstrap statistics from demo_candidates
+        demo_sets = demo_candidates[0] || []
+        bootstrap_statistics = {
+          num_predictors: demo_candidates.keys.size,
+          demo_sets_per_predictor: demo_sets.size,
+          avg_demos_per_set: demo_sets.empty? ? 0 : demo_sets.map(&:size).sum.to_f / demo_sets.size
+        }
+
         MIPROv2Result.new(
           optimized_program: best_program,
           scores: scores,
@@ -830,7 +855,7 @@ module DSPy
           metadata: metadata,
           evaluated_candidates: @evaluated_candidates,
           optimization_trace: serialize_optimization_trace(optimization_result[:optimization_state]),
-          bootstrap_statistics: bootstrap_result.statistics,
+          bootstrap_statistics: bootstrap_statistics,
           proposal_statistics: proposal_result.analysis,
           best_evaluation_result: best_evaluation_result
         )

data/lib/dspy/teleprompt/simple_optimizer.rb

@@ -142,15 +142,15 @@ module DSPy
           evaluation_set = typed_valset || typed_trainset.take(10)
 
           # Bootstrap few-shot examples if enabled
-          bootstrap_result = nil
+          demo_candidates = nil
           if @optimizer_config.use_few_shot_optimization
-            bootstrap_result = bootstrap_examples(program, typed_trainset)
+            demo_candidates = bootstrap_examples(program, typed_trainset)
           end
 
           # Generate instruction candidates if enabled
           instruction_candidates = []
           if @optimizer_config.use_instruction_optimization && @proposer
-            instruction_candidates = generate_instruction_candidates(program, typed_trainset, bootstrap_result)
+            instruction_candidates = generate_instruction_candidates(program, typed_trainset, demo_candidates)
           end
 
           # Run optimization trials
@@ -158,7 +158,7 @@ module DSPy
             program,
             evaluation_set,
             instruction_candidates,
-            bootstrap_result
+            demo_candidates
           )
 
           # Find best trial
@@ -175,16 +175,18 @@ module DSPy
       private
 
       # Bootstrap few-shot examples from training set
-      sig { params(program: T.untyped, trainset: T::Array[DSPy::Example]).returns(Utils::BootstrapResult) }
+      sig { params(program: T.untyped, trainset: T::Array[DSPy::Example]).returns(T::Hash[Integer, T::Array[T::Array[DSPy::FewShotExample]]]) }
       def bootstrap_examples(program, trainset)
-        bootstrap_config = Utils::BootstrapConfig.new
-        bootstrap_config.max_bootstrapped_examples = @optimizer_config.max_bootstrapped_examples
-        bootstrap_config.max_labeled_examples = @optimizer_config.max_labeled_examples
-        bootstrap_config.num_candidate_sets = [@optimizer_config.num_trials / 2, 5].max
-        bootstrap_config.max_errors = @optimizer_config.max_errors
-        bootstrap_config.num_threads = @optimizer_config.num_threads
-
-        Utils.create_n_fewshot_demo_sets(program, trainset, config: bootstrap_config, metric: @metric)
+        num_candidate_sets = [@optimizer_config.num_trials / 2, 5].max
+
+        Utils.create_n_fewshot_demo_sets(
+          program,
+          num_candidate_sets,
+          trainset,
+          max_bootstrapped_demos: @optimizer_config.max_bootstrapped_examples,
+          max_labeled_demos: @optimizer_config.max_labeled_examples,
+          metric: @metric
+        )
       end
 
       # Generate instruction candidates using the proposer
@@ -192,17 +194,18 @@ module DSPy
         params(
           program: T.untyped,
           trainset: T::Array[DSPy::Example],
-          bootstrap_result: T.nilable(Utils::BootstrapResult)
+          demo_candidates: T.nilable(T::Hash[Integer, T::Array[T::Array[DSPy::FewShotExample]]])
         ).returns(T::Array[String])
       end
-      def generate_instruction_candidates(program, trainset, bootstrap_result)
+      def generate_instruction_candidates(program, trainset, demo_candidates)
         return [] unless @proposer
 
         # Get current instruction if available
         current_instruction = extract_current_instruction(program)
-        
+
         # Use few-shot examples from bootstrap if available
-        few_shot_examples = bootstrap_result&.successful_examples&.take(5)
+        # Flatten demo sets from first predictor and take first 5 examples
+        few_shot_examples = demo_candidates&.dig(0)&.flatten&.take(5) || []
 
         # Get signature class from program
         signature_class = extract_signature_class(program)
@@ -224,14 +227,14 @@ module DSPy
           program: T.untyped,
           evaluation_set: T::Array[DSPy::Example],
           instruction_candidates: T::Array[String],
-          bootstrap_result: T.nilable(Utils::BootstrapResult)
+          demo_candidates: T.nilable(T::Hash[Integer, T::Array[T::Array[DSPy::FewShotExample]]])
         ).returns(T::Array[TrialResult])
       end
-      def run_optimization_trials(program, evaluation_set, instruction_candidates, bootstrap_result)
+      def run_optimization_trials(program, evaluation_set, instruction_candidates, demo_candidates)
         trials = []
-        
+
         # Generate trial configurations
-        trial_configs = generate_trial_configurations(instruction_candidates, bootstrap_result)
+        trial_configs = generate_trial_configurations(instruction_candidates, demo_candidates)
         
         trial_configs.take(@optimizer_config.num_trials).each_with_index do |config, index|
           trial_number = index + 1
@@ -270,36 +273,39 @@ module DSPy
       sig do
         params(
           instruction_candidates: T::Array[String],
-          bootstrap_result: T.nilable(Utils::BootstrapResult)
+          demo_candidates: T.nilable(T::Hash[Integer, T::Array[T::Array[DSPy::FewShotExample]]])
         ).returns(T::Array[T::Hash[Symbol, T.untyped]])
       end
-      def generate_trial_configurations(instruction_candidates, bootstrap_result)
+      def generate_trial_configurations(instruction_candidates, demo_candidates)
         configs = []
-        
+
+        # Extract demo sets from first predictor
+        demo_sets = demo_candidates&.dig(0) || []
+
         # Base configuration (no changes)
         configs << { instruction: nil, few_shot_examples: [] }
-        
+
         # Instruction-only trials
         instruction_candidates.each do |instruction|
           configs << { instruction: instruction, few_shot_examples: [] }
         end
-        
+
         # Few-shot only trials
-        if bootstrap_result&.candidate_sets&.any?
-          bootstrap_result.candidate_sets.each do |candidate_set|
-            configs << { instruction: nil, few_shot_examples: candidate_set }
+        if demo_sets.any?
+          demo_sets.each do |demo_set|
+            configs << { instruction: nil, few_shot_examples: demo_set }
           end
         end
-        
+
         # Combined instruction + few-shot trials
-        if instruction_candidates.any? && bootstrap_result&.candidate_sets&.any?
+        if instruction_candidates.any? && demo_sets.any?
           instruction_candidates.take(3).each do |instruction|
-            bootstrap_result.candidate_sets.take(2).each do |candidate_set|
-              configs << { instruction: instruction, few_shot_examples: candidate_set }
+            demo_sets.take(2).each do |demo_set|
+              configs << { instruction: instruction, few_shot_examples: demo_set }
             end
           end
         end
-        
+
         # Shuffle for random strategy
         if @optimizer_config.search_strategy == "random"
           configs.shuffle