dspy 0.28.1 → 0.28.2

data/lib/dspy/propose/dataset_summary_generator.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+require_relative '../signature'
+require_relative '../predict'
+
+module DSPy
+  module Propose
+    # Dataset Summary Generator for creating concise dataset descriptions
+    # Used by GroundedProposer for data-aware instruction generation
+    module DatasetSummaryGenerator
+      extend T::Sig
+
+      # Signature for summarizing observations into a brief summary
+      class ObservationSummarizer < DSPy::Signature
+        description "Given a series of observations I have made about my dataset, please summarize them into a brief 2-3 sentence summary which highlights only the most important details."
+
+        input do
+          const :observations, String, description: "Observations I have made about my dataset"
+        end
+
+        output do
+          const :summary, String, description: "Two to Three sentence summary of only the most significant highlights of my observations"
+        end
+      end
+
+      # Signature for generating initial dataset observations
+      class DatasetDescriptor < DSPy::Signature
+        description "Given several examples from a dataset please write observations about trends that hold for most or all of the samples. " \
+                   "Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. " \
+                   "It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative"
+
+        input do
+          const :examples, String, description: "Sample data points from the dataset"
+        end
+
+        output do
+          const :observations, String, description: "Somethings that holds true for most or all of the data you observed"
+        end
+      end
+
+      # Signature for refining observations with prior context
+      class DatasetDescriptorWithPriorObservations < DSPy::Signature
+        description "Given several examples from a dataset please write observations about trends that hold for most or all of the samples. " \
+                   "I will also provide you with a few observations I have already made.  Please add your own observations or if you feel the observations are comprehensive say 'COMPLETE' " \
+                   "Some areas you may consider in your observations: topics, content, syntax, conciceness, etc. " \
+                   "It will be useful to make an educated guess as to the nature of the task this dataset will enable. Don't be afraid to be creative"
+
+        input do
+          const :examples, String, description: "Sample data points from the dataset"
+          const :prior_observations, String, description: "Some prior observations I made about the data"
+        end
+
+        output do
+          const :observations, String, description: "Somethings that holds true for most or all of the data you observed or COMPLETE if you have nothing to add"
+        end
+      end
+
+      # Helper function to ensure consistent ordering of input keys in string representations
+      # This helps with caching and consistent LLM prompts
+      sig { params(unordered_repr: String).returns(String) }
+      def self.order_input_keys_in_string(unordered_repr)
+        # Regex pattern to match the input keys structure
+        pattern = /input_keys=\{([^}]+)\}/
+
+        # Function to reorder keys
+        unordered_repr.gsub(pattern) do |match|
+          keys_str = Regexp.last_match(1)
+          # Split the keys, strip extra spaces, and sort them
+          keys = keys_str.split(',').map(&:strip).sort
+          # Format the sorted keys back into the expected structure
+          "input_keys={#{keys.join(', ')}}"
+        end
+      end
+
+      # Strip common prefixes from LLM outputs (e.g., "Answer:", "Output:")
+      sig { params(text: String).returns(String) }
+      def self.strip_prefix(text)
+        # Pattern matches up to 4 words followed by a colon
+        pattern = /^[\*\s]*(([\w'\-]+\s+){0,4}[\w'\-]+):\s*/
+        modified_text = text.gsub(pattern, '')
+        modified_text.strip.gsub(/^["']|["']$/, '')
+      end
+
+      # Generate a concise 2-3 sentence summary of a training dataset
+      # Used for data-aware instruction proposal in MIPROv2
+      #
+      # @param trainset [Array<DSPy::Example>] Training examples to summarize
+      # @param view_data_batch_size [Integer] Number of examples to process per batch
+      # @param prompt_model [DSPy::LM, nil] Language model to use (defaults to DSPy.lm)
+      # @param verbose [Boolean] Whether to print progress information
+      # @return [String] 2-3 sentence summary of the dataset characteristics
+      #
+      # @example Basic usage
+      #   summary = DatasetSummaryGenerator.create_dataset_summary(
+      #     trainset,
+      #     view_data_batch_size: 10,
+      #     prompt_model: DSPy::LM.new('gpt-4o-mini')
+      #   )
+      #
+      sig do
+        params(
+          trainset: T::Array[DSPy::Example],
+          view_data_batch_size: Integer,
+          prompt_model: T.nilable(DSPy::LM),
+          verbose: T::Boolean
+        ).returns(String)
+      end
+      def self.create_dataset_summary(trainset, view_data_batch_size, prompt_model, verbose: false)
+        if verbose
+          puts "\nBootstrapping dataset summary (this will be used to generate instructions)..."
+        end
+
+        # Use provided model or fall back to global LM
+        lm = prompt_model || DSPy.lm
+        raise ArgumentError, "No language model configured. Set prompt_model or DSPy.lm" unless lm
+
+        # Use provided LM in a block context
+        DSPy.with_lm(lm) do
+          # Initial observation from first batch
+          upper_lim = [trainset.length, view_data_batch_size].min
+          examples_repr = order_input_keys_in_string(trainset[0...upper_lim].inspect)
+
+          predictor = DSPy::Predict.new(DatasetDescriptor)
+          observation = predictor.call(examples: examples_repr)
+          observations = observation.observations
+
+          # Iteratively refine observations with additional batches
+          skips = 0
+          max_calls = 10
+          calls = 0
+
+          begin
+            (view_data_batch_size...trainset.length).step(view_data_batch_size) do |b|
+              calls += 1
+              break if calls >= max_calls
+
+              puts "Processing batch starting at index #{b}" if verbose
+
+              upper_lim = [trainset.length, b + view_data_batch_size].min
+              examples_repr = order_input_keys_in_string(trainset[b...upper_lim].inspect)
+
+              predictor = DSPy::Predict.new(DatasetDescriptorWithPriorObservations)
+              output = predictor.call(
+                prior_observations: observations,
+                examples: examples_repr
+              )
+
+              # Check if LLM indicates observations are complete
+              if output.observations.length >= 8 && output.observations[0...8].upcase == "COMPLETE"
+                skips += 1
+                break if skips >= 5
+                next
+              end
+
+              observations += output.observations
+            end
+          rescue => e
+            if verbose
+              puts "Error during observation refinement: #{e.message}. Using observations from past round for summary."
+            end
+          end
+
+          # Generate final summary from accumulated observations
+          predictor = DSPy::Predict.new(ObservationSummarizer)
+          summary = predictor.call(observations: observations)
+
+          if verbose
+            puts "\nGenerated summary: #{strip_prefix(summary.summary)}\n"
+          end
+
+          strip_prefix(summary.summary)
+        end
+      end
+    end
+  end
+end

data/lib/dspy/propose/grounded_proposer.rb

@@ -11,40 +11,78 @@ module DSPy
     class GroundedProposer
       extend T::Sig
 
-      # Configuration for instruction proposal
+      # Python-compatible TIPS dictionary for instruction generation
+      TIPS = {
+        "none" => "",
+        "creative" => "Don't be afraid to be creative when creating the new instruction!",
+        "simple" => "Keep the instruction clear and concise.",
+        "description" => "Make sure your instruction is very informative and descriptive.",
+        "high_stakes" => "The instruction should include a high stakes scenario in which the LM must solve the task!",
+        "persona" => 'Include a persona that is relevant to the task in the instruction (ie. "You are a ...")'
+      }.freeze
+
+      # Configuration for instruction proposal (Python-compatible)
       class Config
         extend T::Sig
 
+        # Core parameters
         sig { returns(Integer) }
         attr_accessor :num_instruction_candidates
 
+        # Python-compatible awareness flags (match Python defaults exactly)
+        sig { returns(T::Boolean) }
+        attr_accessor :program_aware
+
+        sig { returns(T::Boolean) }
+        attr_accessor :use_dataset_summary
+
+        sig { returns(T::Boolean) }
+        attr_accessor :use_task_demos
+
+        sig { returns(T::Boolean) }
+        attr_accessor :use_tip
+
+        sig { returns(T::Boolean) }
+        attr_accessor :use_instruct_history
+
+        # Additional parameters
         sig { returns(Integer) }
-        attr_accessor :max_examples_for_analysis
+        attr_accessor :view_data_batch_size
 
         sig { returns(Integer) }
-        attr_accessor :max_instruction_length
+        attr_accessor :num_demos_in_context
 
         sig { returns(T::Boolean) }
-        attr_accessor :use_task_description
+        attr_accessor :set_tip_randomly
 
         sig { returns(T::Boolean) }
-        attr_accessor :use_input_output_analysis
+        attr_accessor :set_history_randomly
 
-        sig { returns(T::Boolean) }
-        attr_accessor :use_few_shot_examples
+        sig { returns(Float) }
+        attr_accessor :init_temperature
 
-        sig { returns(String) }
-        attr_accessor :proposal_model
+        sig { returns(T::Boolean) }
+        attr_accessor :verbose
 
         sig { void }
         def initialize
+          # Core parameters
           @num_instruction_candidates = 5
-          @max_examples_for_analysis = 10
-          @max_instruction_length = 200
-          @use_task_description = true
-          @use_input_output_analysis = true
-          @use_few_shot_examples = true
-          @proposal_model = "gpt-4o-mini"
+
+          # Python-compatible awareness flags (match Python defaults)
+          @program_aware = true
+          @use_dataset_summary = true
+          @use_task_demos = true
+          @use_tip = true
+          @use_instruct_history = true
+
+          # Additional parameters
+          @view_data_batch_size = 10
+          @num_demos_in_context = 3
+          @set_tip_randomly = true
+          @set_history_randomly = true
+          @init_temperature = 1.0
+          @verbose = false
         end
       end
 
@@ -88,11 +126,66 @@ module DSPy
       sig { returns(Config) }
       attr_reader :config
 
-      sig { params(config: T.nilable(Config)).void }
-      def initialize(config: nil)
+      sig do
+        params(
+          config: T.nilable(Config),
+          program: T.nilable(T.untyped),
+          trainset: T.nilable(T::Array[DSPy::Example])
+        ).void
+      end
+      def initialize(config: nil, program: nil, trainset: nil)
         @config = config || Config.new
+        @program = program
+        @trainset = trainset
+        @dataset_summary = nil
+        @program_code_string = nil
+
+        # Generate dataset summary if data-aware mode enabled (Python: use_dataset_summary)
+        if @config.use_dataset_summary && trainset && !trainset.empty?
+          begin
+            require_relative 'dataset_summary_generator'
+            @dataset_summary = DatasetSummaryGenerator.create_dataset_summary(
+              trainset,
+              @config.view_data_batch_size,
+              DSPy.current_lm,
+              verbose: @config.verbose
+            )
+          rescue => e
+            DSPy.logger.warn("Failed to generate dataset summary: #{e.message}")
+            @dataset_summary = nil
+          end
+        end
+
+        # Extract program source code if program-aware mode enabled
+        if @config.program_aware && program
+          @program_code_string = extract_program_source(program)
+        end
+      end
+
+      private
+
+      # Extract source code from program for program-aware mode
+      sig { params(program: T.untyped).returns(T.nilable(String)) }
+      def extract_program_source(program)
+        # Get the program's class
+        klass = program.is_a?(Class) ? program : program.class
+
+        # Try to get source location
+        source_location = klass.instance_method(:forward).source_location rescue nil
+        return nil unless source_location
+
+        file, line = source_location
+        # Read the source file and extract the class definition
+        # This is a simplified version - could be enhanced with method_source gem
+        code = "Program: #{klass.name}\nSource: #{file}:#{line}"
+        code
+      rescue => e
+        DSPy.logger.warn("Could not extract program source: #{e.message}")
+        nil
       end
 
+      public
+
       # Generate instruction candidates for a signature and training examples
       sig do
         params(
@@ -116,9 +209,10 @@ module DSPy
           
           # Generate instruction candidates
           candidates = generate_instruction_candidates(
-            signature_class, 
-            analysis, 
-            current_instruction
+            signature_class,
+            analysis,
+            current_instruction,
+            few_shot_examples: few_shot_examples
           )
 
           # Filter and rank candidates
@@ -126,8 +220,8 @@ module DSPy
 
           metadata = {
             generation_timestamp: Time.now.iso8601,
-            model_used: @config.proposal_model,
-            num_examples_analyzed: [examples.size, @config.max_examples_for_analysis].min,
+            model_used: DSPy.current_lm.model,
+            num_examples_analyzed: [examples.size, @config.view_data_batch_size].min,
             original_instruction: current_instruction
           }
 
@@ -204,7 +298,7 @@ module DSPy
       # Analyze patterns in training examples
       sig { params(examples: T::Array[T.untyped]).returns(T::Hash[Symbol, T.untyped]) }
       def analyze_example_patterns(examples)
-        analysis_examples = examples.take(@config.max_examples_for_analysis)
+        analysis_examples = examples.take(@config.view_data_batch_size)
         
         {
           total_examples: examples.size,
@@ -323,12 +417,18 @@ module DSPy
         params(
           signature_class: T.class_of(DSPy::Signature),
           analysis: T::Hash[Symbol, T.untyped],
-          current_instruction: T.nilable(String)
+          current_instruction: T.nilable(String),
+          few_shot_examples: T.nilable(T::Array[T.untyped])
         ).returns(T::Array[String])
       end
-      def generate_instruction_candidates(signature_class, analysis, current_instruction)
+      def generate_instruction_candidates(signature_class, analysis, current_instruction, few_shot_examples: nil)
         # Build context for instruction generation
-        context = build_generation_context(signature_class, analysis, current_instruction)
+        context = build_generation_context(
+          signature_class,
+          analysis,
+          current_instruction,
+          few_shot_examples: few_shot_examples
+        )
         
         # Create instruction generation signature
         instruction_signature = create_instruction_generation_signature
@@ -346,16 +446,7 @@ module DSPy
             )
             
             instruction = result.instruction.strip
-            
-            # Truncate if too long
-            if instruction.length > @config.max_instruction_length
-              instruction = instruction[0, @config.max_instruction_length].strip
-              # Try to end at a word boundary
-              if instruction.include?(' ')
-                instruction = instruction.rpartition(' ').first + '.'
-              end
-            end
-            
+
             candidates << instruction if instruction.length > 0
           rescue => error
             DSPy.logger.warn("Failed to generate instruction candidate #{i + 1}: #{error.message}")
@@ -375,32 +466,56 @@ module DSPy
         params(
           signature_class: T.class_of(DSPy::Signature),
           analysis: T::Hash[Symbol, T.untyped],
-          current_instruction: T.nilable(String)
+          current_instruction: T.nilable(String),
+          few_shot_examples: T.nilable(T::Array[T.untyped])
         ).returns(String)
       end
-      def build_generation_context(signature_class, analysis, current_instruction)
+      def build_generation_context(signature_class, analysis, current_instruction, few_shot_examples: nil)
         context_parts = []
-        
-        context_parts << "Task: #{signature_class.description}" if @config.use_task_description
-        
-        if @config.use_input_output_analysis
-          # Build detailed field descriptions including enum values
-          input_descriptions = analysis[:input_fields].map { |f| format_field_description(f) }
-          output_descriptions = analysis[:output_fields].map { |f| format_field_description(f) }
-          
-          context_parts << "Input fields: #{input_descriptions.join(', ')}"
-          context_parts << "Output fields: #{output_descriptions.join(', ')}"
+
+        # Include dataset summary if enabled and available
+        if @config.use_dataset_summary && @dataset_summary
+          context_parts << "Dataset Summary: #{@dataset_summary}"
         end
-        
+
+        # Include program code if enabled and available
+        if @config.program_aware && @program_code_string
+          context_parts << "Program Code:\n#{@program_code_string}"
+        end
+
+        # Always include task description (fundamental to understanding the task)
+        context_parts << "Task: #{signature_class.description}"
+
+        # Always include field analysis (fundamental to understanding inputs/outputs)
+        input_descriptions = analysis[:input_fields].map { |f| format_field_description(f) }
+        output_descriptions = analysis[:output_fields].map { |f| format_field_description(f) }
+
+        context_parts << "Input fields: #{input_descriptions.join(', ')}"
+        context_parts << "Output fields: #{output_descriptions.join(', ')}"
+
+        # Include task demos if enabled and available
+        if @config.use_task_demos && few_shot_examples && !few_shot_examples.empty?
+          demo_strings = few_shot_examples.take(@config.num_demos_in_context).map do |example|
+            format_example_as_demo(example)
+          end
+          context_parts << "Task Demos:\n#{demo_strings.join("\n\n")}"
+        end
+
         if analysis[:common_themes] && analysis[:common_themes].any?
           context_parts << "Task themes: #{analysis[:common_themes].join(', ')}"
         end
-        
+
         if current_instruction
           context_parts << "Current instruction: \"#{current_instruction}\""
         end
-        
-        context_parts.join("\n")
+
+        # Include tip if enabled
+        if @config.use_tip
+          tip = select_tip
+          context_parts << "Tip: #{tip}" if tip && !tip.empty?
+        end
+
+        context_parts.join("\n\n")
       end
 
       # Format field description with enum values if applicable
@@ -414,6 +529,42 @@ module DSPy
         end
       end
 
+      # Format an example as a demo for context
+      sig { params(example: T.untyped).returns(String) }
+      def format_example_as_demo(example)
+        return example.to_s unless example.respond_to?(:inputs) && example.respond_to?(:expected)
+
+        parts = []
+
+        # Format inputs
+        if example.inputs && !example.inputs.empty?
+          input_strs = example.inputs.map { |k, v| "#{k}: #{v.inspect}" }
+          parts << "Inputs: #{input_strs.join(', ')}"
+        end
+
+        # Format expected outputs
+        if example.expected && !example.expected.empty?
+          output_strs = example.expected.map { |k, v| "#{k}: #{v.inspect}" }
+          parts << "Expected: #{output_strs.join(', ')}"
+        end
+
+        parts.join(" | ")
+      end
+
+      # Select a tip based on configuration
+      sig { returns(T.nilable(String)) }
+      def select_tip
+        if @config.set_tip_randomly
+          # Randomly select a tip (excluding "none")
+          tip_keys = TIPS.keys.reject { |k| k == "none" }
+          selected_key = tip_keys.sample
+          TIPS[selected_key]
+        else
+          # Return empty string when not using random tips
+          ""
+        end
+      end
+
       # Build requirements text for instruction generation
       sig { params(analysis: T::Hash[Symbol, T.untyped]).returns(String) }
       def build_requirements_text(analysis)
@@ -478,25 +629,21 @@ module DSPy
         # Filter out duplicates and empty candidates
         filtered = candidates.uniq.reject(&:empty?)
         
-        # Simple ranking based on length and content quality
+        # Simple ranking based on content quality (Python-compatible: no length scoring)
         filtered.sort_by do |instruction|
           score = 0
-          
-          # Prefer moderate length instructions
-          length_score = [instruction.length, @config.max_instruction_length].min / @config.max_instruction_length.to_f
-          score += length_score * 0.3
-          
+
           # Prefer instructions with action words
           action_words = %w[analyze classify generate explain solve determine identify]
           action_score = action_words.count { |word| instruction.downcase.include?(word) }
           score += action_score * 0.4
-          
+
           # Prefer instructions that mention reasoning for complex tasks
           if analysis[:complexity_indicators][:requires_reasoning]
             reasoning_score = instruction.downcase.match?(/\b(step|think|reason|explain)\b/) ? 1 : 0
             score += reasoning_score * 0.3
           end
-          
+
           -score # Negative for descending sort
         end
       end
@@ -588,7 +735,7 @@ module DSPy
           'proposal.num_candidates' => result.num_candidates,
           'proposal.best_instruction_length' => result.best_instruction.length,
           'proposal.analysis_themes' => result.analysis[:common_themes] || [],
-          'proposal.model_used' => @config.proposal_model
+          'proposal.model_used' => DSPy.current_lm.model
         })
       end
     end

data/lib/dspy/teleprompt/bootstrap_strategy.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module DSPy
+  module Teleprompt
+    # Bootstrap strategy enum for create_n_fewshot_demo_sets
+    # Provides type-safe alternatives to Python's magic number seeds
+    class BootstrapStrategy < T::Enum
+      enums do
+        # No demonstrations - zero-shot learning (Python seed = -3)
+        ZeroShot = new
+
+        # Labeled examples only - no bootstrap generation (Python seed = -2)
+        LabeledOnly = new
+
+        # Bootstrapped demonstrations without shuffling (Python seed = -1)
+        Unshuffled = new
+
+        # Bootstrapped demonstrations with shuffling and random size (Python seed >= 0)
+        # Requires separate seed parameter for reproducibility
+        Shuffled = new
+      end
+    end
+  end
+end