dspy 0.29.1 → 0.30.1

data/lib/dspy/{evaluate.rb → evals.rb}

@@ -1,12 +1,16 @@
 # frozen_string_literal: true
 
+require 'json'
+require 'polars'
+require 'concurrent'
 require 'sorbet-runtime'
 require_relative 'example'
+require_relative 'callbacks'
 
 module DSPy
   # Core evaluation framework for DSPy programs
   # Supports single evaluations, batch evaluations, and optimization workflows
-  class Evaluate
+  class Evals
     extend T::Sig
 
     # Result of evaluating a single example
@@ -76,6 +80,9 @@ module DSPy
       sig { returns(Float) }
       attr_reader :pass_rate
 
+      sig { returns(Float) }
+      attr_reader :score
+
       sig do
         params(
           results: T::Array[EvaluationResult],
@@ -88,6 +95,8 @@ module DSPy
         @total_examples = results.length
         @passed_examples = results.count(&:passed)
         @pass_rate = @total_examples > 0 ? @passed_examples.to_f / @total_examples : 0.0
+        score_avg = aggregated_metrics[:score_avg] || @pass_rate
+        @score = (score_avg * 100).round(2)
       end
 
       sig { returns(T::Hash[Symbol, T.untyped]) }
@@ -96,10 +105,47 @@ module DSPy
           total_examples: @total_examples,
           passed_examples: @passed_examples,
           pass_rate: @pass_rate,
+          score: @score,
           aggregated_metrics: @aggregated_metrics,
           results: @results.map(&:to_h)
         }
       end
+
+      sig { returns(Polars::DataFrame) }
+      def to_polars
+        rows = @results.each_with_index.map do |result, index|
+          {
+            "index" => index,
+            "passed" => result.passed,
+            "score" => result.metrics[:score],
+            "example" => serialize_for_polars(result.example),
+            "prediction" => serialize_for_polars(result.prediction),
+            "metrics" => serialize_for_polars(result.metrics),
+            "trace" => serialize_for_polars(result.trace)
+          }
+        end
+
+        Polars::DataFrame.new(rows)
+      end
+
+      private
+
+      def serialize_for_polars(value)
+        case value
+        when NilClass, TrueClass, FalseClass, Numeric, String
+          value
+        when Hash
+          JSON.generate(value)
+        when Array
+          JSON.generate(value)
+        else
+          if value.respond_to?(:to_h)
+            JSON.generate(value.to_h)
+          else
+            value.to_s
+          end
+        end
+      end
     end
 
     sig { returns(T.untyped) }
@@ -117,26 +163,70 @@ module DSPy
     sig { returns(T::Boolean) }
     attr_reader :provide_traceback
 
+    sig { returns(Float) }
+    attr_reader :failure_score
+
+    sig { returns(T.nilable(EvaluationResult)) }
+    attr_reader :last_example_result
+
+    sig { returns(T.nilable(BatchEvaluationResult)) }
+    attr_reader :last_batch_result
+
+    include DSPy::Callbacks
+
+    create_before_callback :call, wrap: false
+    create_after_callback :call, wrap: false
+    create_before_callback :evaluate, wrap: false
+    create_after_callback :evaluate, wrap: false
+
+    class << self
+      def before_example(callback = nil, &block)
+        before(callback, target: :call, &block)
+      end
+
+      def after_example(callback = nil, &block)
+        after(callback, target: :call, &block)
+      end
+
+      def before_batch(callback = nil, &block)
+        before(callback, target: :evaluate, &block)
+      end
+
+      def after_batch(callback = nil, &block)
+        after(callback, target: :evaluate, &block)
+      end
+
+      def reset_callbacks!
+        @callbacks = {}
+      end
+    end
+
     sig do
       params(
         program: T.untyped,
         metric: T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean)),
         num_threads: T.nilable(Integer),
         max_errors: T.nilable(Integer),
+        failure_score: T.nilable(Numeric),
         provide_traceback: T::Boolean
       ).void
     end
-    def initialize(program, metric: nil, num_threads: 1, max_errors: 5, provide_traceback: true)
+    def initialize(program, metric: nil, num_threads: 1, max_errors: 5, failure_score: 0.0, provide_traceback: true)
       @program = program
       @metric = metric
       @num_threads = num_threads || 1
       @max_errors = max_errors || 5
       @provide_traceback = provide_traceback
+      @failure_score = failure_score ? failure_score.to_f : 0.0
+      @last_example_result = nil
+      @last_batch_result = nil
     end
 
     # Evaluate program on a single example
     sig { params(example: T.untyped, trace: T.nilable(T.untyped)).returns(EvaluationResult) }
     def call(example, trace: nil)
+      run_callbacks(:before, :call, example: example)
+
       DSPy::Context.with_span(
         operation: 'evaluation.example',
         'dspy.module' => 'Evaluator',
@@ -144,59 +234,15 @@ module DSPy
         'evaluation.has_metric' => !@metric.nil?
       ) do
         begin
-          # Extract input from example - support both hash and object formats
-          input_values = extract_input_values(example)
-          
-          # Run prediction
-          prediction = @program.call(**input_values)
-          
-          # Calculate metrics if provided
-          metrics = {}
-          passed = true
-          
-          if @metric
-            begin
-              metric_result = @metric.call(example, prediction)
-              if metric_result.is_a?(Hash)
-                metrics = metric_result
-                passed = metrics[:passed] || metrics['passed'] || true
-              else
-                passed = !!metric_result
-                metrics[:passed] = passed
-              end
-            rescue => e
-              passed = false
-              metrics[:error] = e.message
-              metrics[:passed] = false
-            end
-          end
-          
-          EvaluationResult.new(
-            example: example,
-            prediction: prediction,
-            trace: trace,
-            metrics: metrics,
-            passed: passed
-          )
+          perform_call(example, trace: trace)
         rescue => e
-          # Return failed evaluation result
-          error_metrics = {
-            error: e.message,
-            passed: false
-          }
-          
-          if @provide_traceback
-            error_metrics[:traceback] = e.backtrace&.first(10) || []
-          end
-          
-          EvaluationResult.new(
-            example: example,
-            prediction: nil,
-            trace: trace,
-            metrics: error_metrics,
-            passed: false
-          )
+          build_error_result(example, e, trace: trace)
         end
+      end.then do |result|
+        @last_example_result = result
+        emit_example_observation(example, result)
+        run_callbacks(:after, :call, example: example, result: result)
+        result
       end
     end
 
@@ -210,6 +256,8 @@ module DSPy
       ).returns(BatchEvaluationResult)
     end
     def evaluate(devset, display_progress: true, display_table: false, return_outputs: true)
+      run_callbacks(:before, :evaluate, devset: devset)
+
       DSPy::Context.with_span(
         operation: 'evaluation.batch',
         'dspy.module' => 'Evaluator',
@@ -218,56 +266,28 @@ module DSPy
         'evaluation.has_metric' => !@metric.nil?,
         'evaluation.num_threads' => @num_threads
       ) do
-        results = []
-        errors = 0
-        
         if display_progress
           puts "Evaluating #{devset.length} examples..."
         end
-        
-        devset.each_with_index do |example, index|
-          break if errors >= @max_errors
-          
-          begin
-            result = call(example)
-            results << result
-            
-            unless result.passed
-              errors += 1
-            end
-            
-            if display_progress && (index + 1) % 10 == 0
-              puts "Processed #{index + 1}/#{devset.length} examples (#{results.count(&:passed)} passed)"
-            end
-            
-          rescue => e
-            errors += 1
-            puts "Error processing example #{index}: #{e.message}" if display_progress
-            
-            # Create error result
-            error_result = EvaluationResult.new(
-              example: example,
-              prediction: nil,
-              trace: nil,
-              metrics: { error: e.message, passed: false },
-              passed: false
-            )
-            results << error_result
-          end
+
+        results = if parallel_execution?
+          evaluate_in_parallel(devset, display_progress: display_progress)
+        else
+          evaluate_sequential(devset, display_progress: display_progress)
         end
-        
+
         # Aggregate metrics
         aggregated_metrics = aggregate_metrics(results)
-        
+
         batch_result = BatchEvaluationResult.new(
           results: results,
           aggregated_metrics: aggregated_metrics
         )
-        
+
         if display_table
           display_results_table(batch_result)
         end
-        
+
         # Emit batch completion event
         DSPy.log('evaluation.batch_complete', **{
           'evaluation.program_class' => @program.class.name,
@@ -276,17 +296,192 @@ module DSPy
           'evaluation.pass_rate' => batch_result.pass_rate,
           'evaluation.aggregated_metrics' => aggregated_metrics
         })
-        
+
         if display_progress
           puts "Evaluation complete: #{batch_result.passed_examples}/#{batch_result.total_examples} passed (#{(batch_result.pass_rate * 100).round(1)}%)"
         end
-        
+
+        batch_result
+      end.then do |batch_result|
+        @last_batch_result = batch_result
+        emit_batch_observation(devset, batch_result)
+        run_callbacks(:after, :evaluate, devset: devset, result: batch_result)
         batch_result
       end
     end
 
     private
 
+    def parallel_execution?
+      (@num_threads || 1) > 1
+    end
+
+    def evaluate_sequential(devset, display_progress:)
+      results = []
+      errors = 0
+      passed_count = 0
+
+      devset.each_with_index do |example, index|
+        break if errors >= @max_errors
+
+        result = safe_call(example)
+        results << result
+
+        if result.passed
+          passed_count += 1
+        else
+          errors += 1
+        end
+
+        if display_progress && (index + 1) % 10 == 0
+          log_progress(index + 1, devset.length, passed_count)
+        end
+      end
+
+      results
+    end
+
+    def evaluate_in_parallel(devset, display_progress:)
+      total = devset.length
+      results = Array.new(total)
+      errors = 0
+      processed = 0
+      passed_count = 0
+
+      executor = Concurrent::ThreadPoolExecutor.new(
+        min_threads: @num_threads,
+        max_threads: @num_threads,
+        max_queue: [total, 1].max,
+        idletime: 60
+      )
+
+      enumerator = devset.each_with_index
+
+      loop do
+        break if errors >= @max_errors
+
+        batch = []
+        @num_threads.times do
+          begin
+            example = enumerator.next
+            batch << { example: example[0], index: example[1] }
+          rescue StopIteration
+            break
+          end
+        end
+
+        break if batch.empty?
+
+        futures = batch.map do |item|
+          Concurrent::Promises.future_on(executor) do
+            [:ok, item[:index], safe_call(item[:example])]
+          rescue => e
+            [:error, item[:index], e]
+          end
+        end
+
+        futures.each do |future|
+          status, index, payload = future.value!
+          example = batch.find { |entry| entry[:index] == index }[:example]
+
+          result = if status == :ok
+            payload
+          else
+            errors += 1
+            puts "Error processing example #{index}: #{payload.message}" if display_progress
+            build_error_result(example, payload)
+          end
+
+          results[index] = result
+          processed += 1
+          if result.passed
+            passed_count += 1
+          else
+            errors += 1 unless status == :error
+          end
+
+          if display_progress && (processed % 10).zero?
+            log_progress(processed, total, passed_count)
+          end
+        end
+      end
+
+      executor.shutdown
+      executor.wait_for_termination
+
+      results.compact
+    end
+
+    def safe_call(example)
+      call(example)
+    rescue => e
+      build_error_result(example, e)
+    end
+
+    def perform_call(example, trace:)
+      # Extract input from example - support both hash and object formats
+      input_values = extract_input_values(example)
+
+      # Run prediction
+      prediction = @program.call(**input_values)
+
+      # Calculate metrics if provided
+      metrics = {}
+      passed = true
+
+      if @metric
+        begin
+          metric_result = @metric.call(example, prediction)
+          if metric_result.is_a?(Hash)
+            metrics = symbolize_keys(metric_result)
+            passed_flag = metrics.key?(:passed) ? metrics[:passed] : metrics['passed']
+            passed = passed_flag.nil? ? true : !!passed_flag
+          else
+            passed = !!metric_result
+            metrics[:passed] = passed
+          end
+        rescue => e
+          passed = false
+          metrics[:error] = e.message
+          metrics[:passed] = false
+          metrics[:score] = @failure_score
+        end
+      end
+
+      metrics[:passed] = passed unless metrics.key?(:passed)
+      metrics[:score] = normalize_score(metrics[:score], passed) if metrics.key?(:score)
+      metrics[:score] ||= passed ? 1.0 : 0.0
+
+      EvaluationResult.new(
+        example: example,
+        prediction: prediction,
+        trace: trace,
+        metrics: metrics,
+        passed: passed
+      )
+    end
+
+    def build_error_result(example, error, trace: nil)
+      metrics = {
+        error: error.message,
+        passed: false,
+        score: @failure_score
+      }
+      metrics[:traceback] = error.backtrace&.first(10) || [] if @provide_traceback
+
+      EvaluationResult.new(
+        example: example,
+        prediction: nil,
+        trace: trace,
+        metrics: metrics,
+        passed: false
+      )
+    end
+
+    def log_progress(processed, total, passed_count)
+      puts "Processed #{processed}/#{total} examples (#{passed_count} passed)"
+    end
+
     # Extract input values from example in various formats
     sig { params(example: T.untyped).returns(T::Hash[Symbol, T.untyped]) }
     def extract_input_values(example)
@@ -376,36 +571,49 @@ module DSPy
     def aggregate_metrics(results)
       return {} if results.empty?
       
-      # Start with basic metrics
+      total = results.length
+      passed = results.count(&:passed)
+
       aggregated = {
-        total_examples: results.length,
-        passed_examples: results.count(&:passed),
+        total_examples: total,
+        passed_examples: passed,
         failed_examples: results.count { |r| !r.passed }
       }
-      
-      # Aggregate numeric metrics
+
+      score_values = results.filter_map do |result|
+        score = result.metrics[:score]
+        score if score.is_a?(Numeric)
+      end
+
+      if score_values.any?
+        aggregated[:score_sum] = score_values.sum
+        aggregated[:score_avg] = score_values.sum.to_f / score_values.length
+        aggregated[:score_min] = score_values.min
+        aggregated[:score_max] = score_values.max
+      else
+        aggregated[:score_avg] = passed.positive? && total.positive? ? passed.to_f / total : 0.0
+      end
+
+      # Aggregate other numeric metrics
       numeric_metrics = {}
       results.each do |result|
         result.metrics.each do |key, value|
-          next if [:error, :traceback, :passed].include?(key)
+          next if [:error, :traceback, :passed, :score].include?(key)
           next unless value.is_a?(Numeric)
-          
+
           numeric_metrics[key] ||= []
           numeric_metrics[key] << value
         end
       end
-      
-      # Calculate averages for numeric metrics
+
       numeric_metrics.each do |key, values|
         aggregated[:"#{key}_avg"] = values.sum.to_f / values.length
         aggregated[:"#{key}_min"] = values.min
         aggregated[:"#{key}_max"] = values.max
       end
-      
-      # Calculate pass rate
-      aggregated[:pass_rate] = aggregated[:total_examples] > 0 ? 
-        aggregated[:passed_examples].to_f / aggregated[:total_examples] : 0.0
-      
+
+      aggregated[:pass_rate] = total.positive? ? passed.to_f / total : 0.0
+
       aggregated
     end
 
@@ -429,6 +637,61 @@ module DSPy
       
       puts "=" * 50
     end
+
+    def emit_example_observation(example, result)
+      DSPy.event('evals.example.complete', {
+        program: @program.class.name,
+        example_id: extract_example_id(example),
+        passed: result.passed,
+        score: result.metrics[:score],
+        error: result.metrics[:error]
+      })
+    rescue => e
+      DSPy.log('evals.example.observation_error', error: e.message)
+    end
+
+    def emit_batch_observation(devset, batch_result)
+      DSPy.event('evals.batch.complete', {
+        program: @program.class.name,
+        dataset_size: devset.length,
+        total_examples: batch_result.total_examples,
+        passed_examples: batch_result.passed_examples,
+        pass_rate: batch_result.pass_rate,
+        score: batch_result.score
+      })
+    rescue => e
+      DSPy.log('evals.batch.observation_error', error: e.message)
+    end
+
+    def extract_example_id(example)
+      if example.respond_to?(:id)
+        example.id
+      elsif example.is_a?(Hash)
+        example[:id] || example['id']
+      else
+        nil
+      end
+    rescue
+      nil
+    end
+
+    def symbolize_keys(hash)
+      hash.each_with_object({}) do |(key, value), memo|
+        memo[key.respond_to?(:to_sym) ? key.to_sym : key] = value
+      end
+    end
+
+    def normalize_score(value, passed)
+      case value
+      when Numeric
+        value.to_f
+      when TrueClass, FalseClass
+        value ? 1.0 : 0.0
+      else
+        passed ? 1.0 : 0.0
+      end
+    end
+
   end
 
   # Common metric functions for evaluation
@@ -447,7 +710,7 @@ module DSPy
         expected = extract_field(example, field)
         actual = extract_field(prediction, field)
         
-        return false if expected.nil? || actual.nil?
+        next false if expected.nil? || actual.nil?
         
         if case_sensitive
           expected.to_s == actual.to_s
@@ -469,7 +732,7 @@ module DSPy
         expected = extract_field(example, field)
         actual = extract_field(prediction, field)
         
-        return false if expected.nil? || actual.nil?
+        next false if expected.nil? || actual.nil?
         
         if case_sensitive
           actual.to_s.include?(expected.to_s)
@@ -491,7 +754,7 @@ module DSPy
         expected = extract_field(example, field)
         actual = extract_field(prediction, field)
         
-        return { passed: false, error: "Missing values" } if expected.nil? || actual.nil?
+        next { passed: false, error: "Missing values" } if expected.nil? || actual.nil?
         
         begin
           expected_num = Float(expected)
@@ -554,4 +817,4 @@ module DSPy
       end
     end
   end
-end
+end

data/lib/dspy/mixins/instruction_updatable.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+require_relative '../errors'
+
+module DSPy
+  module Mixins
+    module InstructionUpdatable
+      extend T::Sig
+
+      sig { params(new_instruction: String).returns(T.untyped) }
+      def with_instruction(new_instruction)
+        raise DSPy::InstructionUpdateError.missing_instruction_capability(self.class)
+      end
+
+      sig { params(few_shot_examples: T::Array[T.untyped]).returns(T.untyped) }
+      def with_examples(few_shot_examples)
+        raise DSPy::InstructionUpdateError.missing_examples_capability(self.class)
+      end
+    end
+  end
+end