dspy-evals 0.29.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f39ede0bd93df0362c4cf8205ed8c5340cd52100cb0ba83a650f39583e496d76
+  data.tar.gz: 96eafcbb25a32b13d4c5b18e1685e7867f71e97fb59ae2aaa20a7aa4940d0db7
+SHA512:
+  metadata.gz: 8f2f94c8cc7f3660a4083a04cf6e4720d473baab59bc3dad06671068c8003731bd256859438ba13f575815c09c098fca4e24bd6b322ac9b07ad0d6c196e3ec1e
+  data.tar.gz: 863806288464a5859e8b9ee04b8ac19def6820ce31a88dfe5fc7b32605a638cb509f67d25b5dfd96641a09cac44627f4d42531489118d0c41c4ca597a6895a57

data/LICENSE

@@ -0,0 +1,45 @@
+MIT License
+
+Copyright (c) 2025 Vicente Services SL
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+This project is a Ruby port of the original Python [DSPy library](https://github.com/stanfordnlp/dspy), which is licensed under the MIT License:
+
+MIT License
+
+Copyright (c) 2023 Stanford Future Data Systems
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,249 @@
+# DSPy.rb
+
+[![Gem Version](https://img.shields.io/gem/v/dspy)](https://rubygems.org/gems/dspy)
+[![Total Downloads](https://img.shields.io/gem/dt/dspy)](https://rubygems.org/gems/dspy)
+[![Build Status](https://img.shields.io/github/actions/workflow/status/vicentereig/dspy.rb/ruby.yml?branch=main&label=build)](https://github.com/vicentereig/dspy.rb/actions/workflows/ruby.yml)
+[![Documentation](https://img.shields.io/badge/docs-vicentereig.github.io%2Fdspy.rb-blue)](https://vicentereig.github.io/dspy.rb/)
+
+> [!NOTE]
+> The core Prompt Engineering Framework is production-ready with
+> comprehensive documentation. I am focusing now on educational content on systematic Prompt Optimization and Context Engineering.
+> Your feedback is invaluable. if you encounter issues, please open an [issue](https://github.com/vicentereig/dspy.rb/issues). If you have suggestions, open a [new thread](https://github.com/vicentereig/dspy.rb/discussions).  
+> 
+> If you want to contribute, feel free to reach out to me to coordinate efforts: hey at vicente.services
+>
+> And, yes, this is 100% a legit project. :) 
+
+
+**Build reliable LLM applications in idiomatic Ruby using composable, type-safe modules.**
+
+The Ruby framework for programming with large language models. DSPy.rb brings structured LLM programming to Ruby developers, programmatic Prompt Engineering and Context Engineering. 
+Instead of wrestling with prompt strings and parsing responses, you define typed signatures using idiomatic Ruby to compose and decompose AI Worklows and AI Agents.
+
+**Prompts are the just Functions.** Traditional prompting is like writing code with string concatenation: it works until it doesn't. DSPy.rb brings you 
+the programming approach pioneered by [dspy.ai](https://dspy.ai/): instead of crafting fragile prompts, you define modular 
+signatures and let the framework handle the messy details.
+
+DSPy.rb is an idiomatic Ruby surgical port of Stanford's [DSPy framework](https://github.com/stanfordnlp/dspy). While implementing 
+the core concepts of signatures, predictors, and the main optimization algorithms from the original Python library, DSPy.rb embraces Ruby 
+conventions and adds Ruby-specific innovations like Sorbet-base Typed system, ReAct loops, and production-ready integrations like non-blocking Open Telemetry Instrumentation.
+
+**What you get?** Ruby LLM applications that actually scale and don't break when you sneeze.
+
+Check the [examples](examples/) and take them for a spin!
+
+## Your First DSPy Program
+### Installation
+
+Add to your Gemfile:
+
+```ruby
+gem 'dspy'
+```
+
+and
+
+```bash
+bundle install
+```
+### Your First Reliable Predictor
+
+```ruby
+
+# Configure DSPy globablly to use your fave LLM - you can override this on an instance levle. 
+DSPy.configure do |c|
+  c.lm = DSPy::LM.new('openai/gpt-4o-mini',
+                      api_key: ENV['OPENAI_API_KEY'],
+                      structured_outputs: true)  # Enable OpenAI's native JSON mode
+end
+
+# Define a signature for sentiment classification - instead of writing a full prompt!
+class Classify < DSPy::Signature
+  description "Classify sentiment of a given sentence." # sets the goal of the underlying prompt
+
+  class Sentiment < T::Enum
+    enums do
+      Positive = new('positive')
+      Negative = new('negative')
+      Neutral = new('neutral')
+    end
+  end
+  
+  # Structured Inputs: makes sure you are sending only valid prompt inputs to your model
+  input do
+    const :sentence, String, description: 'The sentence to analyze'
+  end
+
+  # Structured Outputs: your predictor will validate the output of the model too.
+  output do
+    const :sentiment, Sentiment, description: 'The sentiment of the sentence'
+    const :confidence, Float, description: 'A number between 0.0 and 1.0'
+  end
+end
+
+# Wire it to the simplest prompting technique - a Predictn.
+classify = DSPy::Predict.new(Classify)
+# it may raise an error if you mess the inputs or your LLM messes the outputs.
+result = classify.call(sentence: "This book was super fun to read!")
+
+puts result.sentiment    # => #<Sentiment::Positive>  
+puts result.confidence   # => 0.85
+```
+
+### Access to 200+ Models Across 5 Providers
+
+DSPy.rb provides unified access to major LLM providers with provider-specific optimizations:
+
+```ruby
+# OpenAI (GPT-4, GPT-4o, GPT-4o-mini, GPT-5, etc.)
+DSPy.configure do |c|
+  c.lm = DSPy::LM.new('openai/gpt-4o-mini',
+                      api_key: ENV['OPENAI_API_KEY'],
+                      structured_outputs: true)  # Native JSON mode
+end
+
+# Google Gemini (Gemini 1.5 Pro, Flash, Gemini 2.0, etc.)
+DSPy.configure do |c|
+  c.lm = DSPy::LM.new('gemini/gemini-2.5-flash',
+                      api_key: ENV['GEMINI_API_KEY'],
+                      structured_outputs: true)  # Native structured outputs
+end
+
+# Anthropic Claude (Claude 3.5, Claude 4, etc.)
+DSPy.configure do |c|
+  c.lm = DSPy::LM.new('anthropic/claude-sonnet-4-5-20250929',
+                      api_key: ENV['ANTHROPIC_API_KEY'],
+                      structured_outputs: true)  # Tool-based extraction (default)
+end
+
+# Ollama - Run any local model (Llama, Mistral, Gemma, etc.)
+DSPy.configure do |c|
+  c.lm = DSPy::LM.new('ollama/llama3.2')  # Free, runs locally, no API key needed
+end
+
+# OpenRouter - Access to 200+ models from multiple providers
+DSPy.configure do |c|
+  c.lm = DSPy::LM.new('openrouter/deepseek/deepseek-chat-v3.1:free',
+                      api_key: ENV['OPENROUTER_API_KEY'])
+end
+```
+
+## What You Get
+
+**Developer Experience:**
+- LLM provider support using official Ruby clients:
+  - [OpenAI Ruby](https://github.com/openai/openai-ruby) with vision model support
+  - [Anthropic Ruby SDK](https://github.com/anthropics/anthropic-sdk-ruby) with multimodal capabilities
+  - [Google Gemini API](https://ai.google.dev/) with native structured outputs
+  - [Ollama](https://ollama.com/) via OpenAI compatibility layer for local models
+- **Multimodal Support** - Complete image analysis with DSPy::Image, type-safe bounding boxes, vision-capable models
+- Runtime type checking with [Sorbet](https://sorbet.org/) including T::Enum and union types
+- Type-safe tool definitions for ReAct agents
+- Comprehensive instrumentation and observability
+
+**Core Building Blocks:**
+- **Signatures** - Define input/output schemas using Sorbet types with T::Enum and union type support
+- **Predict** - LLM completion with structured data extraction and multimodal support
+- **Chain of Thought** - Step-by-step reasoning for complex problems with automatic prompt optimization
+- **ReAct** - Tool-using agents with type-safe tool definitions and error recovery
+- **Module Composition** - Combine multiple LLM calls into production-ready workflows
+
+**Optimization & Evaluation:**
+- **Prompt Objects** - Manipulate prompts as first-class objects instead of strings
+- **Typed Examples** - Type-safe training data with automatic validation
+- **Evaluation Framework** - Advanced metrics beyond simple accuracy with error-resilient pipelines
+- **MIPROv2 Optimization** - Advanced Bayesian optimization with Gaussian Processes, multiple optimization strategies, auto-config presets, and storage persistence
+
+**Production Features:**
+- **Reliable JSON Extraction** - Native structured outputs for OpenAI and Gemini, Anthropic tool-based extraction, and automatic strategy selection with fallback
+- **Type-Safe Configuration** - Strategy enums with automatic provider optimization (Strict/Compatible modes)
+- **Smart Retry Logic** - Progressive fallback with exponential backoff for handling transient failures
+- **Zero-Config Langfuse Integration** - Set env vars and get automatic OpenTelemetry traces in Langfuse
+- **Performance Caching** - Schema and capability caching for faster repeated operations
+- **File-based Storage** - Optimization result persistence with versioning
+- **Structured Logging** - JSON and key=value formats with span tracking
+
+## Recent Achievements
+
+DSPy.rb has rapidly evolved from experimental to production-ready:
+
+### Foundation
+- ✅ **JSON Parsing Reliability** - Native OpenAI structured outputs with adaptive retry logic and schema-aware fallbacks
+- ✅ **Type-Safe Strategy Configuration** - Provider-optimized strategy selection and enum-backed optimizer presets
+- ✅ **Core Module System** - Predict, ChainOfThought, ReAct, CodeAct with type safety
+- ✅ **Production Observability** - OpenTelemetry, New Relic, and Langfuse integration
+- ✅ **Advanced Optimization** - MIPROv2 with Bayesian optimization, Gaussian Processes, and multi-mode search
+
+### Recent Advances
+- ✅ **MIPROv2 ADE Integrity (v0.29.1)** - Stratified train/val/test splits, honest precision accounting, and enum-driven `--auto` presets with integration coverage
+- ✅ **Instruction Deduplication (v0.29.1)** - Candidate generation now filters repeated programs so optimization logs highlight unique strategies
+- ✅ **GEPA Teleprompter (v0.29.0)** - Genetic-Pareto reflective prompt evolution with merge proposer scheduling, reflective mutation, and ADE demo parity
+- ✅ **Optimizer Utilities Parity (v0.29.0)** - Bootstrap strategies, dataset summaries, and Layer 3 utilities unlock multi-predictor programs on Ruby
+- ✅ **Observability Hardening (v0.29.0)** - OTLP exporter runs on a single-thread executor preventing frozen SSL contexts without blocking spans
+- ✅ **Documentation Refresh (v0.29.x)** - New GEPA guide plus ADE optimization docs covering presets, stratified splits, and error-handling defaults
+
+**Current Focus Areas:**
+
+### Production Readiness
+- 🚧 **Production Patterns** - Real-world usage validation and performance optimization
+- 🚧 **Ruby Ecosystem Integration** - Rails integration, Sidekiq compatibility, deployment patterns
+
+### Community & Adoption
+- 🚧 **Community Examples** - Real-world applications and case studies
+- 🚧 **Contributor Experience** - Making it easier to contribute and extend
+- 🚧 **Performance Benchmarks** - Comparative analysis vs other frameworks
+
+**v1.0 Philosophy:**
+v1.0 will be released after extensive production battle-testing, not after checking off features.
+The API is already stable - v1.0 represents confidence in production reliability backed by real-world validation.
+
+
+## Documentation
+
+📖 **[Complete Documentation Website](https://vicentereig.github.io/dspy.rb/)**
+
+### LLM-Friendly Documentation
+
+For LLMs and AI assistants working with DSPy.rb:
+- **[llms.txt](https://vicentereig.github.io/dspy.rb/llms.txt)** - Concise reference optimized for LLMs
+- **[llms-full.txt](https://vicentereig.github.io/dspy.rb/llms-full.txt)** - Comprehensive API documentation
+
+### Getting Started
+- **[Installation & Setup](docs/src/getting-started/installation.md)** - Detailed installation and configuration
+- **[Quick Start Guide](docs/src/getting-started/quick-start.md)** - Your first DSPy programs
+- **[Core Concepts](docs/src/getting-started/core-concepts.md)** - Understanding signatures, predictors, and modules
+
+### Prompt Engineering
+- **[Signatures & Types](docs/src/core-concepts/signatures.md)** - Define typed interfaces for LLM operations
+- **[Predictors](docs/src/core-concepts/predictors.md)** - Predict, ChainOfThought, ReAct, and more
+- **[Modules & Pipelines](docs/src/core-concepts/modules.md)** - Compose complex multi-stage workflows
+- **[Multimodal Support](docs/src/core-concepts/multimodal.md)** - Image analysis with vision-capable models
+- **[Examples & Validation](docs/src/core-concepts/examples.md)** - Type-safe training data
+- **[Rich Types](docs/src/advanced/complex-types.md)** - Sorbet type integration with automatic coercion for structs, enums, and arrays
+- **[Composable Pipelines](docs/src/advanced/pipelines.md)** - Manual module composition patterns
+
+### Prompt Optimization
+- **[Evaluation Framework](docs/src/optimization/evaluation.md)** - Advanced metrics beyond simple accuracy
+- **[Prompt Optimization](docs/src/optimization/prompt-optimization.md)** - Manipulate prompts as objects
+- **[MIPROv2 Optimizer](docs/src/optimization/miprov2.md)** - Advanced Bayesian optimization with Gaussian Processes
+- **[GEPA Optimizer](docs/src/optimization/gepa.md)** *(beta)* - Reflective mutation with optional reflection LMs
+
+### Context Engineering
+- **[Tools](docs/src/core-concepts/toolsets.md)** - Tool wieldint agents.
+- **[Agentic Memory](docs/src/core-concepts/memory.md)** - Memory Tools & Agentic Loops
+- **[RAG Patterns](docs/src/advanced/rag.md)** - Manual RAG implementation with external services
+
+### Production Features
+- **[Observability](docs/src/production/observability.md)** - Zero-config Langfuse integration with a dedicated export worker that never blocks your LLMs
+- **[Storage System](docs/src/production/storage.md)** - Persistence and optimization result storage
+- **[Custom Metrics](docs/src/advanced/custom-metrics.md)** - Proc-based evaluation logic 
+
+
+
+
+
+
+
+
+## License
+This project is licensed under the MIT License.

data/lib/dspy/evals/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module DSPy
+  class Evals
+    VERSION = DSPy::VERSION
+  end
+end

data/lib/dspy/evals.rb

@@ -0,0 +1,820 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'polars'
+require 'concurrent'
+require 'sorbet-runtime'
+require_relative 'example'
+require_relative 'callbacks'
+
+module DSPy
+  # Core evaluation framework for DSPy programs
+  # Supports single evaluations, batch evaluations, and optimization workflows
+  class Evals
+    extend T::Sig
+
+    # Result of evaluating a single example
+    class EvaluationResult
+      extend T::Sig
+
+      sig { returns(T.untyped) }
+      attr_reader :example
+
+      sig { returns(T.untyped) }
+      attr_reader :prediction
+
+      sig { returns(T.untyped) }
+      attr_reader :trace
+
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      attr_reader :metrics
+
+      sig { returns(T::Boolean) }
+      attr_reader :passed
+
+      sig do
+        params(
+          example: T.untyped,
+          prediction: T.untyped,
+          trace: T.untyped,
+          metrics: T::Hash[Symbol, T.untyped],
+          passed: T::Boolean
+        ).void
+      end
+      def initialize(example:, prediction:, trace:, metrics:, passed:)
+        @example = example
+        @prediction = prediction
+        @trace = trace
+        @metrics = metrics
+        @passed = passed
+      end
+
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      def to_h
+        {
+          example: @example,
+          prediction: @prediction.respond_to?(:to_h) ? @prediction.to_h : @prediction,
+          trace: @trace,
+          metrics: @metrics,
+          passed: @passed
+        }
+      end
+    end
+
+    # Batch evaluation results with aggregated metrics
+    class BatchEvaluationResult
+      extend T::Sig
+
+      sig { returns(T::Array[EvaluationResult]) }
+      attr_reader :results
+
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      attr_reader :aggregated_metrics
+
+      sig { returns(Integer) }
+      attr_reader :total_examples
+
+      sig { returns(Integer) }
+      attr_reader :passed_examples
+
+      sig { returns(Float) }
+      attr_reader :pass_rate
+
+      sig { returns(Float) }
+      attr_reader :score
+
+      sig do
+        params(
+          results: T::Array[EvaluationResult],
+          aggregated_metrics: T::Hash[Symbol, T.untyped]
+        ).void
+      end
+      def initialize(results:, aggregated_metrics:)
+        @results = results.freeze
+        @aggregated_metrics = aggregated_metrics.freeze
+        @total_examples = results.length
+        @passed_examples = results.count(&:passed)
+        @pass_rate = @total_examples > 0 ? @passed_examples.to_f / @total_examples : 0.0
+        score_avg = aggregated_metrics[:score_avg] || @pass_rate
+        @score = (score_avg * 100).round(2)
+      end
+
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      def to_h
+        {
+          total_examples: @total_examples,
+          passed_examples: @passed_examples,
+          pass_rate: @pass_rate,
+          score: @score,
+          aggregated_metrics: @aggregated_metrics,
+          results: @results.map(&:to_h)
+        }
+      end
+
+      sig { returns(Polars::DataFrame) }
+      def to_polars
+        rows = @results.each_with_index.map do |result, index|
+          {
+            "index" => index,
+            "passed" => result.passed,
+            "score" => result.metrics[:score],
+            "example" => serialize_for_polars(result.example),
+            "prediction" => serialize_for_polars(result.prediction),
+            "metrics" => serialize_for_polars(result.metrics),
+            "trace" => serialize_for_polars(result.trace)
+          }
+        end
+
+        Polars::DataFrame.new(rows)
+      end
+
+      private
+
+      def serialize_for_polars(value)
+        case value
+        when NilClass, TrueClass, FalseClass, Numeric, String
+          value
+        when Hash
+          JSON.generate(value)
+        when Array
+          JSON.generate(value)
+        else
+          if value.respond_to?(:to_h)
+            JSON.generate(value.to_h)
+          else
+            value.to_s
+          end
+        end
+      end
+    end
+
+    sig { returns(T.untyped) }
+    attr_reader :program
+
+    sig { returns(T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))) }
+    attr_reader :metric
+
+    sig { returns(T.nilable(Integer)) }
+    attr_reader :num_threads
+
+    sig { returns(T.nilable(Integer)) }
+    attr_reader :max_errors
+
+    sig { returns(T::Boolean) }
+    attr_reader :provide_traceback
+
+    sig { returns(Float) }
+    attr_reader :failure_score
+
+    sig { returns(T.nilable(EvaluationResult)) }
+    attr_reader :last_example_result
+
+    sig { returns(T.nilable(BatchEvaluationResult)) }
+    attr_reader :last_batch_result
+
+    include DSPy::Callbacks
+
+    create_before_callback :call, wrap: false
+    create_after_callback :call, wrap: false
+    create_before_callback :evaluate, wrap: false
+    create_after_callback :evaluate, wrap: false
+
+    class << self
+      def before_example(callback = nil, &block)
+        before(callback, target: :call, &block)
+      end
+
+      def after_example(callback = nil, &block)
+        after(callback, target: :call, &block)
+      end
+
+      def before_batch(callback = nil, &block)
+        before(callback, target: :evaluate, &block)
+      end
+
+      def after_batch(callback = nil, &block)
+        after(callback, target: :evaluate, &block)
+      end
+
+      def reset_callbacks!
+        @callbacks = {}
+      end
+    end
+
+    sig do
+      params(
+        program: T.untyped,
+        metric: T.nilable(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean)),
+        num_threads: T.nilable(Integer),
+        max_errors: T.nilable(Integer),
+        failure_score: T.nilable(Numeric),
+        provide_traceback: T::Boolean
+      ).void
+    end
+    def initialize(program, metric: nil, num_threads: 1, max_errors: 5, failure_score: 0.0, provide_traceback: true)
+      @program = program
+      @metric = metric
+      @num_threads = num_threads || 1
+      @max_errors = max_errors || 5
+      @provide_traceback = provide_traceback
+      @failure_score = failure_score ? failure_score.to_f : 0.0
+      @last_example_result = nil
+      @last_batch_result = nil
+    end
+
+    # Evaluate program on a single example
+    sig { params(example: T.untyped, trace: T.nilable(T.untyped)).returns(EvaluationResult) }
+    def call(example, trace: nil)
+      run_callbacks(:before, :call, example: example)
+
+      DSPy::Context.with_span(
+        operation: 'evaluation.example',
+        'dspy.module' => 'Evaluator',
+        'evaluation.program' => @program.class.name,
+        'evaluation.has_metric' => !@metric.nil?
+      ) do
+        begin
+          perform_call(example, trace: trace)
+        rescue => e
+          build_error_result(example, e, trace: trace)
+        end
+      end.then do |result|
+        @last_example_result = result
+        emit_example_observation(example, result)
+        run_callbacks(:after, :call, example: example, result: result)
+        result
+      end
+    end
+
+    # Evaluate program on multiple examples
+    sig do
+      params(
+        devset: T::Array[T.untyped],
+        display_progress: T::Boolean,
+        display_table: T::Boolean,
+        return_outputs: T::Boolean
+      ).returns(BatchEvaluationResult)
+    end
+    def evaluate(devset, display_progress: true, display_table: false, return_outputs: true)
+      run_callbacks(:before, :evaluate, devset: devset)
+
+      DSPy::Context.with_span(
+        operation: 'evaluation.batch',
+        'dspy.module' => 'Evaluator',
+        'evaluation.program' => @program.class.name,
+        'evaluation.num_examples' => devset.length,
+        'evaluation.has_metric' => !@metric.nil?,
+        'evaluation.num_threads' => @num_threads
+      ) do
+        if display_progress
+          puts "Evaluating #{devset.length} examples..."
+        end
+
+        results = if parallel_execution?
+          evaluate_in_parallel(devset, display_progress: display_progress)
+        else
+          evaluate_sequential(devset, display_progress: display_progress)
+        end
+
+        # Aggregate metrics
+        aggregated_metrics = aggregate_metrics(results)
+
+        batch_result = BatchEvaluationResult.new(
+          results: results,
+          aggregated_metrics: aggregated_metrics
+        )
+
+        if display_table
+          display_results_table(batch_result)
+        end
+
+        # Emit batch completion event
+        DSPy.log('evaluation.batch_complete', **{
+          'evaluation.program_class' => @program.class.name,
+          'evaluation.total_examples' => batch_result.total_examples,
+          'evaluation.passed_examples' => batch_result.passed_examples,
+          'evaluation.pass_rate' => batch_result.pass_rate,
+          'evaluation.aggregated_metrics' => aggregated_metrics
+        })
+
+        if display_progress
+          puts "Evaluation complete: #{batch_result.passed_examples}/#{batch_result.total_examples} passed (#{(batch_result.pass_rate * 100).round(1)}%)"
+        end
+
+        batch_result
+      end.then do |batch_result|
+        @last_batch_result = batch_result
+        emit_batch_observation(devset, batch_result)
+        run_callbacks(:after, :evaluate, devset: devset, result: batch_result)
+        batch_result
+      end
+    end
+
+    private
+
+    def parallel_execution?
+      (@num_threads || 1) > 1
+    end
+
+    def evaluate_sequential(devset, display_progress:)
+      results = []
+      errors = 0
+      passed_count = 0
+
+      devset.each_with_index do |example, index|
+        break if errors >= @max_errors
+
+        result = safe_call(example)
+        results << result
+
+        if result.passed
+          passed_count += 1
+        else
+          errors += 1
+        end
+
+        if display_progress && (index + 1) % 10 == 0
+          log_progress(index + 1, devset.length, passed_count)
+        end
+      end
+
+      results
+    end
+
+    def evaluate_in_parallel(devset, display_progress:)
+      total = devset.length
+      results = Array.new(total)
+      errors = 0
+      processed = 0
+      passed_count = 0
+
+      executor = Concurrent::ThreadPoolExecutor.new(
+        min_threads: @num_threads,
+        max_threads: @num_threads,
+        max_queue: [total, 1].max,
+        idletime: 60
+      )
+
+      enumerator = devset.each_with_index
+
+      loop do
+        break if errors >= @max_errors
+
+        batch = []
+        @num_threads.times do
+          begin
+            example = enumerator.next
+            batch << { example: example[0], index: example[1] }
+          rescue StopIteration
+            break
+          end
+        end
+
+        break if batch.empty?
+
+        futures = batch.map do |item|
+          Concurrent::Promises.future_on(executor) do
+            [:ok, item[:index], safe_call(item[:example])]
+          rescue => e
+            [:error, item[:index], e]
+          end
+        end
+
+        futures.each do |future|
+          status, index, payload = future.value!
+          example = batch.find { |entry| entry[:index] == index }[:example]
+
+          result = if status == :ok
+            payload
+          else
+            errors += 1
+            puts "Error processing example #{index}: #{payload.message}" if display_progress
+            build_error_result(example, payload)
+          end
+
+          results[index] = result
+          processed += 1
+          if result.passed
+            passed_count += 1
+          else
+            errors += 1 unless status == :error
+          end
+
+          if display_progress && (processed % 10).zero?
+            log_progress(processed, total, passed_count)
+          end
+        end
+      end
+
+      executor.shutdown
+      executor.wait_for_termination
+
+      results.compact
+    end
+
+    def safe_call(example)
+      call(example)
+    rescue => e
+      build_error_result(example, e)
+    end
+
+    def perform_call(example, trace:)
+      # Extract input from example - support both hash and object formats
+      input_values = extract_input_values(example)
+
+      # Run prediction
+      prediction = @program.call(**input_values)
+
+      # Calculate metrics if provided
+      metrics = {}
+      passed = true
+
+      if @metric
+        begin
+          metric_result = @metric.call(example, prediction)
+          if metric_result.is_a?(Hash)
+            metrics = symbolize_keys(metric_result)
+            passed_flag = metrics.key?(:passed) ? metrics[:passed] : metrics['passed']
+            passed = passed_flag.nil? ? true : !!passed_flag
+          else
+            passed = !!metric_result
+            metrics[:passed] = passed
+          end
+        rescue => e
+          passed = false
+          metrics[:error] = e.message
+          metrics[:passed] = false
+          metrics[:score] = @failure_score
+        end
+      end
+
+      metrics[:passed] = passed unless metrics.key?(:passed)
+      metrics[:score] = normalize_score(metrics[:score], passed) if metrics.key?(:score)
+      metrics[:score] ||= passed ? 1.0 : 0.0
+
+      EvaluationResult.new(
+        example: example,
+        prediction: prediction,
+        trace: trace,
+        metrics: metrics,
+        passed: passed
+      )
+    end
+
+    def build_error_result(example, error, trace: nil)
+      metrics = {
+        error: error.message,
+        passed: false,
+        score: @failure_score
+      }
+      metrics[:traceback] = error.backtrace&.first(10) || [] if @provide_traceback
+
+      EvaluationResult.new(
+        example: example,
+        prediction: nil,
+        trace: trace,
+        metrics: metrics,
+        passed: false
+      )
+    end
+
+    def log_progress(processed, total, passed_count)
+      puts "Processed #{processed}/#{total} examples (#{passed_count} passed)"
+    end
+
+    # Extract input values from example in various formats
+    sig { params(example: T.untyped).returns(T::Hash[Symbol, T.untyped]) }
+    def extract_input_values(example)
+      case example
+      when DSPy::Example
+        # Preferred format: DSPy::Example object with type safety
+        example.input_values
+      when Hash
+        # Check if it has an :input key (structured format)
+        if example.key?(:input)
+          input_data = example[:input]
+          input_data.is_a?(Hash) ? input_data.transform_keys(&:to_sym) : input_data
+        elsif example.key?('input')
+          input_data = example['input']
+          input_data.is_a?(Hash) ? input_data.transform_keys(&:to_sym) : input_data
+        else
+          # Legacy format - assume the whole hash is input
+          if example.keys.first.is_a?(String)
+            example.transform_keys(&:to_sym)
+          else
+            example
+          end
+        end
+      when ->(ex) { ex.respond_to?(:input_values) }
+        # Object with input_values method (Example-like)
+        example.input_values
+      when ->(ex) { ex.respond_to?(:input) }
+        # Object with input method
+        input_data = example.input
+        input_data.is_a?(Hash) ? input_data.transform_keys(&:to_sym) : input_data
+      when ->(ex) { ex.respond_to?(:to_h) }
+        # Object that can be converted to hash
+        hash = example.to_h
+        if hash.key?(:input)
+          input_data = hash[:input]
+          input_data.is_a?(Hash) ? input_data.transform_keys(&:to_sym) : input_data
+        elsif hash.key?('input')
+          input_data = hash['input']
+          input_data.is_a?(Hash) ? input_data.transform_keys(&:to_sym) : input_data
+        else
+          hash.is_a?(Hash) ? hash.transform_keys(&:to_sym) : hash
+        end
+      else
+        # Try to extract by introspection
+        if example.respond_to?(:instance_variables)
+          vars = {}
+          example.instance_variables.each do |var|
+            key = var.to_s.delete('@').to_sym
+            vars[key] = example.instance_variable_get(var)
+          end
+          vars
+        else
+          raise ArgumentError, "Cannot extract input values from example: #{example.class}"
+        end
+      end
+    end
+
+    # Extract expected values for metric comparison (used internally)
+    sig { params(example: T.untyped).returns(T.nilable(T::Hash[Symbol, T.untyped])) }
+    def extract_expected_values(example)
+      case example
+      when DSPy::Example
+        example.expected_values
+      when Hash
+        if example.key?(:expected)
+          expected_data = example[:expected]
+          expected_data.is_a?(Hash) ? expected_data.transform_keys(&:to_sym) : expected_data
+        elsif example.key?('expected')
+          expected_data = example['expected']
+          expected_data.is_a?(Hash) ? expected_data.transform_keys(&:to_sym) : expected_data
+        else
+          # Legacy format - no separate expected values
+          nil
+        end
+      when ->(ex) { ex.respond_to?(:expected_values) }
+        example.expected_values
+      when ->(ex) { ex.respond_to?(:expected) }
+        expected_data = example.expected
+        expected_data.is_a?(Hash) ? expected_data.transform_keys(&:to_sym) : expected_data
+      else
+        nil
+      end
+    end
+
+    # Aggregate metrics across all results
+    sig { params(results: T::Array[EvaluationResult]).returns(T::Hash[Symbol, T.untyped]) }
+    def aggregate_metrics(results)
+      return {} if results.empty?
+      
+      total = results.length
+      passed = results.count(&:passed)
+
+      aggregated = {
+        total_examples: total,
+        passed_examples: passed,
+        failed_examples: results.count { |r| !r.passed }
+      }
+
+      score_values = results.filter_map do |result|
+        score = result.metrics[:score]
+        score if score.is_a?(Numeric)
+      end
+
+      if score_values.any?
+        aggregated[:score_sum] = score_values.sum
+        aggregated[:score_avg] = score_values.sum.to_f / score_values.length
+        aggregated[:score_min] = score_values.min
+        aggregated[:score_max] = score_values.max
+      else
+        aggregated[:score_avg] = passed.positive? && total.positive? ? passed.to_f / total : 0.0
+      end
+
+      # Aggregate other numeric metrics
+      numeric_metrics = {}
+      results.each do |result|
+        result.metrics.each do |key, value|
+          next if [:error, :traceback, :passed, :score].include?(key)
+          next unless value.is_a?(Numeric)
+
+          numeric_metrics[key] ||= []
+          numeric_metrics[key] << value
+        end
+      end
+
+      numeric_metrics.each do |key, values|
+        aggregated[:"#{key}_avg"] = values.sum.to_f / values.length
+        aggregated[:"#{key}_min"] = values.min
+        aggregated[:"#{key}_max"] = values.max
+      end
+
+      aggregated[:pass_rate] = total.positive? ? passed.to_f / total : 0.0
+
+      aggregated
+    end
+
+    # Display results in a table format
+    sig { params(batch_result: BatchEvaluationResult).void }
+    def display_results_table(batch_result)
+      puts "\nEvaluation Results:"
+      puts "=" * 50
+      puts "Total Examples: #{batch_result.total_examples}"
+      puts "Passed: #{batch_result.passed_examples}"
+      puts "Failed: #{batch_result.total_examples - batch_result.passed_examples}"
+      puts "Pass Rate: #{(batch_result.pass_rate * 100).round(1)}%"
+      
+      if batch_result.aggregated_metrics.any?
+        puts "\nAggregated Metrics:"
+        batch_result.aggregated_metrics.each do |key, value|
+          next if [:total_examples, :passed_examples, :failed_examples, :pass_rate].include?(key)
+          puts "  #{key}: #{value.is_a?(Float) ? value.round(3) : value}"
+        end
+      end
+      
+      puts "=" * 50
+    end
+
+    def emit_example_observation(example, result)
+      DSPy.event('evals.example.complete', {
+        program: @program.class.name,
+        example_id: extract_example_id(example),
+        passed: result.passed,
+        score: result.metrics[:score],
+        error: result.metrics[:error]
+      })
+    rescue => e
+      DSPy.log('evals.example.observation_error', error: e.message)
+    end
+
+    def emit_batch_observation(devset, batch_result)
+      DSPy.event('evals.batch.complete', {
+        program: @program.class.name,
+        dataset_size: devset.length,
+        total_examples: batch_result.total_examples,
+        passed_examples: batch_result.passed_examples,
+        pass_rate: batch_result.pass_rate,
+        score: batch_result.score
+      })
+    rescue => e
+      DSPy.log('evals.batch.observation_error', error: e.message)
+    end
+
+    def extract_example_id(example)
+      if example.respond_to?(:id)
+        example.id
+      elsif example.is_a?(Hash)
+        example[:id] || example['id']
+      else
+        nil
+      end
+    rescue
+      nil
+    end
+
+    def symbolize_keys(hash)
+      hash.each_with_object({}) do |(key, value), memo|
+        memo[key.respond_to?(:to_sym) ? key.to_sym : key] = value
+      end
+    end
+
+    def normalize_score(value, passed)
+      case value
+      when Numeric
+        value.to_f
+      when TrueClass, FalseClass
+        value ? 1.0 : 0.0
+      else
+        passed ? 1.0 : 0.0
+      end
+    end
+
+  end
+
+  # Common metric functions for evaluation
+  module Metrics
+    extend T::Sig
+
+    # Exact match metric - checks if prediction exactly matches expected output
+    sig do
+      params(
+        field: Symbol,
+        case_sensitive: T::Boolean
+      ).returns(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))
+    end
+    def self.exact_match(field: :answer, case_sensitive: true)
+      proc do |example, prediction|
+        expected = extract_field(example, field)
+        actual = extract_field(prediction, field)
+        
+        next false if expected.nil? || actual.nil?
+        
+        if case_sensitive
+          expected.to_s == actual.to_s
+        else
+          expected.to_s.downcase == actual.to_s.downcase
+        end
+      end
+    end
+
+    # Contains metric - checks if prediction contains expected substring
+    sig do
+      params(
+        field: Symbol,
+        case_sensitive: T::Boolean
+      ).returns(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Boolean))
+    end
+    def self.contains(field: :answer, case_sensitive: false)
+      proc do |example, prediction|
+        expected = extract_field(example, field)
+        actual = extract_field(prediction, field)
+        
+        next false if expected.nil? || actual.nil?
+        
+        if case_sensitive
+          actual.to_s.include?(expected.to_s)
+        else
+          actual.to_s.downcase.include?(expected.to_s.downcase)
+        end
+      end
+    end
+
+    # Numeric difference metric - checks if prediction is within tolerance of expected value
+    sig do
+      params(
+        field: Symbol,
+        tolerance: Float
+      ).returns(T.proc.params(arg0: T.untyped, arg1: T.untyped).returns(T::Hash[Symbol, T.untyped]))
+    end
+    def self.numeric_difference(field: :answer, tolerance: 0.01)
+      proc do |example, prediction|
+        expected = extract_field(example, field)
+        actual = extract_field(prediction, field)
+        
+        next { passed: false, error: "Missing values" } if expected.nil? || actual.nil?
+        
+        begin
+          expected_num = Float(expected)
+          actual_num = Float(actual)
+          difference = (expected_num - actual_num).abs
+          passed = difference <= tolerance
+          
+          {
+            passed: passed,
+            difference: difference,
+            expected: expected_num,
+            actual: actual_num,
+            tolerance: tolerance
+          }
+        rescue ArgumentError
+          { passed: false, error: "Non-numeric values" }
+        end
+      end
+    end
+
+    # Composite metric - combines multiple metrics with AND logic
+    def self.composite_and(*metrics)
+      proc do |example, prediction|
+        results = {}
+        all_passed = true
+        
+        metrics.each_with_index do |metric, index|
+          result = metric.call(example, prediction)
+          
+          if result.is_a?(Hash)
+            results[:"metric_#{index}"] = result
+            all_passed &&= result[:passed] || result['passed'] || false
+          else
+            passed = !!result
+            results[:"metric_#{index}"] = { passed: passed }
+            all_passed &&= passed
+          end
+        end
+        
+        results[:passed] = all_passed
+        results
+      end
+    end
+
+    private
+
+    # Extract field value from example or prediction
+    sig { params(obj: T.untyped, field: Symbol).returns(T.untyped) }
+    def self.extract_field(obj, field)
+      case obj
+      when Hash
+        obj[field] || obj[field.to_s]
+      when ->(o) { o.respond_to?(field) }
+        obj.send(field)
+      when ->(o) { o.respond_to?(:to_h) }
+        hash = obj.to_h
+        hash[field] || hash[field.to_s]
+      else
+        nil
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification
+name: dspy-evals
+version: !ruby/object:Gem::Version
+  version: 0.29.1
+platform: ruby
+authors:
+- Vicente Reig Rincón de Arellano
+bindir: bin
+cert_chain: []
+date: 2025-10-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: dspy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.29.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.29.1
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: polars-df
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.15'
+description: Provides the DSPy::Evals runtime, concurrency, callbacks, and export
+  helpers for benchmarking Ruby DSPy programs.
+email:
+- hey@vicente.services
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/dspy/evals.rb
+- lib/dspy/evals/version.rb
+homepage: https://github.com/vicentereig/dspy.rb
+licenses:
+- MIT
+metadata:
+  github_repo: git@github.com:vicentereig/dspy.rb
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.5
+specification_version: 4
+summary: Evaluation utilities for DSPy.rb programs.
+test_files: []