dspy 0.3.1 → 0.5.0

data/lib/dspy/example.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+require_relative 'signature'
+
+module DSPy
+  # Represents a typed training/evaluation example with Signature validation
+  # Provides early validation and type safety for evaluation workflows
+  class Example
+    extend T::Sig
+
+    sig { returns(T.class_of(Signature)) }
+    attr_reader :signature_class
+
+    sig { returns(T::Struct) }
+    attr_reader :input
+
+    sig { returns(T::Struct) }
+    attr_reader :expected
+
+    sig { returns(T.nilable(String)) }
+    attr_reader :id
+
+    sig { returns(T.nilable(T::Hash[Symbol, T.untyped])) }
+    attr_reader :metadata
+
+    sig do
+      params(
+        signature_class: T.class_of(Signature),
+        input: T::Hash[Symbol, T.untyped],
+        expected: T::Hash[Symbol, T.untyped],
+        id: T.nilable(String),
+        metadata: T.nilable(T::Hash[Symbol, T.untyped])
+      ).void
+    end
+    def initialize(signature_class:, input:, expected:, id: nil, metadata: nil)
+      @signature_class = signature_class
+      @id = id
+      @metadata = metadata&.freeze
+
+      # Validate and create input struct
+      begin
+        @input = signature_class.input_struct_class.new(**input)
+      rescue ArgumentError => e
+        raise ArgumentError, "Invalid input for #{signature_class.name}: #{e.message}"
+      rescue TypeError => e
+        raise TypeError, "Type error in input for #{signature_class.name}: #{e.message}"
+      end
+
+      # Validate and create expected output struct
+      begin
+        @expected = signature_class.output_struct_class.new(**expected)
+      rescue ArgumentError => e
+        raise ArgumentError, "Invalid expected output for #{signature_class.name}: #{e.message}"
+      rescue TypeError => e
+        raise TypeError, "Type error in expected output for #{signature_class.name}: #{e.message}"
+      end
+    end
+
+    # Convert input struct to hash for program execution
+    sig { returns(T::Hash[Symbol, T.untyped]) }
+    def input_values
+      input_hash = {}
+      @input.class.props.keys.each do |key|
+        input_hash[key] = @input.send(key)
+      end
+      input_hash
+    end
+
+    # Convert expected struct to hash for comparison
+    sig { returns(T::Hash[Symbol, T.untyped]) }
+    def expected_values
+      expected_hash = {}
+      @expected.class.props.keys.each do |key|
+        expected_hash[key] = @expected.send(key)
+      end
+      expected_hash
+    end
+
+    # Check if prediction matches expected output using struct comparison
+    sig { params(prediction: T.untyped).returns(T::Boolean) }
+    def matches_prediction?(prediction)
+      return false unless prediction
+
+      # Compare each expected field with prediction
+      @expected.class.props.keys.all? do |key|
+        expected_value = @expected.send(key)
+        
+        # Extract prediction value
+        prediction_value = case prediction
+                          when T::Struct
+                            prediction.respond_to?(key) ? prediction.send(key) : nil
+                          when Hash
+                            prediction[key] || prediction[key.to_s]
+                          else
+                            prediction.respond_to?(key) ? prediction.send(key) : nil
+                          end
+        
+        expected_value == prediction_value
+      end
+    end
+
+    # Serialization for persistence and debugging
+    sig { returns(T::Hash[Symbol, T.untyped]) }
+    def to_h
+      result = {
+        signature_class: @signature_class.name,
+        input: input_values,
+        expected: expected_values
+      }
+      
+      result[:id] = @id if @id
+      result[:metadata] = @metadata if @metadata
+      result
+    end
+
+    # Create Example from hash representation
+    sig do
+      params(
+        hash: T::Hash[Symbol, T.untyped],
+        signature_registry: T.nilable(T::Hash[String, T.class_of(Signature)])
+      ).returns(Example)
+    end
+    def self.from_h(hash, signature_registry: nil)
+      signature_class_name = hash[:signature_class]
+      
+      # Resolve signature class
+      signature_class = if signature_registry && signature_registry[signature_class_name]
+                         signature_registry[signature_class_name]
+                       else
+                         # Try to resolve from constant
+                         Object.const_get(signature_class_name)
+                       end
+      
+      new(
+        signature_class: signature_class,
+        input: hash[:input] || {},
+        expected: hash[:expected] || {},
+        id: hash[:id],
+        metadata: hash[:metadata]
+      )
+    end
+
+
+    # Batch validation for multiple examples
+    sig do
+      params(
+        signature_class: T.class_of(Signature),
+        examples_data: T::Array[T::Hash[Symbol, T.untyped]]
+      ).returns(T::Array[Example])
+    end
+    def self.validate_batch(signature_class, examples_data)
+      errors = []
+      examples = []
+      
+      examples_data.each_with_index do |example_data, index|
+        begin
+          # Only support structured format with :input and :expected keys
+          unless example_data.key?(:input) && example_data.key?(:expected)
+            raise ArgumentError, "Example must have :input and :expected keys. Legacy flat format is no longer supported."
+          end
+          
+          example = new(
+            signature_class: signature_class,
+            input: example_data[:input],
+            expected: example_data[:expected],
+            id: example_data[:id] || "example_#{index}"
+          )
+          examples << example
+        rescue => e
+          errors << "Example #{index}: #{e.message}"
+        end
+      end
+      
+      unless errors.empty?
+        raise ArgumentError, "Validation errors:\n#{errors.join("\n")}"
+      end
+      
+      examples
+    end
+
+    # Equality comparison
+    sig { params(other: T.untyped).returns(T::Boolean) }
+    def ==(other)
+      return false unless other.is_a?(Example)
+      
+      @signature_class == other.signature_class &&
+        input_values == other.input_values &&
+        expected_values == other.expected_values
+    end
+
+    # String representation for debugging
+    sig { returns(String) }
+    def to_s
+      "DSPy::Example(#{@signature_class.name}) input=#{input_values} expected=#{expected_values}"
+    end
+
+    sig { returns(String) }
+    def inspect
+      to_s
+    end
+  end
+end

data/lib/dspy/few_shot_example.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module DSPy
+  class FewShotExample
+    extend T::Sig
+
+    sig { returns(T::Hash[Symbol, T.untyped]) }
+    attr_reader :input
+
+    sig { returns(T::Hash[Symbol, T.untyped]) }
+    attr_reader :output
+
+    sig { returns(T.nilable(String)) }
+    attr_reader :reasoning
+
+    sig do
+      params(
+        input: T::Hash[Symbol, T.untyped],
+        output: T::Hash[Symbol, T.untyped],
+        reasoning: T.nilable(String)
+      ).void
+    end
+    def initialize(input:, output:, reasoning: nil)
+      @input = input.freeze
+      @output = output.freeze
+      @reasoning = reasoning
+    end
+
+    sig { returns(String) }
+    def to_prompt_section
+      sections = []
+      
+      sections << "## Input"
+      sections << "```json"
+      sections << JSON.pretty_generate(@input)
+      sections << "```"
+      
+      if @reasoning
+        sections << "## Reasoning"
+        sections << @reasoning
+      end
+      
+      sections << "## Output"
+      sections << "```json"
+      sections << JSON.pretty_generate(@output)
+      sections << "```"
+      
+      sections.join("\n")
+    end
+
+    sig { returns(T::Hash[Symbol, T.untyped]) }
+    def to_h
+      result = {
+        input: @input,
+        output: @output
+      }
+      result[:reasoning] = @reasoning if @reasoning
+      result
+    end
+
+    sig { params(hash: T::Hash[Symbol, T.untyped]).returns(FewShotExample) }
+    def self.from_h(hash)
+      new(
+        input: hash[:input] || {},
+        output: hash[:output] || {},
+        reasoning: hash[:reasoning]
+      )
+    end
+
+    sig { params(other: T.untyped).returns(T::Boolean) }
+    def ==(other)
+      return false unless other.is_a?(FewShotExample)
+      
+      @input == other.input &&
+        @output == other.output &&
+        @reasoning == other.reasoning
+    end
+  end
+end

data/lib/dspy/instrumentation/token_tracker.rb

@@ -28,9 +28,9 @@ module DSPy
         return {} unless usage.is_a?(Hash)
         
         {
-          tokens_input: usage[:prompt_tokens] || usage['prompt_tokens'],
-          tokens_output: usage[:completion_tokens] || usage['completion_tokens'],
-          tokens_total: usage[:total_tokens] || usage['total_tokens']
+          input_tokens: usage[:prompt_tokens] || usage['prompt_tokens'],
+          output_tokens: usage[:completion_tokens] || usage['completion_tokens'],
+          total_tokens: usage[:total_tokens] || usage['total_tokens']
         }
       end
 
@@ -44,9 +44,9 @@ module DSPy
         output_tokens = usage[:output_tokens] || usage['output_tokens'] || 0
         
         {
-          tokens_input: input_tokens,
-          tokens_output: output_tokens,
-          tokens_total: input_tokens + output_tokens
+          input_tokens: input_tokens,
+          output_tokens: output_tokens,
+          total_tokens: input_tokens + output_tokens
         }
       end
     end

data/lib/dspy/instrumentation.rb

@@ -2,17 +2,34 @@
 
 require 'dry-monitor'
 require 'dry-configurable'
+require 'time'
 
 module DSPy
   # Core instrumentation module using dry-monitor for event emission
-  # Provides extension points for logging, Langfuse, New Relic, and custom monitoring
+  # Provides extension points for logging, OpenTelemetry, New Relic, Langfuse, and custom monitoring
   module Instrumentation
-    # Get the current logger subscriber instance (lazy initialization)
-    def self.logger_subscriber
-      @logger_subscriber ||= begin
-        require_relative 'subscribers/logger_subscriber'
-        DSPy::Subscribers::LoggerSubscriber.new
-      end
+    # Get a logger subscriber instance (creates new instance each time)
+    def self.logger_subscriber(**options)
+      require_relative 'subscribers/logger_subscriber'
+      DSPy::Subscribers::LoggerSubscriber.new(**options)
+    end
+
+    # Get an OpenTelemetry subscriber instance (creates new instance each time)
+    def self.otel_subscriber(**options)
+      require_relative 'subscribers/otel_subscriber'
+      DSPy::Subscribers::OtelSubscriber.new(**options)
+    end
+
+    # Get a New Relic subscriber instance (creates new instance each time)
+    def self.newrelic_subscriber(**options)
+      require_relative 'subscribers/newrelic_subscriber'
+      DSPy::Subscribers::NewrelicSubscriber.new(**options)
+    end
+
+    # Get a Langfuse subscriber instance (creates new instance each time)
+    def self.langfuse_subscriber(**options)
+      require_relative 'subscribers/langfuse_subscriber'
+      DSPy::Subscribers::LangfuseSubscriber.new(**options)
     end
 
     def self.notifications
@@ -29,6 +46,55 @@ module DSPy
         n.register_event('dspy.react.tool_call')
         n.register_event('dspy.react.iteration_complete')
         n.register_event('dspy.react.max_iterations')
+        
+        # Evaluation events
+        n.register_event('dspy.evaluation.start')
+        n.register_event('dspy.evaluation.example')
+        n.register_event('dspy.evaluation.batch')
+        n.register_event('dspy.evaluation.batch_complete')
+        
+        # Optimization events
+        n.register_event('dspy.optimization.start')
+        n.register_event('dspy.optimization.complete')
+        n.register_event('dspy.optimization.trial_start')
+        n.register_event('dspy.optimization.trial_complete')
+        n.register_event('dspy.optimization.bootstrap_start')
+        n.register_event('dspy.optimization.bootstrap_complete')
+        n.register_event('dspy.optimization.bootstrap_example')
+        n.register_event('dspy.optimization.minibatch_evaluation')
+        n.register_event('dspy.optimization.instruction_proposal_start')
+        n.register_event('dspy.optimization.instruction_proposal_complete')
+        n.register_event('dspy.optimization.error')
+        n.register_event('dspy.optimization.save')
+        n.register_event('dspy.optimization.load')
+        
+        # Storage events
+        n.register_event('dspy.storage.save_start')
+        n.register_event('dspy.storage.save_complete')
+        n.register_event('dspy.storage.save_error')
+        n.register_event('dspy.storage.load_start')
+        n.register_event('dspy.storage.load_complete')
+        n.register_event('dspy.storage.load_error')
+        n.register_event('dspy.storage.delete')
+        n.register_event('dspy.storage.export')
+        n.register_event('dspy.storage.import')
+        n.register_event('dspy.storage.cleanup')
+        
+        # Registry events
+        n.register_event('dspy.registry.register_start')
+        n.register_event('dspy.registry.register_complete')
+        n.register_event('dspy.registry.register_error')
+        n.register_event('dspy.registry.deploy_start')
+        n.register_event('dspy.registry.deploy_complete')
+        n.register_event('dspy.registry.deploy_error')
+        n.register_event('dspy.registry.rollback_start')
+        n.register_event('dspy.registry.rollback_complete')
+        n.register_event('dspy.registry.rollback_error')
+        n.register_event('dspy.registry.performance_update')
+        n.register_event('dspy.registry.export')
+        n.register_event('dspy.registry.import')
+        n.register_event('dspy.registry.auto_deployment')
+        n.register_event('dspy.registry.automatic_rollback')
       end
     end
 
@@ -49,9 +115,8 @@ module DSPy
         enhanced_payload = payload.merge(
           duration_ms: ((end_time - start_time) * 1000).round(2),
           cpu_time_ms: ((end_cpu - start_cpu) * 1000).round(2),
-          status: 'success',
-          timestamp: Time.now.iso8601
-        )
+          status: 'success'
+        ).merge(generate_timestamp)
 
         self.emit_event(event_name, enhanced_payload)
         result
@@ -64,9 +129,8 @@ module DSPy
           cpu_time_ms: ((end_cpu - start_cpu) * 1000).round(2),
           status: 'error',
           error_type: error.class.name,
-          error_message: error.message,
-          timestamp: Time.now.iso8601
-        )
+          error_message: error.message
+        ).merge(generate_timestamp)
 
         self.emit_event(event_name, error_payload)
         raise
@@ -75,10 +139,12 @@ module DSPy
 
     # Emit event without timing (for discrete events)
     def self.emit(event_name, payload = {})
+      # Handle nil payload
+      payload ||= {}
+      
       enhanced_payload = payload.merge(
-        timestamp: Time.now.iso8601,
         status: payload[:status] || 'success'
-      )
+      ).merge(generate_timestamp)
 
       self.emit_event(event_name, enhanced_payload)
     end
@@ -101,13 +167,128 @@ module DSPy
     end
 
     def self.emit_event(event_name, payload)
-      # Ensure logger subscriber is initialized
-      logger_subscriber
+      # Only emit events - subscribers self-register when explicitly created
       notifications.instrument(event_name, payload)
     end
 
     def self.setup_subscribers
-      # Lazy initialization - will be created when first accessed
+      config = DSPy.config.instrumentation
+      
+      # Return early if instrumentation is disabled
+      return unless config.enabled
+      
+      # Validate configuration first
+      DSPy.validate_instrumentation!
+      
+      # Setup each configured subscriber
+      config.subscribers.each do |subscriber_type|
+        setup_subscriber(subscriber_type)
+      end
+    end
+
+    def self.setup_subscriber(subscriber_type)
+      case subscriber_type
+      when :logger
+        setup_logger_subscriber
+      when :otel
+        setup_otel_subscriber if otel_available?
+      when :newrelic
+        setup_newrelic_subscriber if newrelic_available?
+      when :langfuse
+        setup_langfuse_subscriber if langfuse_available?
+      else
+        raise ArgumentError, "Unknown subscriber type: #{subscriber_type}"
+      end
+    rescue LoadError => e
+      DSPy.logger.warn "Failed to setup #{subscriber_type} subscriber: #{e.message}"
+    end
+
+    def self.setup_logger_subscriber
+      # Create subscriber - it will read configuration when handling events
+      logger_subscriber
+    end
+
+    def self.setup_otel_subscriber
+      # Create subscriber - it will read configuration when handling events
+      otel_subscriber
+    end
+
+    def self.setup_newrelic_subscriber
+      # Create subscriber - it will read configuration when handling events
+      newrelic_subscriber
+    end
+
+    def self.setup_langfuse_subscriber
+      # Create subscriber - it will read configuration when handling events
+      langfuse_subscriber
+    end
+
+    # Dependency checking methods
+    def self.otel_available?
+      begin
+        require 'opentelemetry/sdk'
+        true
+      rescue LoadError
+        false
+      end
+    end
+
+    def self.newrelic_available?
+      begin
+        require 'newrelic_rpm'
+        true
+      rescue LoadError
+        false
+      end
+    end
+
+    def self.langfuse_available?
+      begin
+        require 'langfuse'
+        true
+      rescue LoadError
+        false
+      end
+    end
+
+    # Generate timestamp in the configured format
+    def self.generate_timestamp
+      case DSPy.config.instrumentation.timestamp_format
+      when DSPy::TimestampFormat::ISO8601
+        { timestamp: Time.now.iso8601 }
+      when DSPy::TimestampFormat::RFC3339_NANO
+        { timestamp: Time.now.strftime('%Y-%m-%dT%H:%M:%S.%9N%z') }
+      when DSPy::TimestampFormat::UNIX_NANO
+        { timestamp_ns: (Time.now.to_f * 1_000_000_000).to_i }
+      else
+        { timestamp: Time.now.iso8601 }  # Fallback to iso8601
+      end
+    end
+
+    # Legacy setup method for backward compatibility
+    def self.setup_subscribers_legacy
+      # Legacy initialization - will be created when first accessed
+      # Force initialization of enabled subscribers
+      logger_subscriber
+      
+      # Only initialize if dependencies are available
+      begin
+        otel_subscriber if ENV['OTEL_EXPORTER_OTLP_ENDPOINT'] || defined?(OpenTelemetry)
+      rescue LoadError
+        # OpenTelemetry not available, skip
+      end
+
+      begin
+        newrelic_subscriber if defined?(NewRelic)
+      rescue LoadError
+        # New Relic not available, skip
+      end
+
+      begin
+        langfuse_subscriber if ENV['LANGFUSE_SECRET_KEY'] || defined?(Langfuse)
+      rescue LoadError
+        # Langfuse not available, skip
+      end
     end
   end
 end

data/lib/dspy/lm/adapter_factory.rb

@@ -7,8 +7,7 @@ module DSPy
       # Maps provider prefixes to adapter classes
       ADAPTER_MAP = {
         'openai' => 'OpenAIAdapter',
-        'anthropic' => 'AnthropicAdapter',
-        'ruby_llm' => 'RubyLLMAdapter'
+        'anthropic' => 'AnthropicAdapter'
       }.freeze
 
       class << self
@@ -27,13 +26,12 @@ module DSPy
 
         # Parse model_id to determine provider and model
         def parse_model_id(model_id)
-          if model_id.include?('/')
-            provider, model = model_id.split('/', 2)
-            [provider, model]
-          else
-            # Legacy format: assume ruby_llm for backward compatibility
-            ['ruby_llm', model_id]
+          unless model_id.include?('/')
+            raise ArgumentError, "model_id must include provider (e.g., 'openai/gpt-4', 'anthropic/claude-3'). Legacy format without provider is no longer supported."
           end
+          
+          provider, model = model_id.split('/', 2)
+          [provider, model]
         end
 
         def get_adapter_class(provider)