dspy 0.1.0 → 0.3.0

data/lib/dspy/instrumentation/token_tracker.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module DSPy
+  module Instrumentation
+    # Utility for extracting token usage from different LM adapters
+    # Uses actual token counts from API responses for accuracy
+    module TokenTracker
+      extend self
+
+      # Extract actual token usage from API responses
+      def extract_token_usage(response, provider)
+        case provider.to_s.downcase
+        when 'openai'
+          extract_openai_tokens(response)
+        when 'anthropic'
+          extract_anthropic_tokens(response)
+        else
+          {} # No token information for other providers
+        end
+      end
+
+      private
+
+      def extract_openai_tokens(response)
+        return {} unless response&.usage
+
+        usage = response.usage
+        return {} unless usage.is_a?(Hash)
+        
+        {
+          tokens_input: usage[:prompt_tokens] || usage['prompt_tokens'],
+          tokens_output: usage[:completion_tokens] || usage['completion_tokens'],
+          tokens_total: usage[:total_tokens] || usage['total_tokens']
+        }
+      end
+
+      def extract_anthropic_tokens(response)
+        return {} unless response&.usage
+
+        usage = response.usage
+        return {} unless usage.is_a?(Hash)
+        
+        input_tokens = usage[:input_tokens] || usage['input_tokens'] || 0
+        output_tokens = usage[:output_tokens] || usage['output_tokens'] || 0
+        
+        {
+          tokens_input: input_tokens,
+          tokens_output: output_tokens,
+          tokens_total: input_tokens + output_tokens
+        }
+      end
+    end
+  end
+end

data/lib/dspy/instrumentation.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'dry-monitor'
+require 'dry-configurable'
+
+module DSPy
+  # Core instrumentation module using dry-monitor for event emission
+  # Provides extension points for logging, Langfuse, New Relic, and custom monitoring
+  module Instrumentation
+
+    def self.notifications
+      @notifications ||= Dry::Monitor::Notifications.new(:dspy).tap do |n|
+        # Register all DSPy events
+        n.register_event('dspy.lm.request')
+        n.register_event('dspy.lm.tokens')
+        n.register_event('dspy.lm.response.parsed')
+        n.register_event('dspy.predict')
+        n.register_event('dspy.predict.validation_error')
+        n.register_event('dspy.chain_of_thought')
+        n.register_event('dspy.chain_of_thought.reasoning_step')
+        n.register_event('dspy.react')
+        n.register_event('dspy.react.tool_call')
+        n.register_event('dspy.react.iteration_complete')
+        n.register_event('dspy.react.max_iterations')
+      end
+    end
+
+    # High-precision timing for performance tracking
+    def self.instrument(event_name, payload = {}, &block)
+      # If no block is given, return early
+      return unless block_given?
+
+      start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      start_cpu = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID)
+
+      begin
+        result = yield
+
+        end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end_cpu = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID)
+
+        enhanced_payload = payload.merge(
+          duration_ms: ((end_time - start_time) * 1000).round(2),
+          cpu_time_ms: ((end_cpu - start_cpu) * 1000).round(2),
+          status: 'success',
+          timestamp: Time.now.iso8601
+        )
+
+        self.emit_event(event_name, enhanced_payload)
+        result
+      rescue => error
+        end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end_cpu = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID)
+
+        error_payload = payload.merge(
+          duration_ms: ((end_time - start_time) * 1000).round(2),
+          cpu_time_ms: ((end_cpu - start_cpu) * 1000).round(2),
+          status: 'error',
+          error_type: error.class.name,
+          error_message: error.message,
+          timestamp: Time.now.iso8601
+        )
+
+        self.emit_event(event_name, error_payload)
+        raise
+      end
+    end
+
+    # Emit event without timing (for discrete events)
+    def self.emit(event_name, payload = {})
+      enhanced_payload = payload.merge(
+        timestamp: Time.now.iso8601,
+        status: payload[:status] || 'success'
+      )
+
+      self.emit_event(event_name, enhanced_payload)
+    end
+
+    # Register additional events dynamically (useful for testing)
+    def self.register_event(event_name)
+      notifications.register_event(event_name)
+    end
+
+    # Subscribe to DSPy instrumentation events
+    def self.subscribe(event_pattern = nil, &block)
+      if event_pattern
+        notifications.subscribe(event_pattern, &block)
+      else
+        # Subscribe to all DSPy events
+        %w[dspy.lm.request dspy.lm.tokens dspy.lm.response.parsed dspy.predict dspy.predict.validation_error dspy.chain_of_thought dspy.chain_of_thought.reasoning_step dspy.react dspy.react.tool_call dspy.react.iteration_complete dspy.react.max_iterations].each do |event_name|
+          notifications.subscribe(event_name, &block)
+        end
+      end
+    end
+
+    def self.emit_event(event_name, payload)
+      notifications.instrument(event_name, payload)
+    end
+  end
+end

data/lib/dspy/lm/adapter.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module DSPy
+  class LM
+    # Base adapter interface for all LM providers
+    class Adapter
+      attr_reader :model, :api_key
+
+      def initialize(model:, api_key:)
+        @model = model
+        @api_key = api_key
+        validate_configuration!
+      end
+
+      # Chat interface that all adapters must implement
+      # @param messages [Array<Hash>] Array of message hashes with :role and :content
+      # @param block [Proc] Optional streaming block
+      # @return [DSPy::LM::Response] Normalized response
+      def chat(messages:, &block)
+        raise NotImplementedError, "Subclasses must implement #chat method"
+      end
+
+      private
+
+      def validate_configuration!
+        raise ConfigurationError, "Model is required" if model.nil? || model.empty?
+        raise ConfigurationError, "API key is required" if api_key.nil? || api_key.empty?
+      end
+
+      # Helper method to normalize message format
+      def normalize_messages(messages)
+        messages.map do |msg|
+          {
+            role: msg[:role].to_s,
+            content: msg[:content].to_s
+          }
+        end
+      end
+    end
+  end
+end

data/lib/dspy/lm/adapter_factory.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module DSPy
+  class LM
+    # Factory for creating appropriate adapters based on model_id
+    class AdapterFactory
+      # Maps provider prefixes to adapter classes
+      ADAPTER_MAP = {
+        'openai' => 'OpenAIAdapter',
+        'anthropic' => 'AnthropicAdapter',
+        'ruby_llm' => 'RubyLLMAdapter'
+      }.freeze
+
+      class << self
+        # Creates an adapter instance based on model_id
+        # @param model_id [String] Full model identifier (e.g., "openai/gpt-4")
+        # @param api_key [String] API key for the provider
+        # @return [DSPy::LM::Adapter] Appropriate adapter instance
+        def create(model_id, api_key:)
+          provider, model = parse_model_id(model_id)
+          adapter_class = get_adapter_class(provider)
+          
+          adapter_class.new(model: model, api_key: api_key)
+        end
+
+        private
+
+        # Parse model_id to determine provider and model
+        def parse_model_id(model_id)
+          if model_id.include?('/')
+            provider, model = model_id.split('/', 2)
+            [provider, model]
+          else
+            # Legacy format: assume ruby_llm for backward compatibility
+            ['ruby_llm', model_id]
+          end
+        end
+
+        def get_adapter_class(provider)
+          adapter_class_name = ADAPTER_MAP[provider]
+          
+          unless adapter_class_name
+            available_providers = ADAPTER_MAP.keys.join(', ')
+            raise UnsupportedProviderError, 
+                  "Unsupported provider: #{provider}. Available: #{available_providers}"
+          end
+
+          begin
+            Object.const_get("DSPy::LM::#{adapter_class_name}")
+          rescue NameError
+            raise UnsupportedProviderError, 
+                  "Adapter not found: DSPy::LM::#{adapter_class_name}. " \
+                  "Make sure the corresponding gem is installed."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dspy/lm/adapters/anthropic_adapter.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require 'anthropic'
+
+module DSPy
+  class LM
+    class AnthropicAdapter < Adapter
+      def initialize(model:, api_key:)
+        super
+        @client = Anthropic::Client.new(api_key: api_key)
+      end
+
+      def chat(messages:, &block)
+        # Anthropic requires system message to be separate from messages
+        system_message, user_messages = extract_system_message(normalize_messages(messages))
+        
+        request_params = {
+          model: model,
+          messages: user_messages,
+          max_tokens: 4096, # Required for Anthropic
+          temperature: 0.0 # DSPy default for deterministic responses
+        }
+
+        # Add system message if present
+        request_params[:system] = system_message if system_message
+
+        # Add streaming if block provided
+        if block_given?
+          request_params[:stream] = true
+        end
+
+        begin
+          if block_given?
+            content = ""
+            @client.messages.stream(**request_params) do |chunk|
+              if chunk.respond_to?(:delta) && chunk.delta.respond_to?(:text)
+                chunk_text = chunk.delta.text
+                content += chunk_text
+                block.call(chunk)
+              end
+            end
+            
+            Response.new(
+              content: content,
+              usage: nil, # Usage not available in streaming
+              metadata: {
+                provider: 'anthropic',
+                model: model,
+                streaming: true
+              }
+            )
+          else
+            response = @client.messages.create(**request_params)
+            
+            if response.respond_to?(:error) && response.error
+              raise AdapterError, "Anthropic API error: #{response.error}"
+            end
+
+            content = response.content.first.text if response.content.is_a?(Array) && response.content.first
+            usage = response.usage
+
+            Response.new(
+              content: content,
+              usage: usage.respond_to?(:to_h) ? usage.to_h : usage,
+              metadata: {
+                provider: 'anthropic',
+                model: model,
+                response_id: response.id,
+                role: response.role
+              }
+            )
+          end
+        rescue => e
+          raise AdapterError, "Anthropic adapter error: #{e.message}"
+        end
+      end
+
+      private
+
+      def extract_system_message(messages)
+        system_message = nil
+        user_messages = []
+
+        messages.each do |msg|
+          if msg[:role] == 'system'
+            system_message = msg[:content]
+          else
+            user_messages << msg
+          end
+        end
+
+        [system_message, user_messages]
+      end
+    end
+  end
+end

data/lib/dspy/lm/adapters/openai_adapter.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'openai'
+
+module DSPy
+  class LM
+    class OpenAIAdapter < Adapter
+      def initialize(model:, api_key:)
+        super
+        @client = OpenAI::Client.new(api_key: api_key)
+      end
+
+      def chat(messages:, &block)
+        request_params = {
+          model: model,
+          messages: normalize_messages(messages),
+          temperature: 0.0 # DSPy default for deterministic responses
+        }
+
+        # Add streaming if block provided
+        if block_given?
+          request_params[:stream] = proc do |chunk, _bytesize|
+            block.call(chunk) if chunk.dig("choices", 0, "delta", "content")
+          end
+        end
+
+        begin
+          response = @client.chat.completions.create(**request_params)
+          
+          if response.respond_to?(:error) && response.error
+            raise AdapterError, "OpenAI API error: #{response.error}"
+          end
+
+          content = response.choices.first.message.content
+          usage = response.usage
+
+          Response.new(
+            content: content,
+            usage: usage.respond_to?(:to_h) ? usage.to_h : usage,
+            metadata: {
+              provider: 'openai',
+              model: model,
+              response_id: response.id,
+              created: response.created
+            }
+          )
+        rescue => e
+          raise AdapterError, "OpenAI adapter error: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/dspy/lm/adapters/ruby_llm_adapter.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+begin
+  require 'ruby_llm'
+rescue LoadError
+  # ruby_llm is optional for backward compatibility
+end
+
+module DSPy
+  class LM
+    class RubyLLMAdapter < Adapter
+      def initialize(model:, api_key:)
+        super
+        
+        unless defined?(RubyLLM)
+          raise ConfigurationError, 
+                "ruby_llm gem is required for RubyLLMAdapter. " \
+                "Add 'gem \"ruby_llm\"' to your Gemfile."
+        end
+
+        configure_ruby_llm
+      end
+
+      def chat(messages:, &block)
+        begin
+          chat = RubyLLM.chat(model: model)
+          
+          # Add messages to chat
+          messages.each do |msg|
+            chat.add_message(role: msg[:role].to_sym, content: msg[:content])
+          end
+
+          # Get the last user message for ask method
+          last_user_message = messages.reverse.find { |msg| msg[:role] == 'user' }
+          
+          if last_user_message
+            # Remove the last user message since ask() will add it
+            chat.messages.pop if chat.messages.last&.content == last_user_message[:content]
+            chat.ask(last_user_message[:content], &block)
+          else
+            raise AdapterError, "No user message found in conversation"
+          end
+
+          content = chat.messages.last&.content || ""
+
+          Response.new(
+            content: content,
+            usage: nil, # ruby_llm doesn't provide usage info
+            metadata: {
+              provider: 'ruby_llm',
+              model: model,
+              message_count: chat.messages.length
+            }
+          )
+        rescue => e
+          raise AdapterError, "RubyLLM adapter error: #{e.message}"
+        end
+      end
+
+      private
+
+      def configure_ruby_llm
+        # Determine provider from model for configuration
+        if model.include?('gpt') || model.include?('openai')
+          RubyLLM.configure do |config|
+            config.openai_api_key = api_key
+          end
+        elsif model.include?('claude') || model.include?('anthropic')
+          RubyLLM.configure do |config|
+            config.anthropic_api_key = api_key
+          end
+        else
+          # Default to OpenAI configuration
+          RubyLLM.configure do |config|
+            config.openai_api_key = api_key
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dspy/lm/errors.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module DSPy
+  class LM
+    class Error < StandardError; end
+    class AdapterError < Error; end
+    class UnsupportedProviderError < Error; end
+    class ConfigurationError < Error; end
+  end
+end

data/lib/dspy/lm/response.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module DSPy
+  class LM
+    # Normalized response format for all LM providers
+    class Response
+      attr_reader :content, :usage, :metadata
+
+      def initialize(content:, usage: nil, metadata: {})
+        @content = content
+        @usage = usage
+        @metadata = metadata
+      end
+
+      def to_s
+        content
+      end
+
+      def to_h
+        {
+          content: content,
+          usage: usage,
+          metadata: metadata
+        }
+      end
+    end
+  end
+end

data/lib/dspy/lm.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+# Load adapter infrastructure
+require_relative 'lm/errors'
+require_relative 'lm/response'
+require_relative 'lm/adapter'
+require_relative 'lm/adapter_factory'
+
+# Load instrumentation
+require_relative 'instrumentation'
+require_relative 'instrumentation/token_tracker'
+
+# Load adapters
+require_relative 'lm/adapters/openai_adapter'
+require_relative 'lm/adapters/anthropic_adapter'
+require_relative 'lm/adapters/ruby_llm_adapter'
+
+module DSPy
+  class LM
+    attr_reader :model_id, :api_key, :model, :provider, :adapter
+
+    def initialize(model_id, api_key: nil)
+      @model_id = model_id
+      @api_key = api_key
+      
+      # Parse provider and model from model_id
+      @provider, @model = parse_model_id(model_id)
+      
+      # Create appropriate adapter
+      @adapter = AdapterFactory.create(model_id, api_key: api_key)
+    end
+
+    def chat(inference_module, input_values, &block)
+      signature_class = inference_module.signature_class
+      
+      # Build messages from inference module
+      messages = build_messages(inference_module, input_values)
+      
+      # Calculate input size for monitoring
+      input_text = messages.map { |m| m[:content] }.join(' ')
+      input_size = input_text.length
+      
+      # Instrument LM request
+      response = Instrumentation.instrument('dspy.lm.request', {
+        gen_ai_operation_name: 'chat',
+        gen_ai_system: provider,
+        gen_ai_request_model: model,
+        signature_class: signature_class.name,
+        provider: provider,
+        adapter_class: adapter.class.name,
+        input_size: input_size
+      }) do
+        adapter.chat(messages: messages, &block)
+      end
+      
+      # Extract actual token usage from response (more accurate than estimation)
+      token_usage = Instrumentation::TokenTracker.extract_token_usage(response, provider)
+      
+      # Emit token usage event if available
+      if token_usage.any?
+        Instrumentation.emit('dspy.lm.tokens', token_usage.merge({
+          gen_ai_system: provider,
+          gen_ai_request_model: model,
+          signature_class: signature_class.name
+        }))
+      end
+      
+      # Instrument response parsing
+      parsed_result = Instrumentation.instrument('dspy.lm.response.parsed', {
+        signature_class: signature_class.name,
+        provider: provider,
+        response_length: response.content&.length || 0
+      }) do
+        parse_response(response, input_values, signature_class)
+      end
+      
+      parsed_result
+    end
+
+    private
+
+    def parse_model_id(model_id)
+      if model_id.include?('/')
+        provider, model = model_id.split('/', 2)
+        [provider, model]
+      else
+        # Legacy format: assume ruby_llm for backward compatibility
+        ['ruby_llm', model_id]
+      end
+    end
+
+    def build_messages(inference_module, input_values)
+      messages = []
+      
+      # Add system message
+      system_prompt = inference_module.system_signature
+      messages << { role: 'system', content: system_prompt } if system_prompt
+      
+      # Add user message
+      user_prompt = inference_module.user_signature(input_values)
+      messages << { role: 'user', content: user_prompt }
+      
+      messages
+    end
+
+    def parse_response(response, input_values, signature_class)
+      # Try to parse the response as JSON
+      content = response.content
+
+      # Extract JSON if it's in a code block
+      if content.include?('```json')
+        content = content.split('```json').last.split('```').first.strip
+      elsif content.include?('```')
+        content = content.split('```').last.split('```').first.strip
+      end
+
+      begin
+        json_payload = JSON.parse(content)
+
+        # For Sorbet signatures, just return the parsed JSON
+        # The Predict will handle validation
+        json_payload
+      rescue JSON::ParserError
+        raise "Failed to parse LLM response as JSON: #{content}"
+      end
+    end
+  end
+end