rubyllm-observ 0.6.6 → 0.6.8

data/app/services/observ/chat_instrumenter.rb

@@ -11,6 +11,7 @@ module Observ
       @current_trace = nil
       @current_tool_span = nil
       @original_ask_method = nil
+      @original_complete_method = nil
       @instrumented = false
     end
 
@@ -18,6 +19,7 @@ module Observ
       return if @instrumented
 
       wrap_ask_method
+      wrap_complete_method
       setup_event_handlers
       @instrumented = true
 
@@ -101,6 +103,66 @@ module Observ
       raise
     end
 
+    def wrap_complete_method
+      return if @original_complete_method
+
+      @original_complete_method = chat.method(:complete)
+      instrumenter = self
+
+      chat.define_singleton_method(:complete) do |**kwargs, &block|
+        instrumenter.send(:handle_complete_call, self, kwargs, block)
+      end
+    end
+
+    # Handle complete calls - similar to ask but uses existing messages
+    # instead of adding a new user message
+    def handle_complete_call(chat_instance, kwargs, block)
+      # Get the last user message for trace input
+      last_user_message = find_messages_by_role(chat_instance.messages, :user).last
+      user_message_content = last_user_message&.content
+
+      # Track if this is an ephemeral trace (created just for this call)
+      is_ephemeral_trace = @current_trace.nil?
+
+      trace = @current_trace || create_trace(
+        name: "chat.complete",
+        input: { text: user_message_content },
+        metadata: {}
+      )
+
+      model_id = extract_model_id(chat_instance)
+
+      # Extract prompt metadata from the chat's agent (if available)
+      prompt_metadata = extract_prompt_metadata(chat_instance)
+
+      generation = trace.create_generation(
+        name: "llm_call",
+        metadata: @context.merge(kwargs.slice(:temperature, :max_tokens)),
+        model: model_id,
+        model_parameters: extract_model_parameters(chat_instance),
+        **prompt_metadata
+      )
+
+      messages_snapshot = capture_messages(chat_instance)
+      generation.set_input(user_message_content, messages: messages_snapshot)
+
+      call_start_time = Time.current
+      result = @original_complete_method.call(**kwargs, &block)
+
+      finalize_generation(generation, result, call_start_time)
+
+      if is_ephemeral_trace
+        link_trace_to_message(trace, chat_instance, call_start_time)
+        trace.finalize(output: result.content)
+        @current_trace = nil
+      end
+
+      result
+    rescue StandardError => e
+      handle_error(e, trace, generation)
+      raise
+    end
+
     def setup_event_handlers
       setup_tool_call_handler
       setup_tool_result_handler
@@ -310,6 +372,26 @@ module Observ
       []
     end
 
+    # Find messages by role, handling both ActiveRecord relations and plain Arrays.
+    # ActiveRecord-backed Chat models return relations with .where(), while raw
+    # RubyLLM::Chat objects return plain Arrays.
+    #
+    # Note: We use a rescue block because some objects may claim to respond_to?(:where)
+    # but fail when the method is actually called (edge cases with proxies or custom objects).
+    def find_messages_by_role(messages, role)
+      role_str = role.to_s
+      if messages.respond_to?(:where)
+        begin
+          messages.where(role: role)
+        rescue NoMethodError
+          # Fallback to array filtering if where method doesn't actually exist
+          messages.select { |m| m.role.to_s == role_str }
+        end
+      else
+        messages.select { |m| m.role.to_s == role_str }
+      end
+    end
+
     def extract_usage(result)
       usage = {
         input_tokens: result.input_tokens || 0,
@@ -496,7 +578,7 @@ module Observ
         elsif value.is_a?(Hash)
           truncate_large_hash(value)
         elsif value.is_a?(Array) && value.size > 100
-          value[0..99] + [ "... #{value.size - 100} more items" ]
+          value[0..99] + ["... #{value.size - 100} more items"]
         else
           value
         end
@@ -506,13 +588,21 @@ module Observ
     def link_trace_to_message(trace, chat_instance, call_start_time)
       return unless chat_instance.respond_to?(:messages)
 
-      assistant_message = chat_instance.messages
-        .where(role: "assistant")
-        .where("created_at >= ?", call_start_time)
-        .order(created_at: :desc)
-        .first
+      messages = chat_instance.messages
+      assistant_message = if messages.respond_to?(:where)
+        # ActiveRecord-backed Chat models support query methods
+        messages
+          .where(role: "assistant")
+          .where("created_at >= ?", call_start_time)
+          .order(created_at: :desc)
+          .first
+      else
+        # Raw RubyLLM::Chat objects return plain Arrays without timestamps.
+        # Get the last assistant message (most recent from this call).
+        find_messages_by_role(messages, :assistant).last
+      end
 
-      if assistant_message
+      if assistant_message&.respond_to?(:id) && assistant_message.id
         trace.update(message_id: assistant_message.id)
         Rails.logger.info "[Observability] Linked trace #{trace.trace_id} to message #{assistant_message.id}"
       end

data/app/services/observ/concerns/observable_service.rb

@@ -12,11 +12,12 @@ module Observ
     #   class MyService
     #     include Observ::Concerns::ObservableService
     #
-    #     def initialize(observability_session: nil)
+    #     def initialize(observability_session: nil, moderate: false)
     #       initialize_observability(
     #         observability_session,
     #         service_name: "my_service",
-    #         metadata: { custom: "data" }
+    #         metadata: { custom: "data" },
+    #         moderate: moderate
     #       )
     #     end
     #
@@ -24,6 +25,7 @@ module Observ
     #       with_observability do |session|
     #         # Your service logic here
     #         # Session automatically finalized on success/error
+    #         # If moderate: true, content moderation runs after finalization
     #       end
     #     end
     #   end
@@ -39,7 +41,10 @@ module Observ
       # @param session_or_false [Observ::Session, false, nil] Session to use, false to disable, nil to auto-create
       # @param service_name [String] Name of the service (used in session metadata)
       # @param metadata [Hash] Additional metadata to include in the session
-      def initialize_observability(session_or_false = nil, service_name:, metadata: {})
+      # @param moderate [Boolean] Whether to run content moderation after session finalization
+      def initialize_observability(session_or_false = nil, service_name:, metadata: {}, moderate: false)
+        @moderate_on_complete = moderate
+
         if session_or_false == false
           # Explicitly disable observability
           @observability = nil
@@ -61,6 +66,9 @@ module Observ
       # whether it succeeds or raises an error. Only sessions owned by this
       # service instance (i.e., auto-created sessions) will be finalized.
       #
+      # If moderate: true was passed to initialize_observability, content moderation
+      # will be enqueued after the session is finalized.
+      #
       # @yield [session] The observability session (may be nil if disabled)
       # @return The result of the block
       #
@@ -72,9 +80,11 @@ module Observ
       def with_observability(&block)
         result = block.call(@observability)
         finalize_service_session if @owns_session
+        enqueue_moderation if should_moderate?
         result
       rescue StandardError
         finalize_service_session if @owns_session
+        enqueue_moderation if should_moderate?
         raise
       end
 
@@ -97,6 +107,74 @@ module Observ
         @observability.instrument_chat(chat, context: context)
       end
 
+      # Instrument RubyLLM.embed for observability
+      #
+      # This wraps the RubyLLM.embed class method to automatically create traces
+      # and track embedding calls within the observability session.
+      #
+      # @param context [Hash] Additional context to include in traces
+      # @return [Observ::EmbeddingInstrumenter, nil] The instrumenter or nil if observability is disabled
+      #
+      # @example
+      #   instrument_embedding(context: { operation: "semantic_search" })
+      #   embedding = RubyLLM.embed("Search query")
+      def instrument_embedding(context: {})
+        return unless @observability
+
+        @observability.instrument_embedding(context: context)
+      end
+
+      # Instrument RubyLLM.paint for observability
+      #
+      # This wraps the RubyLLM.paint class method to automatically create traces
+      # and track image generation calls within the observability session.
+      #
+      # @param context [Hash] Additional context to include in traces
+      # @return [Observ::ImageGenerationInstrumenter, nil] The instrumenter or nil if observability is disabled
+      #
+      # @example
+      #   instrument_image_generation(context: { operation: "product_image" })
+      #   image = RubyLLM.paint("A modern logo")
+      def instrument_image_generation(context: {})
+        return unless @observability
+
+        @observability.instrument_image_generation(context: context)
+      end
+
+      # Instrument RubyLLM.transcribe for observability
+      #
+      # This wraps the RubyLLM.transcribe class method to automatically create traces
+      # and track transcription calls within the observability session.
+      #
+      # @param context [Hash] Additional context to include in traces
+      # @return [Observ::TranscriptionInstrumenter, nil] The instrumenter or nil if observability is disabled
+      #
+      # @example
+      #   instrument_transcription(context: { operation: "meeting_notes" })
+      #   transcript = RubyLLM.transcribe("meeting.wav")
+      def instrument_transcription(context: {})
+        return unless @observability
+
+        @observability.instrument_transcription(context: context)
+      end
+
+      # Instrument RubyLLM.moderate for observability
+      #
+      # This wraps the RubyLLM.moderate class method to automatically create traces
+      # and track moderation calls within the observability session.
+      #
+      # @param context [Hash] Additional context to include in traces
+      # @return [Observ::ModerationInstrumenter, nil] The instrumenter or nil if observability is disabled
+      #
+      # @example
+      #   instrument_moderation(context: { operation: "user_input_check" })
+      #   result = RubyLLM.moderate(user_input)
+      def instrument_moderation(context: {})
+        return unless @observability
+
+        @observability.instrument_moderation(context: context)
+      end
+
       private
 
       # Create a new observability session for this service
@@ -138,6 +216,33 @@ module Observ
           "[#{self.class.name}] Failed to finalize session: #{e.message}"
         )
       end
+
+      # Check if moderation should be enqueued
+      #
+      # Moderation is only enqueued when:
+      # - moderate: true was passed to initialize_observability
+      # - This service owns the session (created it)
+      # - The session exists
+      #
+      # @return [Boolean] Whether to enqueue moderation
+      def should_moderate?
+        @moderate_on_complete && @owns_session && @observability.present?
+      end
+
+      # Enqueue content moderation for the session
+      #
+      # This schedules a background job to run content moderation on all
+      # traces in the session, flagging any problematic content for review.
+      def enqueue_moderation
+        Observ::ModerationGuardrailJob.perform_later(session_id: @observability.id)
+        Rails.logger.debug(
+          "[#{self.class.name}] Moderation enqueued for session: #{@observability.session_id}"
+        )
+      rescue StandardError => e
+        Rails.logger.error(
+          "[#{self.class.name}] Failed to enqueue moderation: #{e.message}"
+        )
+      end
     end
   end
 end

data/app/services/observ/dataset_runner_service.rb

@@ -82,7 +82,7 @@ module Observ
           dataset_item_id: run_item.dataset_item_id,
           agent_class: dataset.agent_class
         },
-        tags: [ "dataset_evaluation", dataset.name, dataset_run.name ]
+        tags: ["dataset_evaluation", dataset.name, dataset_run.name]
       )
     end
 

data/app/services/observ/embedding_instrumenter.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+module Observ
+  class EmbeddingInstrumenter
+    attr_reader :session, :context
+
+    def initialize(session, context: {})
+      @session = session
+      @context = context
+      @original_embed_method = nil
+      @instrumented = false
+    end
+
+    def instrument!
+      return if @instrumented
+
+      wrap_embed_method
+      @instrumented = true
+
+      Rails.logger.info "[Observability] Instrumented RubyLLM.embed for session #{session.session_id}"
+    end
+
+    def uninstrument!
+      return unless @instrumented
+      return unless @original_embed_method
+
+      RubyLLM.define_singleton_method(:embed, @original_embed_method)
+      @instrumented = false
+
+      Rails.logger.info "[Observability] Uninstrumented RubyLLM.embed"
+    end
+
+    private
+
+    def wrap_embed_method
+      return if @original_embed_method
+
+      @original_embed_method = RubyLLM.method(:embed)
+      instrumenter = self
+
+      RubyLLM.define_singleton_method(:embed) do |*args, **kwargs|
+        instrumenter.send(:handle_embed_call, args, kwargs)
+      end
+    end
+
+    def handle_embed_call(args, kwargs)
+      texts = args[0]
+      model_id = kwargs[:model] || default_embedding_model
+
+      trace = session.create_trace(
+        name: "embedding",
+        input: format_input(texts),
+        metadata: @context.merge(
+          batch_size: Array(texts).size,
+          model: model_id
+        )
+      )
+
+      embedding_obs = trace.create_embedding(
+        name: "embed",
+        model: model_id,
+        metadata: {
+          batch_size: Array(texts).size
+        }
+      )
+      embedding_obs.set_input(texts)
+
+      call_start_time = Time.current
+      result = @original_embed_method.call(*args, **kwargs)
+
+      finalize_embedding(embedding_obs, result, call_start_time)
+      trace.finalize(
+        output: format_output(result),
+        metadata: { dimensions: extract_dimensions(result) }
+      )
+
+      result
+    rescue StandardError => e
+      handle_error(e, trace, embedding_obs)
+      raise
+    end
+
+    def finalize_embedding(embedding_obs, result, _call_start_time)
+      usage = extract_usage(result)
+      cost = calculate_cost(result)
+      dimensions = extract_dimensions(result)
+      vectors_count = extract_vectors_count(result)
+
+      embedding_obs.finalize(
+        output: format_output(result),
+        usage: usage,
+        cost_usd: cost
+      )
+
+      embedding_obs.update!(
+        metadata: embedding_obs.metadata.merge(
+          dimensions: dimensions,
+          vectors_count: vectors_count
+        )
+      )
+    end
+
+    def extract_usage(result)
+      {
+        input_tokens: result.input_tokens || 0,
+        total_tokens: result.input_tokens || 0
+      }
+    end
+
+    def calculate_cost(result)
+      model_id = result.model
+      return 0.0 unless model_id
+
+      model_info = RubyLLM.models.find(model_id)
+      return 0.0 unless model_info&.input_price_per_million
+
+      input_tokens = result.input_tokens || 0
+      (input_tokens * model_info.input_price_per_million / 1_000_000.0).round(6)
+    rescue StandardError => e
+      Rails.logger.warn "[Observability] Failed to calculate embedding cost: #{e.message}"
+      0.0
+    end
+
+    def extract_dimensions(result)
+      vectors = result.vectors
+      return nil unless vectors
+
+      # Handle both single embedding and batch embeddings
+      if vectors.first.is_a?(Array)
+        vectors.first.length
+      else
+        vectors.length
+      end
+    end
+
+    def extract_vectors_count(result)
+      vectors = result.vectors
+      return 1 unless vectors
+
+      # Handle both single embedding and batch embeddings
+      if vectors.first.is_a?(Array)
+        vectors.length
+      else
+        1
+      end
+    end
+
+    def format_input(texts)
+      if texts.is_a?(Array)
+        { texts: texts, count: texts.size }
+      else
+        { text: texts }
+      end
+    end
+
+    def format_output(result)
+      {
+        model: result.model,
+        dimensions: extract_dimensions(result),
+        vectors_count: extract_vectors_count(result)
+      }
+    end
+
+    def default_embedding_model
+      if RubyLLM.config.respond_to?(:default_embedding_model)
+        RubyLLM.config.default_embedding_model
+      else
+        "text-embedding-3-small"
+      end
+    end
+
+    def handle_error(error, trace, embedding_obs)
+      return unless trace
+
+      error_span = trace.create_span(
+        name: "error",
+        metadata: {
+          error_type: error.class.name,
+          level: "ERROR"
+        },
+        input: {
+          error_message: error.message,
+          backtrace: error.backtrace&.first(10)
+        }.to_json
+      )
+      error_span.finalize(output: { error_captured: true }.to_json)
+
+      embedding_obs&.update(status_message: "FAILED") rescue nil
+
+      Rails.logger.error "[Observability] Embedding error captured: #{error.class.name} - #{error.message}"
+    end
+  end
+end

data/app/services/observ/evaluator_runner_service.rb

@@ -42,7 +42,7 @@ module Observ
 
     def default_evaluator_configs
       # Default to exact_match if no config specified
-      [ { "type" => "exact_match" } ]
+      [{ "type" => "exact_match" }]
     end
 
     def build_evaluator(config)

data/app/services/observ/evaluators/contains_evaluator.rb

@@ -32,7 +32,7 @@ module Observ
         when Array
           expected
         when String
-          [ expected ]
+          [expected]
         else
           []
         end

data/app/services/observ/guardrail_service.rb

@@ -48,7 +48,7 @@ module Observ
                      .left_joins(:review_item)
                      .where(observ_review_items: { id: nil })
 
-        sample_size = [ (items.count * percentage / 100.0).ceil, 1 ].max
+        sample_size = [(items.count * percentage / 100.0).ceil, 1].max
 
         items.order("RANDOM()").limit(sample_size).find_each do |item|
           item.enqueue_for_review!(reason: "random_sample", priority: :normal)
@@ -65,6 +65,15 @@ module Observ
             condition: ->(t) { t.metadata&.dig("error").present? },
             details: ->(t) { { error: t.metadata["error"] } }
           },
+          {
+            name: :error_span,
+            priority: :critical,
+            condition: ->(t) { t.observations.exists?(type: "Observ::Span", name: "error") },
+            details: ->(t) {
+              error_span = t.observations.find_by(type: "Observ::Span", name: "error")
+              { span_id: error_span&.observation_id, metadata: error_span&.metadata }
+            }
+          },
           {
             name: :high_cost,
             priority: :high,