rubyllm-observ 0.6.5 → 0.6.7

data/app/services/observ/embedding_instrumenter.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+module Observ
+  class EmbeddingInstrumenter
+    attr_reader :session, :context
+
+    def initialize(session, context: {})
+      @session = session
+      @context = context
+      @original_embed_method = nil
+      @instrumented = false
+    end
+
+    def instrument!
+      return if @instrumented
+
+      wrap_embed_method
+      @instrumented = true
+
+      Rails.logger.info "[Observability] Instrumented RubyLLM.embed for session #{session.session_id}"
+    end
+
+    def uninstrument!
+      return unless @instrumented
+      return unless @original_embed_method
+
+      RubyLLM.define_singleton_method(:embed, @original_embed_method)
+      @instrumented = false
+
+      Rails.logger.info "[Observability] Uninstrumented RubyLLM.embed"
+    end
+
+    private
+
+    def wrap_embed_method
+      return if @original_embed_method
+
+      @original_embed_method = RubyLLM.method(:embed)
+      instrumenter = self
+
+      RubyLLM.define_singleton_method(:embed) do |*args, **kwargs|
+        instrumenter.send(:handle_embed_call, args, kwargs)
+      end
+    end
+
+    def handle_embed_call(args, kwargs)
+      texts = args[0]
+      model_id = kwargs[:model] || default_embedding_model
+
+      trace = session.create_trace(
+        name: "embedding",
+        input: format_input(texts),
+        metadata: @context.merge(
+          batch_size: Array(texts).size,
+          model: model_id
+        )
+      )
+
+      embedding_obs = trace.create_embedding(
+        name: "embed",
+        model: model_id,
+        metadata: {
+          batch_size: Array(texts).size
+        }
+      )
+      embedding_obs.set_input(texts)
+
+      call_start_time = Time.current
+      result = @original_embed_method.call(*args, **kwargs)
+
+      finalize_embedding(embedding_obs, result, call_start_time)
+      trace.finalize(
+        output: format_output(result),
+        metadata: { dimensions: extract_dimensions(result) }
+      )
+
+      result
+    rescue StandardError => e
+      handle_error(e, trace, embedding_obs)
+      raise
+    end
+
+    def finalize_embedding(embedding_obs, result, _call_start_time)
+      usage = extract_usage(result)
+      cost = calculate_cost(result)
+      dimensions = extract_dimensions(result)
+      vectors_count = extract_vectors_count(result)
+
+      embedding_obs.finalize(
+        output: format_output(result),
+        usage: usage,
+        cost_usd: cost
+      )
+
+      embedding_obs.update!(
+        metadata: embedding_obs.metadata.merge(
+          dimensions: dimensions,
+          vectors_count: vectors_count
+        )
+      )
+    end
+
+    def extract_usage(result)
+      {
+        input_tokens: result.input_tokens || 0,
+        total_tokens: result.input_tokens || 0
+      }
+    end
+
+    def calculate_cost(result)
+      model_id = result.model
+      return 0.0 unless model_id
+
+      model_info = RubyLLM.models.find(model_id)
+      return 0.0 unless model_info&.input_price_per_million
+
+      input_tokens = result.input_tokens || 0
+      (input_tokens * model_info.input_price_per_million / 1_000_000.0).round(6)
+    rescue StandardError => e
+      Rails.logger.warn "[Observability] Failed to calculate embedding cost: #{e.message}"
+      0.0
+    end
+
+    def extract_dimensions(result)
+      vectors = result.vectors
+      return nil unless vectors
+
+      # Handle both single embedding and batch embeddings
+      if vectors.first.is_a?(Array)
+        vectors.first.length
+      else
+        vectors.length
+      end
+    end
+
+    def extract_vectors_count(result)
+      vectors = result.vectors
+      return 1 unless vectors
+
+      # Handle both single embedding and batch embeddings
+      if vectors.first.is_a?(Array)
+        vectors.length
+      else
+        1
+      end
+    end
+
+    def format_input(texts)
+      if texts.is_a?(Array)
+        { texts: texts, count: texts.size }
+      else
+        { text: texts }
+      end
+    end
+
+    def format_output(result)
+      {
+        model: result.model,
+        dimensions: extract_dimensions(result),
+        vectors_count: extract_vectors_count(result)
+      }
+    end
+
+    def default_embedding_model
+      if RubyLLM.config.respond_to?(:default_embedding_model)
+        RubyLLM.config.default_embedding_model
+      else
+        "text-embedding-3-small"
+      end
+    end
+
+    def handle_error(error, trace, embedding_obs)
+      return unless trace
+
+      error_span = trace.create_span(
+        name: "error",
+        metadata: {
+          error_type: error.class.name,
+          level: "ERROR"
+        },
+        input: {
+          error_message: error.message,
+          backtrace: error.backtrace&.first(10)
+        }.to_json
+      )
+      error_span.finalize(output: { error_captured: true }.to_json)
+
+      embedding_obs&.update(status_message: "FAILED") rescue nil
+
+      Rails.logger.error "[Observability] Embedding error captured: #{error.class.name} - #{error.message}"
+    end
+  end
+end

data/app/services/observ/guardrail_service.rb

@@ -65,6 +65,15 @@ module Observ
             condition: ->(t) { t.metadata&.dig("error").present? },
             details: ->(t) { { error: t.metadata["error"] } }
           },
+          {
+            name: :error_span,
+            priority: :critical,
+            condition: ->(t) { t.observations.exists?(type: "Observ::Span", name: "error") },
+            details: ->(t) {
+              error_span = t.observations.find_by(type: "Observ::Span", name: "error")
+              { span_id: error_span&.observation_id, metadata: error_span&.metadata }
+            }
+          },
           {
             name: :high_cost,
             priority: :high,

data/app/services/observ/image_generation_instrumenter.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+module Observ
+  class ImageGenerationInstrumenter
+    # Hardcoded pricing for image generation models (USD per image)
+    # Prices are organized by model_id, then by size, then by quality
+    # Source: https://openai.com/pricing, https://cloud.google.com/vertex-ai/pricing
+    IMAGE_PRICING = {
+      # OpenAI DALL-E 3 (size and quality based)
+      # Quality options: "standard", "hd"
+      "dall-e-3" => {
+        "1024x1024" => { "standard" => 0.04, "hd" => 0.08 },
+        "1792x1024" => { "standard" => 0.08, "hd" => 0.12 },
+        "1024x1792" => { "standard" => 0.08, "hd" => 0.12 }
+      },
+      # OpenAI DALL-E 2 (size based, no quality option)
+      "dall-e-2" => {
+        "1024x1024" => { "default" => 0.02 },
+        "512x512" => { "default" => 0.018 },
+        "256x256" => { "default" => 0.016 }
+      },
+      # OpenAI GPT-image-1 (token-based, estimated per-image costs)
+      # Quality options: "low", "medium", "high" (maps "standard" -> "medium")
+      # Source: "Image outputs cost approximately $0.01 (low), $0.04 (medium), $0.17 (high) for square images"
+      # Larger sizes are estimated at ~1.7x for 1792x1024 and ~2.9x for 1792x1792
+      "gpt-image-1" => {
+        "1024x1024" => { "low" => 0.01, "medium" => 0.04, "high" => 0.17 },
+        "1792x1024" => { "low" => 0.017, "medium" => 0.068, "high" => 0.29 },
+        "1024x1792" => { "low" => 0.017, "medium" => 0.068, "high" => 0.29 },
+        "1792x1792" => { "low" => 0.029, "medium" => 0.116, "high" => 0.49 },
+        "default" => { "low" => 0.01, "medium" => 0.04, "high" => 0.17 }
+      },
+      # OpenAI GPT-image-1-mini (token-based, estimated per-image costs)
+      # Approximately 5x cheaper than gpt-image-1 based on token pricing ratio
+      "gpt-image-1-mini" => {
+        "1024x1024" => { "low" => 0.002, "medium" => 0.008, "high" => 0.034 },
+        "1792x1024" => { "low" => 0.0034, "medium" => 0.0136, "high" => 0.058 },
+        "1024x1792" => { "low" => 0.0034, "medium" => 0.0136, "high" => 0.058 },
+        "1792x1792" => { "low" => 0.0058, "medium" => 0.0232, "high" => 0.098 },
+        "default" => { "low" => 0.002, "medium" => 0.008, "high" => 0.034 }
+      },
+      # Google Imagen models (flat rate per image)
+      "imagen-3.0-generate-002" => {
+        "default" => { "default" => 0.04 }
+      },
+      "imagen-4.0-generate-001" => {
+        "default" => { "default" => 0.04 }
+      },
+      "imagen-4.0-generate-preview-06-06" => {
+        "default" => { "default" => 0.04 }
+      },
+      "imagen-4.0-ultra-generate-preview-06-06" => {
+        "default" => { "default" => 0.08 }
+      }
+    }.freeze
+
+    # Maps quality names between different conventions
+    # DALL-E uses: "standard", "hd"
+    # GPT-image uses: "low", "medium", "high"
+    QUALITY_MAPPINGS = {
+      "standard" => "medium",  # Map DALL-E "standard" to GPT-image "medium"
+      "hd" => "high"           # Map DALL-E "hd" to GPT-image "high"
+    }.freeze
+
+    attr_reader :session, :context
+
+    def initialize(session, context: {})
+      @session = session
+      @context = context
+      @original_paint_method = nil
+      @instrumented = false
+    end
+
+    def instrument!
+      return if @instrumented
+
+      wrap_paint_method
+      @instrumented = true
+
+      Rails.logger.info "[Observability] Instrumented RubyLLM.paint for session #{session.session_id}"
+    end
+
+    def uninstrument!
+      return unless @instrumented
+      return unless @original_paint_method
+
+      RubyLLM.define_singleton_method(:paint, @original_paint_method)
+      @instrumented = false
+
+      Rails.logger.info "[Observability] Uninstrumented RubyLLM.paint"
+    end
+
+    private
+
+    def wrap_paint_method
+      return if @original_paint_method
+
+      @original_paint_method = RubyLLM.method(:paint)
+      instrumenter = self
+
+      RubyLLM.define_singleton_method(:paint) do |*args, **kwargs|
+        instrumenter.send(:handle_paint_call, args, kwargs)
+      end
+    end
+
+    def handle_paint_call(args, kwargs)
+      prompt = args[0]
+      model_id = kwargs[:model] || default_image_model
+      size = kwargs[:size] || "1024x1024"
+      quality = kwargs[:quality] || "standard"
+
+      trace = session.create_trace(
+        name: "image_generation",
+        input: { prompt: prompt },
+        metadata: @context.merge(
+          model: model_id,
+          size: size,
+          quality: quality
+        ).compact
+      )
+
+      image_obs = trace.create_image_generation(
+        name: "paint",
+        model: model_id,
+        metadata: {
+          size: size,
+          quality: quality
+        }.compact
+      )
+
+      result = @original_paint_method.call(*args, **kwargs)
+
+      finalize_image_generation(image_obs, result, prompt, size: size, quality: quality)
+      trace.finalize(
+        output: format_output(result),
+        metadata: { size: extract_size(result) || size, quality: quality }
+      )
+
+      result
+    rescue StandardError => e
+      handle_error(e, trace, image_obs)
+      raise
+    end
+
+    def finalize_image_generation(image_obs, result, prompt, size:, quality:)
+      cost = calculate_cost(result, size: size, quality: quality)
+
+      image_obs.finalize(
+        output: format_output(result),
+        usage: {},
+        cost_usd: cost
+      )
+
+      image_obs.update!(
+        input: prompt,
+        metadata: image_obs.metadata.merge(
+          revised_prompt: result.revised_prompt,
+          output_format: result.base64? ? "base64" : "url",
+          mime_type: result.mime_type,
+          size: extract_size(result) || size,
+          quality: quality
+        ).compact
+      )
+    end
+
+    def calculate_cost(result, size:, quality:)
+      model_id = result.model_id
+      return 0.0 unless model_id
+
+      lookup_image_price(model_id, size, quality)
+    rescue StandardError => e
+      Rails.logger.warn "[Observability] Failed to calculate image generation cost: #{e.message}"
+      0.0
+    end
+
+    def lookup_image_price(model_id, size, quality)
+      model_pricing = IMAGE_PRICING[model_id]
+      return 0.0 unless model_pricing
+
+      # Try exact size match, then "default"
+      size_pricing = model_pricing[size] || model_pricing["default"]
+      return 0.0 unless size_pricing
+
+      # Try exact quality match first
+      return size_pricing[quality] if size_pricing[quality]
+
+      # Try mapped quality (e.g., "standard" -> "medium" for GPT-image models)
+      mapped_quality = QUALITY_MAPPINGS[quality]
+      return size_pricing[mapped_quality] if mapped_quality && size_pricing[mapped_quality]
+
+      # Fall back to "standard", "medium", "default", or first available
+      size_pricing["standard"] ||
+        size_pricing["medium"] ||
+        size_pricing["default"] ||
+        size_pricing.values.first ||
+        0.0
+    end
+
+    def extract_size(result)
+      # Try to get size from result if available
+      result.respond_to?(:size) ? result.size : nil
+    end
+
+    def format_output(result)
+      {
+        model: result.model_id,
+        has_url: result.respond_to?(:url) && result.url.present?,
+        base64: result.base64?,
+        mime_type: result.mime_type,
+        revised_prompt: result.revised_prompt
+      }.compact
+    end
+
+    def default_image_model
+      if RubyLLM.config.respond_to?(:default_image_model)
+        RubyLLM.config.default_image_model
+      else
+        "dall-e-3"
+      end
+    end
+
+    def handle_error(error, trace, image_obs)
+      return unless trace
+
+      error_span = trace.create_span(
+        name: "error",
+        metadata: {
+          error_type: error.class.name,
+          level: "ERROR"
+        },
+        input: {
+          error_message: error.message,
+          backtrace: error.backtrace&.first(10)
+        }.to_json
+      )
+      error_span.finalize(output: { error_captured: true }.to_json)
+
+      image_obs&.update(status_message: "FAILED") rescue nil
+
+      Rails.logger.error "[Observability] Image generation error captured: #{error.class.name} - #{error.message}"
+    end
+  end
+end

data/app/services/observ/moderation_guardrail_service.rb

@@ -0,0 +1,235 @@
+# frozen_string_literal: true
+
+module Observ
+  class ModerationGuardrailService
+    include Observ::Concerns::ObservableService
+
+    # Score thresholds for different actions
+    THRESHOLDS = {
+      critical: 0.9,  # Auto-flag as critical
+      high: 0.7,      # Flag as high priority
+      review: 0.5     # Flag for normal review
+    }.freeze
+
+    # Categories that always trigger critical review
+    CRITICAL_CATEGORIES = %w[
+      sexual/minors
+      self-harm/intent
+      self-harm/instructions
+      violence/graphic
+    ].freeze
+
+    class Result
+      attr_reader :action, :reason, :priority, :details
+
+      def initialize(action:, reason: nil, priority: nil, details: {})
+        @action = action
+        @reason = reason
+        @priority = priority
+        @details = details
+      end
+
+      def flagged? = action == :flagged
+      def skipped? = action == :skipped
+      def passed? = action == :passed
+    end
+
+    def initialize(observability_session: nil)
+      initialize_observability(
+        observability_session,
+        service_name: "moderation_guardrail",
+        metadata: {}
+      )
+    end
+
+    # Evaluate a trace for moderation issues
+    #
+    # @param trace [Observ::Trace] The trace to evaluate
+    # @param moderate_input [Boolean] Whether to moderate input content
+    # @param moderate_output [Boolean] Whether to moderate output content
+    # @return [Result] The evaluation result
+    def evaluate_trace(trace, moderate_input: true, moderate_output: true)
+      return Result.new(action: :skipped, reason: "already_in_queue") if trace.in_review_queue?
+      return Result.new(action: :skipped, reason: "already_has_moderation") if has_existing_flags?(trace)
+
+      with_observability do |_session|
+        content = extract_trace_content(trace, moderate_input:, moderate_output:)
+        return Result.new(action: :skipped, reason: "no_content") if content.blank?
+
+        perform_moderation(trace, content)
+      end
+    rescue StandardError => e
+      Rails.logger.error "[ModerationGuardrailService] Failed to evaluate trace #{trace.id}: #{e.message}"
+      Result.new(action: :skipped, reason: "error", details: { error: e.message })
+    end
+
+    # Evaluate all traces in a session
+    #
+    # @param session [Observ::Session] The session to evaluate
+    # @return [Array<Result>] Results for each trace
+    def evaluate_session(session)
+      return [] if session.traces.empty?
+
+      session.traces.map do |trace|
+        evaluate_trace(trace)
+      end
+    end
+
+    # Evaluate session-level content (aggregated input/output)
+    #
+    # @param session [Observ::Session] The session to evaluate
+    # @return [Result] The evaluation result
+    def evaluate_session_content(session)
+      return Result.new(action: :skipped, reason: "already_in_queue") if session.in_review_queue?
+
+      with_observability do |_session|
+        content = extract_session_content(session)
+        return Result.new(action: :skipped, reason: "no_content") if content.blank?
+
+        perform_session_moderation(session, content)
+      end
+    rescue StandardError => e
+      Rails.logger.error "[ModerationGuardrailService] Failed to evaluate session #{session.id}: #{e.message}"
+      Result.new(action: :skipped, reason: "error", details: { error: e.message })
+    end
+
+    private
+
+    def has_existing_flags?(trace)
+      trace.moderations.any?(&:flagged?)
+    end
+
+    def extract_trace_content(trace, moderate_input:, moderate_output:)
+      parts = []
+      parts << extract_text(trace.input) if moderate_input
+      parts << extract_text(trace.output) if moderate_output
+      parts.compact.reject(&:blank?).join("\n\n---\n\n")
+    end
+
+    def extract_session_content(session)
+      session.traces.flat_map do |trace|
+        [extract_text(trace.input), extract_text(trace.output)]
+      end.compact.reject(&:blank?).join("\n\n---\n\n").truncate(10_000)
+    end
+
+    def extract_text(content)
+      return nil if content.blank?
+
+      case content
+      when String
+        content
+      when Hash
+        # Try common keys for text content
+        content["text"] || content["content"] || content["message"] ||
+          content[:text] || content[:content] || content[:message] ||
+          content.to_json
+      else
+        content.to_s
+      end
+    end
+
+    def perform_moderation(trace, content)
+      instrument_moderation(context: {
+        service: "moderation_guardrail",
+        trace_id: trace.id,
+        content_length: content.length
+      })
+
+      result = RubyLLM.moderate(content)
+
+      evaluate_and_enqueue(trace, result)
+    end
+
+    def perform_session_moderation(session, content)
+      instrument_moderation(context: {
+        service: "moderation_guardrail",
+        session_id: session.id,
+        content_length: content.length
+      })
+
+      result = RubyLLM.moderate(content)
+
+      evaluate_and_enqueue_session(session, result)
+    end
+
+    def evaluate_and_enqueue(trace, moderation_result)
+      priority = determine_priority(moderation_result)
+
+      if priority
+        details = build_details(moderation_result)
+        trace.enqueue_for_review!(
+          reason: "content_moderation",
+          priority: priority,
+          details: details
+        )
+
+        Result.new(
+          action: :flagged,
+          priority: priority,
+          details: details
+        )
+      else
+        Result.new(action: :passed)
+      end
+    end
+
+    def evaluate_and_enqueue_session(session, moderation_result)
+      priority = determine_priority(moderation_result)
+
+      if priority
+        details = build_details(moderation_result)
+        session.enqueue_for_review!(
+          reason: "content_moderation",
+          priority: priority,
+          details: details
+        )
+
+        Result.new(
+          action: :flagged,
+          priority: priority,
+          details: details
+        )
+      else
+        Result.new(action: :passed)
+      end
+    end
+
+    def determine_priority(result)
+      # Check for critical categories first
+      if (result.flagged_categories & CRITICAL_CATEGORIES).any?
+        return :critical
+      end
+
+      # Check if explicitly flagged
+      if result.flagged?
+        max_score = result.category_scores.values.max || 0
+        return max_score >= THRESHOLDS[:critical] ? :critical : :high
+      end
+
+      # Check score thresholds even if not flagged
+      max_score = result.category_scores.values.max || 0
+
+      if max_score >= THRESHOLDS[:high]
+        :high
+      elsif max_score >= THRESHOLDS[:review]
+        :normal
+      end
+    end
+
+    def build_details(result)
+      {
+        flagged: result.flagged?,
+        flagged_categories: result.flagged_categories,
+        highest_category: highest_category(result),
+        highest_score: result.category_scores.values.max&.round(4),
+        category_scores: result.category_scores.transform_values { |v| v.round(4) }
+      }
+    end
+
+    def highest_category(result)
+      return nil if result.category_scores.empty?
+
+      result.category_scores.max_by { |_, score| score }&.first
+    end
+  end
+end