rubyllm-observ 0.6.6 → 0.6.8

data/app/services/observ/image_generation_instrumenter.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+module Observ
+  class ImageGenerationInstrumenter
+    # Hardcoded pricing for image generation models (USD per image)
+    # Prices are organized by model_id, then by size, then by quality
+    # Source: https://openai.com/pricing, https://cloud.google.com/vertex-ai/pricing
+    IMAGE_PRICING = {
+      # OpenAI DALL-E 3 (size and quality based)
+      # Quality options: "standard", "hd"
+      "dall-e-3" => {
+        "1024x1024" => { "standard" => 0.04, "hd" => 0.08 },
+        "1792x1024" => { "standard" => 0.08, "hd" => 0.12 },
+        "1024x1792" => { "standard" => 0.08, "hd" => 0.12 }
+      },
+      # OpenAI DALL-E 2 (size based, no quality option)
+      "dall-e-2" => {
+        "1024x1024" => { "default" => 0.02 },
+        "512x512" => { "default" => 0.018 },
+        "256x256" => { "default" => 0.016 }
+      },
+      # OpenAI GPT-image-1 (token-based, estimated per-image costs)
+      # Quality options: "low", "medium", "high" (maps "standard" -> "medium")
+      # Source: "Image outputs cost approximately $0.01 (low), $0.04 (medium), $0.17 (high) for square images"
+      # Larger sizes are estimated at ~1.7x for 1792x1024 and ~2.9x for 1792x1792
+      "gpt-image-1" => {
+        "1024x1024" => { "low" => 0.01, "medium" => 0.04, "high" => 0.17 },
+        "1792x1024" => { "low" => 0.017, "medium" => 0.068, "high" => 0.29 },
+        "1024x1792" => { "low" => 0.017, "medium" => 0.068, "high" => 0.29 },
+        "1792x1792" => { "low" => 0.029, "medium" => 0.116, "high" => 0.49 },
+        "default" => { "low" => 0.01, "medium" => 0.04, "high" => 0.17 }
+      },
+      # OpenAI GPT-image-1-mini (token-based, estimated per-image costs)
+      # Approximately 5x cheaper than gpt-image-1 based on token pricing ratio
+      "gpt-image-1-mini" => {
+        "1024x1024" => { "low" => 0.002, "medium" => 0.008, "high" => 0.034 },
+        "1792x1024" => { "low" => 0.0034, "medium" => 0.0136, "high" => 0.058 },
+        "1024x1792" => { "low" => 0.0034, "medium" => 0.0136, "high" => 0.058 },
+        "1792x1792" => { "low" => 0.0058, "medium" => 0.0232, "high" => 0.098 },
+        "default" => { "low" => 0.002, "medium" => 0.008, "high" => 0.034 }
+      },
+      # Google Imagen models (flat rate per image)
+      "imagen-3.0-generate-002" => {
+        "default" => { "default" => 0.04 }
+      },
+      "imagen-4.0-generate-001" => {
+        "default" => { "default" => 0.04 }
+      },
+      "imagen-4.0-generate-preview-06-06" => {
+        "default" => { "default" => 0.04 }
+      },
+      "imagen-4.0-ultra-generate-preview-06-06" => {
+        "default" => { "default" => 0.08 }
+      }
+    }.freeze
+
+    # Maps quality names between different conventions
+    # DALL-E uses: "standard", "hd"
+    # GPT-image uses: "low", "medium", "high"
+    QUALITY_MAPPINGS = {
+      "standard" => "medium",  # Map DALL-E "standard" to GPT-image "medium"
+      "hd" => "high"           # Map DALL-E "hd" to GPT-image "high"
+    }.freeze
+
+    attr_reader :session, :context
+
+    def initialize(session, context: {})
+      @session = session
+      @context = context
+      @original_paint_method = nil
+      @instrumented = false
+    end
+
+    def instrument!
+      return if @instrumented
+
+      wrap_paint_method
+      @instrumented = true
+
+      Rails.logger.info "[Observability] Instrumented RubyLLM.paint for session #{session.session_id}"
+    end
+
+    def uninstrument!
+      return unless @instrumented
+      return unless @original_paint_method
+
+      RubyLLM.define_singleton_method(:paint, @original_paint_method)
+      @instrumented = false
+
+      Rails.logger.info "[Observability] Uninstrumented RubyLLM.paint"
+    end
+
+    private
+
+    def wrap_paint_method
+      return if @original_paint_method
+
+      @original_paint_method = RubyLLM.method(:paint)
+      instrumenter = self
+
+      RubyLLM.define_singleton_method(:paint) do |*args, **kwargs|
+        instrumenter.send(:handle_paint_call, args, kwargs)
+      end
+    end
+
+    def handle_paint_call(args, kwargs)
+      prompt = args[0]
+      model_id = kwargs[:model] || default_image_model
+      size = kwargs[:size] || "1024x1024"
+      quality = kwargs[:quality] || "standard"
+
+      trace = session.create_trace(
+        name: "image_generation",
+        input: { prompt: prompt },
+        metadata: @context.merge(
+          model: model_id,
+          size: size,
+          quality: quality
+        ).compact
+      )
+
+      image_obs = trace.create_image_generation(
+        name: "paint",
+        model: model_id,
+        metadata: {
+          size: size,
+          quality: quality
+        }.compact
+      )
+
+      result = @original_paint_method.call(*args, **kwargs)
+
+      finalize_image_generation(image_obs, result, prompt, size: size, quality: quality)
+      trace.finalize(
+        output: format_output(result),
+        metadata: { size: extract_size(result) || size, quality: quality }
+      )
+
+      result
+    rescue StandardError => e
+      handle_error(e, trace, image_obs)
+      raise
+    end
+
+    def finalize_image_generation(image_obs, result, prompt, size:, quality:)
+      cost = calculate_cost(result, size: size, quality: quality)
+
+      image_obs.finalize(
+        output: format_output(result),
+        usage: {},
+        cost_usd: cost
+      )
+
+      image_obs.update!(
+        input: prompt,
+        metadata: image_obs.metadata.merge(
+          revised_prompt: result.revised_prompt,
+          output_format: result.base64? ? "base64" : "url",
+          mime_type: result.mime_type,
+          size: extract_size(result) || size,
+          quality: quality
+        ).compact
+      )
+    end
+
+    def calculate_cost(result, size:, quality:)
+      model_id = result.model_id
+      return 0.0 unless model_id
+
+      lookup_image_price(model_id, size, quality)
+    rescue StandardError => e
+      Rails.logger.warn "[Observability] Failed to calculate image generation cost: #{e.message}"
+      0.0
+    end
+
+    def lookup_image_price(model_id, size, quality)
+      model_pricing = IMAGE_PRICING[model_id]
+      return 0.0 unless model_pricing
+
+      # Try exact size match, then "default"
+      size_pricing = model_pricing[size] || model_pricing["default"]
+      return 0.0 unless size_pricing
+
+      # Try exact quality match first
+      return size_pricing[quality] if size_pricing[quality]
+
+      # Try mapped quality (e.g., "standard" -> "medium" for GPT-image models)
+      mapped_quality = QUALITY_MAPPINGS[quality]
+      return size_pricing[mapped_quality] if mapped_quality && size_pricing[mapped_quality]
+
+      # Fall back to "standard", "medium", "default", or first available
+      size_pricing["standard"] ||
+        size_pricing["medium"] ||
+        size_pricing["default"] ||
+        size_pricing.values.first ||
+        0.0
+    end
+
+    def extract_size(result)
+      # Try to get size from result if available
+      result.respond_to?(:size) ? result.size : nil
+    end
+
+    def format_output(result)
+      {
+        model: result.model_id,
+        has_url: result.respond_to?(:url) && result.url.present?,
+        base64: result.base64?,
+        mime_type: result.mime_type,
+        revised_prompt: result.revised_prompt
+      }.compact
+    end
+
+    def default_image_model
+      if RubyLLM.config.respond_to?(:default_image_model)
+        RubyLLM.config.default_image_model
+      else
+        "dall-e-3"
+      end
+    end
+
+    def handle_error(error, trace, image_obs)
+      return unless trace
+
+      error_span = trace.create_span(
+        name: "error",
+        metadata: {
+          error_type: error.class.name,
+          level: "ERROR"
+        },
+        input: {
+          error_message: error.message,
+          backtrace: error.backtrace&.first(10)
+        }.to_json
+      )
+      error_span.finalize(output: { error_captured: true }.to_json)
+
+      image_obs&.update(status_message: "FAILED") rescue nil
+
+      Rails.logger.error "[Observability] Image generation error captured: #{error.class.name} - #{error.message}"
+    end
+  end
+end

data/app/services/observ/moderation_guardrail_service.rb

@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+
+module Observ
+  class ModerationGuardrailService
+    include Observ::Concerns::ObservableService
+
+    # Score thresholds for different actions
+    THRESHOLDS = {
+      critical: 0.9,  # Auto-flag as critical
+      high: 0.7,      # Flag as high priority
+      review: 0.5     # Flag for normal review
+    }.freeze
+
+    # Categories that always trigger critical review
+    CRITICAL_CATEGORIES = %w[
+      sexual/minors
+      self-harm/intent
+      self-harm/instructions
+      violence/graphic
+    ].freeze
+
+    class Result
+      attr_reader :action, :reason, :priority, :details
+
+      def initialize(action:, reason: nil, priority: nil, details: {})
+        @action = action
+        @reason = reason
+        @priority = priority
+        @details = details
+      end
+
+      def flagged? = action == :flagged
+      def skipped? = action == :skipped
+      def passed? = action == :passed
+    end
+
+    def initialize(observability_session: nil)
+      initialize_observability(
+        observability_session,
+        service_name: "moderation_guardrail",
+        metadata: {}
+      )
+    end
+
+    # Evaluate a trace for moderation issues
+    #
+    # @param trace [Observ::Trace] The trace to evaluate
+    # @param moderate_input [Boolean] Whether to moderate input content
+    # @param moderate_output [Boolean] Whether to moderate output content
+    # @return [Result] The evaluation result
+    def evaluate_trace(trace, moderate_input: true, moderate_output: true)
+      return Result.new(action: :skipped, reason: "already_in_queue") if trace.in_review_queue?
+      return Result.new(action: :skipped, reason: "already_has_moderation") if has_existing_flags?(trace)
+
+      with_observability do |_session|
+        content = extract_trace_content(
+          trace,
+          moderate_input: moderate_input,
+          moderate_output: moderate_output
+        )
+        return Result.new(action: :skipped, reason: "no_content") if content.blank?
+
+        perform_moderation(trace, content)
+      end
+    rescue StandardError => e
+      Rails.logger.error "[ModerationGuardrailService] Failed to evaluate trace #{trace.id}: #{e.message}"
+      Result.new(action: :skipped, reason: "error", details: { error: e.message })
+    end
+
+    # Evaluate all traces in a session
+    #
+    # @param session [Observ::Session] The session to evaluate
+    # @return [Array<Result>] Results for each trace
+    def evaluate_session(session)
+      return [] if session.traces.empty?
+
+      session.traces.map do |trace|
+        evaluate_trace(trace)
+      end
+    end
+
+    # Evaluate session-level content (aggregated input/output)
+    #
+    # @param session [Observ::Session] The session to evaluate
+    # @return [Result] The evaluation result
+    def evaluate_session_content(session)
+      return Result.new(action: :skipped, reason: "already_in_queue") if session.in_review_queue?
+
+      with_observability do |_session|
+        content = extract_session_content(session)
+        return Result.new(action: :skipped, reason: "no_content") if content.blank?
+
+        perform_session_moderation(session, content)
+      end
+    rescue StandardError => e
+      Rails.logger.error "[ModerationGuardrailService] Failed to evaluate session #{session.id}: #{e.message}"
+      Result.new(action: :skipped, reason: "error", details: { error: e.message })
+    end
+
+    private
+
+    def has_existing_flags?(trace)
+      trace.moderations.any?(&:flagged?)
+    end
+
+    def extract_trace_content(trace, moderate_input:, moderate_output:)
+      parts = []
+      parts << extract_text(trace.input) if moderate_input
+      parts << extract_text(trace.output) if moderate_output
+      parts.compact.reject(&:blank?).join("\n\n---\n\n")
+    end
+
+    def extract_session_content(session)
+      session.traces.flat_map do |trace|
+        [extract_text(trace.input), extract_text(trace.output)]
+      end.compact.reject(&:blank?).join("\n\n---\n\n").truncate(10_000)
+    end
+
+    def extract_text(content)
+      return nil if content.blank?
+
+      case content
+      when String
+        content
+      when Hash
+        # Try common keys for text content
+        content["text"] || content["content"] || content["message"] ||
+          content[:text] || content[:content] || content[:message] ||
+          content.to_json
+      else
+        content.to_s
+      end
+    end
+
+    def perform_moderation(trace, content)
+      instrument_moderation(context: {
+        service: "moderation_guardrail",
+        trace_id: trace.id,
+        content_length: content.length
+      })
+
+      result = RubyLLM.moderate(content)
+
+      evaluate_and_enqueue(trace, result)
+    end
+
+    def perform_session_moderation(session, content)
+      instrument_moderation(context: {
+        service: "moderation_guardrail",
+        session_id: session.id,
+        content_length: content.length
+      })
+
+      result = RubyLLM.moderate(content)
+
+      evaluate_and_enqueue_session(session, result)
+    end
+
+    def evaluate_and_enqueue(trace, moderation_result)
+      priority = determine_priority(moderation_result)
+
+      if priority
+        details = build_details(moderation_result)
+        trace.enqueue_for_review!(
+          reason: "content_moderation",
+          priority: priority,
+          details: details
+        )
+
+        Result.new(
+          action: :flagged,
+          priority: priority,
+          details: details
+        )
+      else
+        Result.new(action: :passed)
+      end
+    end
+
+    def evaluate_and_enqueue_session(session, moderation_result)
+      priority = determine_priority(moderation_result)
+
+      if priority
+        details = build_details(moderation_result)
+        session.enqueue_for_review!(
+          reason: "content_moderation",
+          priority: priority,
+          details: details
+        )
+
+        Result.new(
+          action: :flagged,
+          priority: priority,
+          details: details
+        )
+      else
+        Result.new(action: :passed)
+      end
+    end
+
+    def determine_priority(result)
+      # Check for critical categories first
+      if (result.flagged_categories & CRITICAL_CATEGORIES).any?
+        return :critical
+      end
+
+      # Check if explicitly flagged
+      if result.flagged?
+        max_score = result.category_scores.values.max || 0
+        return max_score >= THRESHOLDS[:critical] ? :critical : :high
+      end
+
+      # Check score thresholds even if not flagged
+      max_score = result.category_scores.values.max || 0
+
+      if max_score >= THRESHOLDS[:high]
+        :high
+      elsif max_score >= THRESHOLDS[:review]
+        :normal
+      end
+    end
+
+    def build_details(result)
+      {
+        flagged: result.flagged?,
+        flagged_categories: result.flagged_categories,
+        highest_category: highest_category(result),
+        highest_score: result.category_scores.values.max&.round(4),
+        category_scores: result.category_scores.transform_values { |v| v.round(4) }
+      }
+    end
+
+    def highest_category(result)
+      return nil if result.category_scores.empty?
+
+      result.category_scores.max_by { |_, score| score }&.first
+    end
+  end
+end

data/app/services/observ/moderation_instrumenter.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module Observ
+  class ModerationInstrumenter
+    attr_reader :session, :context
+
+    def initialize(session, context: {})
+      @session = session
+      @context = context
+      @original_moderate_method = nil
+      @instrumented = false
+    end
+
+    def instrument!
+      return if @instrumented
+
+      wrap_moderate_method
+      @instrumented = true
+
+      Rails.logger.info "[Observability] Instrumented RubyLLM.moderate for session #{session.session_id}"
+    end
+
+    def uninstrument!
+      return unless @instrumented
+      return unless @original_moderate_method
+
+      RubyLLM.define_singleton_method(:moderate, @original_moderate_method)
+      @instrumented = false
+
+      Rails.logger.info "[Observability] Uninstrumented RubyLLM.moderate"
+    end
+
+    private
+
+    def wrap_moderate_method
+      return if @original_moderate_method
+
+      @original_moderate_method = RubyLLM.method(:moderate)
+      instrumenter = self
+
+      RubyLLM.define_singleton_method(:moderate) do |*args, **kwargs|
+        instrumenter.send(:handle_moderate_call, args, kwargs)
+      end
+    end
+
+    def handle_moderate_call(args, kwargs)
+      text = args[0]
+      model_id = kwargs[:model] || default_moderation_model
+
+      trace = session.create_trace(
+        name: "moderation",
+        input: { text: text&.truncate(500) },
+        metadata: @context.merge(
+          model: model_id
+        ).compact
+      )
+
+      moderation_obs = trace.create_moderation(
+        name: "moderate",
+        model: model_id,
+        metadata: {}
+      )
+
+      result = @original_moderate_method.call(*args, **kwargs)
+
+      finalize_moderation(moderation_obs, result, text)
+      trace.finalize(
+        output: format_output(result),
+        metadata: extract_trace_metadata(result)
+      )
+
+      result
+    rescue StandardError => e
+      handle_error(e, trace, moderation_obs)
+      raise
+    end
+
+    def finalize_moderation(moderation_obs, result, text)
+      moderation_obs.finalize(
+        output: format_output(result),
+        usage: {},
+        cost_usd: 0.0 # Moderation is typically free
+      )
+
+      moderation_obs.update!(
+        input: text&.truncate(1000),
+        metadata: moderation_obs.metadata.merge(
+          flagged: result.flagged?,
+          categories: result.categories,
+          category_scores: result.category_scores,
+          flagged_categories: result.flagged_categories
+        ).compact
+      )
+    end
+
+    def format_output(result)
+      {
+        model: result.model,
+        flagged: result.flagged?,
+        flagged_categories: result.flagged_categories,
+        id: result.respond_to?(:id) ? result.id : nil
+      }.compact
+    end
+
+    def extract_trace_metadata(result)
+      {
+        flagged: result.flagged?,
+        flagged_categories_count: result.flagged_categories&.count || 0
+      }.compact
+    end
+
+    def default_moderation_model
+      if RubyLLM.config.respond_to?(:default_moderation_model)
+        RubyLLM.config.default_moderation_model
+      else
+        "omni-moderation-latest"
+      end
+    end
+
+    def handle_error(error, trace, moderation_obs)
+      return unless trace
+
+      error_span = trace.create_span(
+        name: "error",
+        metadata: {
+          error_type: error.class.name,
+          level: "ERROR"
+        },
+        input: {
+          error_message: error.message,
+          backtrace: error.backtrace&.first(10)
+        }.to_json
+      )
+      error_span.finalize(output: { error_captured: true }.to_json)
+
+      moderation_obs&.update(status_message: "FAILED") rescue nil
+
+      Rails.logger.error "[Observability] Moderation error captured: #{error.class.name} - #{error.message}"
+    end
+  end
+end

data/app/services/observ/prompt_manager/caching.rb

@@ -82,13 +82,14 @@ module Observ
       # @return [Boolean] true if successful
       def invalidate_cache(name:, version: nil)
         keys = if version
-          [ cache_key(name: name, version: version) ]
+          [cache_key(name: name, version: version)]
         else
           # Invalidate all state-based keys for this prompt
-          [ :draft, :production, :archived ].map { |state| cache_key(name: name, state: state) }
+          [:draft, :production, :archived].map { |state| cache_key(name: name, state: state) }
         end
 
         keys.each { |key| Rails.cache.delete(key) }
+        bump_cache_stamp(name: name)
         Rails.logger.info("Cache invalidated for #{name}#{version ? " v#{version}" : ""}")
 
         true
@@ -121,6 +122,18 @@ module Observ
         results
       end
 
+      def cache_stamp_key(name:)
+        "#{Observ.config.prompt_cache_namespace}:#{name}:stamp"
+      end
+
+      def cache_stamp(name:)
+        Rails.cache.read(cache_stamp_key(name: name))
+      end
+
+      def bump_cache_stamp(name:)
+        Rails.cache.write(cache_stamp_key(name: name), Time.current.to_f)
+      end
+
       # Get list of critical prompts (prompts used by agents)
       # @return [Array<String>] Array of prompt names
       def critical_prompt_names