rubyllm-observ 0.6.6 → 0.6.8

data/app/jobs/observ/moderation_guardrail_job.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Observ
+  class ModerationGuardrailJob < ApplicationJob
+    queue_as :moderation
+
+    # Retry configuration
+    retry_on StandardError, wait: :polynomially_longer, attempts: 3
+    discard_on ActiveRecord::RecordNotFound
+
+    # Process a single trace or session
+    #
+    # @param trace_id [Integer] ID of the trace to moderate
+    # @param session_id [Integer] ID of the session to moderate
+    # @param options [Hash] Options for moderation
+    # @option options [Boolean] :moderate_input Whether to moderate input (default: true)
+    # @option options [Boolean] :moderate_output Whether to moderate output (default: true)
+    # @option options [Boolean] :aggregate Whether to moderate aggregated session content
+    def perform(trace_id: nil, session_id: nil, **options)
+      if trace_id
+        moderate_trace(trace_id, options)
+      elsif session_id
+        moderate_session(session_id, options)
+      else
+        Rails.logger.warn "[ModerationGuardrailJob] No trace_id or session_id provided"
+      end
+    end
+
+    # Class method to enqueue moderation for traces matching criteria
+    #
+    # @param scope [ActiveRecord::Relation] Scope of traces to moderate
+    # @param sample_percentage [Integer] Percentage of traces to sample (1-100)
+    def self.enqueue_for_scope(scope, sample_percentage: 100)
+      traces = scope.left_joins(:review_item)
+                    .where(observ_review_items: { id: nil })
+
+      if sample_percentage < 100
+        sample_size = (traces.count * sample_percentage / 100.0).ceil
+        traces = traces.order("RANDOM()").limit(sample_size)
+      end
+
+      traces.find_each do |trace|
+        perform_later(trace_id: trace.id)
+      end
+    end
+
+    # Enqueue moderation for user-facing sessions only
+    #
+    # @param since [Time] Only process sessions created after this time
+    def self.enqueue_user_facing(since: 1.hour.ago)
+      Observ::Session
+        .where(created_at: since..)
+        .where("metadata->>'user_facing' = ?", "true")
+        .find_each do |session|
+          perform_later(session_id: session.id)
+        end
+    end
+
+    # Enqueue moderation for specific agent types
+    #
+    # @param agent_types [Array<String>] Agent types to moderate
+    # @param since [Time] Only process sessions created after this time
+    def self.enqueue_for_agent_types(agent_types, since: 1.hour.ago)
+      Observ::Session
+        .where(created_at: since..)
+        .where("metadata->>'agent_type' IN (?)", agent_types)
+        .find_each do |session|
+          perform_later(session_id: session.id)
+        end
+    end
+
+    private
+
+    def moderate_trace(trace_id, options)
+      trace = Observ::Trace.find(trace_id)
+
+      service = ModerationGuardrailService.new
+      result = service.evaluate_trace(
+        trace,
+        moderate_input: options.fetch(:moderate_input, true),
+        moderate_output: options.fetch(:moderate_output, true)
+      )
+
+      log_result("Trace #{trace_id}", result)
+    end
+
+    def moderate_session(session_id, options)
+      session = Observ::Session.find(session_id)
+
+      service = ModerationGuardrailService.new
+
+      if options[:aggregate]
+        # Moderate aggregated session content
+        result = service.evaluate_session_content(session)
+        log_result("Session #{session_id} (aggregated)", result)
+      else
+        # Moderate each trace individually
+        results = service.evaluate_session(session)
+        flagged_count = results.count(&:flagged?)
+        Rails.logger.info "[ModerationGuardrailJob] Session #{session_id}: #{flagged_count}/#{results.size} traces flagged"
+      end
+    end
+
+    def log_result(identifier, result)
+      case result.action
+      when :flagged
+        Rails.logger.info "[ModerationGuardrailJob] #{identifier} flagged (#{result.priority}): #{result.details[:flagged_categories]}"
+      when :skipped
+        Rails.logger.debug "[ModerationGuardrailJob] #{identifier} skipped: #{result.reason}"
+      when :passed
+        Rails.logger.debug "[ModerationGuardrailJob] #{identifier} passed moderation"
+      end
+    end
+  end
+end

data/app/models/concerns/observ/prompt_management.rb

@@ -135,6 +135,15 @@ module Observ
 
       # Override system_prompt to use prompt management
       def system_prompt
+        config = prompt_config.presence || {}
+        prompt_name = config[:prompt_name] || default_prompt_name
+        current_stamp = Observ::PromptManager.cache_stamp(name: prompt_name)
+
+        if defined?(@_prompt_cache_stamp) && @_prompt_cache_stamp != current_stamp
+          reset_prompt_cache!
+        end
+
+        @_prompt_cache_stamp = current_stamp
         @_system_prompt ||= fetch_prompt(variables: prompt_variables)
       end
 
@@ -159,6 +168,7 @@ module Observ
       def reset_prompt_cache!
         @_system_prompt = nil
         @_prompt_template = nil
+        @_prompt_cache_stamp = nil
       end
 
       # Override model to check prompt metadata first

data/app/models/observ/embedding.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Observ
+  class Embedding < Observation
+    # Set input texts for the embedding call
+    def set_input(texts)
+      update!(
+        input: texts.is_a?(Array) ? texts.to_json : texts
+      )
+    end
+
+    def finalize(output:, usage: {}, cost_usd: 0.0, status_message: nil)
+      merged_usage = (self.usage || {}).merge(usage.stringify_keys)
+
+      update!(
+        output: output.is_a?(String) ? output : output.to_json,
+        usage: merged_usage,
+        cost_usd: cost_usd,
+        end_time: Time.current,
+        status_message: status_message
+      )
+    end
+
+    # Embedding-specific helpers
+    def input_tokens
+      usage&.dig("input_tokens") || 0
+    end
+
+    def total_tokens
+      input_tokens # Embeddings only have input tokens
+    end
+
+    def batch_size
+      metadata&.dig("batch_size") || 1
+    end
+
+    def dimensions
+      metadata&.dig("dimensions")
+    end
+
+    def vectors_count
+      metadata&.dig("vectors_count") || 1
+    end
+  end
+end

data/app/models/observ/image_generation.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Observ
+  class ImageGeneration < Observation
+    def finalize(output:, usage: {}, cost_usd: 0.0, status_message: nil)
+      merged_usage = (self.usage || {}).merge(usage.stringify_keys)
+
+      update!(
+        output: output.is_a?(String) ? output : output.to_json,
+        usage: merged_usage,
+        cost_usd: cost_usd,
+        end_time: Time.current,
+        status_message: status_message
+      )
+    end
+
+    # Image-specific helpers
+    def size
+      metadata&.dig("size")
+    end
+
+    def quality
+      metadata&.dig("quality")
+    end
+
+    def revised_prompt
+      metadata&.dig("revised_prompt")
+    end
+
+    def output_format
+      metadata&.dig("output_format") # "url" or "base64"
+    end
+
+    def mime_type
+      metadata&.dig("mime_type")
+    end
+  end
+end

data/app/models/observ/moderation.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Observ
+  class Moderation < Observation
+    def finalize(output:, usage: {}, cost_usd: 0.0, status_message: nil)
+      merged_usage = (self.usage || {}).merge(usage.stringify_keys)
+
+      update!(
+        output: output.is_a?(String) ? output : output.to_json,
+        usage: merged_usage,
+        cost_usd: cost_usd,
+        end_time: Time.current,
+        status_message: status_message
+      )
+    end
+
+    # Moderation-specific helpers
+    def flagged?
+      metadata&.dig("flagged") || false
+    end
+
+    def categories
+      metadata&.dig("categories") || {}
+    end
+
+    def category_scores
+      metadata&.dig("category_scores") || {}
+    end
+
+    def flagged_categories
+      metadata&.dig("flagged_categories") || []
+    end
+
+    def highest_score_category
+      return nil if category_scores.empty?
+
+      category_scores.max_by { |_, score| score }&.first
+    end
+  end
+end

data/app/models/observ/null_prompt.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "mustache"
+
 module Observ
   # Null Object pattern for Prompt
   # Used when a prompt is not found, providing a fallback with the same interface
@@ -17,9 +19,36 @@ module Observ
       nil
     end
 
-    # Returns the fallback text as-is (no variable compilation)
+    # Compile prompt with Mustache templating (same as Prompt)
     def compile(variables = {})
-      @prompt
+      return @prompt if variables.empty?
+
+      Mustache.render(@prompt, variables)
+    end
+
+    # Compile with validation (raises if missing top-level variables)
+    # Note: Variables inside sections (loops) are validated at render time by Mustache
+    def compile_with_validation(variables = {})
+      required_vars = required_variables
+      provided_keys = variables.keys.map(&:to_s)
+
+      missing_vars = required_vars.reject do |var|
+        # Handle dot notation (e.g., "user.name" - check if "user" key exists)
+        root_key = var.split(".").first
+        provided_keys.include?(var) || provided_keys.include?(root_key)
+      end
+
+      if missing_vars.any?
+        raise Observ::VariableSubstitutionError, "Missing variables: #{missing_vars.join(', ')}"
+      end
+
+      compile(variables)
+    end
+
+    # Extract top-level variables from template (for validation purposes)
+    def required_variables
+      template_without_sections = strip_sections(@prompt)
+      template_without_sections.scan(/\{\{([^#\^\/!>\{\s][^}\s]*)\}\}/).flatten.uniq
     end
 
     # Null prompts are always in a "fallback" state
@@ -55,5 +84,23 @@ module Observ
     def inspect
       "#<Observ::NullPrompt name: #{name.inspect}, fallback: #{prompt[0..50].inspect}...>"
     end
+
+    private
+
+    # Strip section content from template for top-level variable extraction
+    # Removes content between {{#section}}...{{/section}} and {{^section}}...{{/section}}
+    def strip_sections(template)
+      result = template.dup
+
+      # Match sections: {{#name}}...{{/name}} or {{^name}}...{{/name}}
+      # Use non-greedy matching and handle nesting by repeating until stable
+      loop do
+        previous = result
+        result = result.gsub(/\{\{[#\^](\w+)\}\}.*?\{\{\/\1\}\}/m, "")
+        break if result == previous
+      end
+
+      result
+    end
   end
 end

data/app/models/observ/observation.rb

@@ -9,7 +9,9 @@ module Observ
 
     validates :observation_id, presence: true, uniqueness: true
     validates :start_time, presence: true
-    validates :type, presence: true, inclusion: { in: %w[Observ::Generation Observ::Span] }
+    validates :type, presence: true, inclusion: {
+    in: %w[Observ::Generation Observ::Span Observ::Embedding Observ::ImageGeneration Observ::Transcription Observ::Moderation]
+  }
 
     after_save :update_trace_metrics, if: :saved_change_to_cost_or_usage?
 

data/app/models/observ/prompt.rb

@@ -101,7 +101,7 @@ module Observ
 
     def self.clear_cache(name:)
       # Clear all cache keys for this prompt
-      [ :draft, :production, :archived ].each do |state|
+      [:draft, :production, :archived].each do |state|
         Rails.cache.delete(cache_key_for(name: name, version: nil, state: state))
       end
     end
@@ -206,7 +206,7 @@ module Observ
 
     # Export
     def to_json_export
-      as_json(except: [ :id, :created_at, :updated_at ])
+      as_json(except: [:id, :created_at, :updated_at])
     end
 
     def to_yaml_export

data/app/models/observ/review_item.rb

@@ -12,7 +12,7 @@ module Observ
     validates :reviewable, presence: true
     validates :reviewable_id, uniqueness: { scope: :reviewable_type }
 
-    scope :actionable, -> { where(status: [ :pending, :in_progress ]) }
+    scope :actionable, -> { where(status: [:pending, :in_progress]) }
     scope :by_priority, -> { order(priority: :desc, created_at: :asc) }
     scope :sessions, -> { where(reviewable_type: "Observ::Session") }
     scope :traces, -> { where(reviewable_type: "Observ::Trace") }

data/app/models/observ/score.rb

@@ -13,7 +13,7 @@ module Observ
     validates :name, presence: true
     validates :value, presence: true, numericality: true
     validates :scoreable_id, uniqueness: {
-      scope: [ :scoreable_type, :name, :source ],
+      scope: [:scoreable_type, :name, :source],
       message: "already has a score with this name and source"
     }
 

data/app/models/observ/session.rb

@@ -17,6 +17,7 @@ module Observ
 
     before_validation :set_session_id, on: :create
     before_validation :set_start_time, on: :create
+    after_commit :evaluate_guardrails, if: :just_completed?
 
     def create_trace(name: nil, input: nil, metadata: {}, tags: [])
       traces.create!(
@@ -103,6 +104,30 @@ module Observ
       instrumenter
     end
 
+    def instrument_embedding(context: {})
+      instrumenter = Observ::EmbeddingInstrumenter.new(self, context: context)
+      instrumenter.instrument!
+      instrumenter
+    end
+
+    def instrument_image_generation(context: {})
+      instrumenter = Observ::ImageGenerationInstrumenter.new(self, context: context)
+      instrumenter.instrument!
+      instrumenter
+    end
+
+    def instrument_transcription(context: {})
+      instrumenter = Observ::TranscriptionInstrumenter.new(self, context: context)
+      instrumenter.instrument!
+      instrumenter
+    end
+
+    def instrument_moderation(context: {})
+      instrumenter = Observ::ModerationInstrumenter.new(self, context: context)
+      instrumenter.instrument!
+      instrumenter
+    end
+
     def chat
       @chat ||= ::Chat.find_by(observability_session_id: session_id) if defined?(::Chat)
     end
@@ -131,5 +156,13 @@ module Observ
         ((g.end_time - g.start_time) * 1000).round(2)
       end || 0
     end
+
+    def just_completed?
+      previous_changes.key?("end_time") && end_time.present?
+    end
+
+    def evaluate_guardrails
+      GuardrailService.evaluate_session(self)
+    end
   end
 end

data/app/models/observ/trace.rb

@@ -17,6 +17,7 @@ module Observ
     validates :start_time, presence: true
 
     after_save :update_session_metrics, if: :saved_change_to_total_cost_or_total_tokens?
+    after_commit :evaluate_guardrails, if: :just_completed?
 
     def create_generation(name: "llm_generation", model: nil, metadata: {}, **options)
       observations.create!(
@@ -42,6 +43,54 @@ module Observ
       )
     end
 
+    def create_embedding(name: "embedding", model: nil, metadata: {}, **options)
+      observations.create!(
+        observation_id: SecureRandom.uuid,
+        type: "Observ::Embedding",
+        name: name,
+        model: model,
+        metadata: metadata,
+        start_time: Time.current,
+        **options.slice(:model_parameters, :parent_observation_id)
+      )
+    end
+
+    def create_image_generation(name: "image_generation", model: nil, metadata: {}, **options)
+      observations.create!(
+        observation_id: SecureRandom.uuid,
+        type: "Observ::ImageGeneration",
+        name: name,
+        model: model,
+        metadata: metadata,
+        start_time: Time.current,
+        **options.slice(:model_parameters, :parent_observation_id)
+      )
+    end
+
+    def create_transcription(name: "transcription", model: nil, metadata: {}, **options)
+      observations.create!(
+        observation_id: SecureRandom.uuid,
+        type: "Observ::Transcription",
+        name: name,
+        model: model,
+        metadata: metadata,
+        start_time: Time.current,
+        **options.slice(:model_parameters, :parent_observation_id)
+      )
+    end
+
+    def create_moderation(name: "moderation", model: nil, metadata: {}, **options)
+      observations.create!(
+        observation_id: SecureRandom.uuid,
+        type: "Observ::Moderation",
+        name: name,
+        model: model,
+        metadata: metadata,
+        start_time: Time.current,
+        **options.slice(:model_parameters, :parent_observation_id)
+      )
+    end
+
     def finalize(output: nil, metadata: {})
       merged_metadata = (self.metadata || {}).merge(metadata)
       update!(
@@ -74,13 +123,26 @@ module Observ
     end
 
     def update_aggregated_metrics
-      new_total_cost = generations.sum(:cost_usd) || 0.0
-
-      # Database-agnostic token calculation
-      new_total_tokens = generations.sum do |gen|
+      # Include generations, embeddings, image generations, transcriptions, and moderations in cost calculation
+      new_total_cost = (generations.sum(:cost_usd) || 0.0) +
+                       (embeddings.sum(:cost_usd) || 0.0) +
+                       (image_generations.sum(:cost_usd) || 0.0) +
+                       (transcriptions.sum(:cost_usd) || 0.0) +
+                       (moderations.sum(:cost_usd) || 0.0)
+
+      # Database-agnostic token calculation for generations
+      generation_tokens = generations.sum do |gen|
         gen.usage&.dig("total_tokens") || 0
       end
 
+      # Embeddings only have input tokens
+      embedding_tokens = embeddings.sum do |emb|
+        emb.usage&.dig("input_tokens") || 0
+      end
+
+      # Image generations, transcriptions, and moderations don't use tokens
+      new_total_tokens = generation_tokens + embedding_tokens
+
       update_columns(
         total_cost: new_total_cost,
         total_tokens: new_total_tokens
@@ -95,6 +157,22 @@ module Observ
       observations.where(type: "Observ::Span")
     end
 
+    def embeddings
+      observations.where(type: "Observ::Embedding")
+    end
+
+    def image_generations
+      observations.where(type: "Observ::ImageGeneration")
+    end
+
+    def transcriptions
+      observations.where(type: "Observ::Transcription")
+    end
+
+    def moderations
+      observations.where(type: "Observ::Moderation")
+    end
+
     def models_used
       generations.where.not(model: nil).distinct.pluck(:model)
     end
@@ -134,5 +212,13 @@ module Observ
     def update_session_metrics
       observ_session&.update_aggregated_metrics
     end
+
+    def just_completed?
+      previous_changes.key?("end_time") && end_time.present?
+    end
+
+    def evaluate_guardrails
+      GuardrailService.evaluate_trace(self)
+    end
   end
 end

data/app/models/observ/transcription.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Observ
+  class Transcription < Observation
+    def finalize(output:, usage: {}, cost_usd: 0.0, status_message: nil)
+      merged_usage = (self.usage || {}).merge(usage.stringify_keys)
+
+      update!(
+        output: output.is_a?(String) ? output : output.to_json,
+        usage: merged_usage,
+        cost_usd: cost_usd,
+        end_time: Time.current,
+        status_message: status_message
+      )
+    end
+
+    # Transcription-specific helpers
+    def audio_duration_s
+      metadata&.dig("audio_duration_s")
+    end
+
+    def language
+      metadata&.dig("language")
+    end
+
+    def segments_count
+      metadata&.dig("segments_count") || 0
+    end
+
+    def speakers_count
+      metadata&.dig("speakers_count")
+    end
+
+    def has_diarization?
+      metadata&.dig("has_diarization") || false
+    end
+  end
+end

data/app/presenters/observ/agent_select_presenter.rb

@@ -31,7 +31,7 @@ module Observ
     # Format: [[display_name, identifier], ...]
     # @return [Array<Array<String>>] options array for select dropdown
     def options
-      [ default_option ] + agent_options
+      [default_option] + agent_options
     end
 
     # Convenience class method that injects agents from Observ::AgentProvider
@@ -47,13 +47,13 @@ module Observ
     # Default option for "no agent selected" state
     # @return [Array<String>] the default option
     def default_option
-      [ "Default Agent", "" ]
+      ["Default Agent", ""]
     end
 
     # Maps agents to [display_name, identifier] pairs
     # @return [Array<Array<String>>] agent options
     def agent_options
-      agents.map { |agent| [ agent.display_name, agent.agent_identifier ] }
+      agents.map { |agent| [agent.display_name, agent.agent_identifier] }
     end
   end
 end