rubyllm-observ 0.5.0

data/app/models/observ/session.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Observ
+  class Session < ApplicationRecord
+    self.table_name = "observ_sessions"
+
+  has_many :traces, class_name: "Observ::Trace",
+           foreign_key: :observ_session_id, dependent: :destroy, inverse_of: :observ_session
+  has_many :annotations, as: :annotatable, dependent: :destroy
+
+    validates :session_id, presence: true, uniqueness: true
+    validates :start_time, presence: true
+
+    before_validation :set_session_id, on: :create
+    before_validation :set_start_time, on: :create
+
+    def create_trace(name: nil, input: nil, metadata: {}, tags: [])
+      traces.create!(
+        trace_id: SecureRandom.uuid,
+        name: name || "chat_exchange",
+        input: input.is_a?(String) ? input : input.to_json,
+        metadata: metadata,
+        tags: tags,
+        user_id: user_id,
+        start_time: Time.current
+      )
+    end
+
+    def finalize
+      update!(end_time: Time.current)
+      update_aggregated_metrics
+    end
+
+    def duration_s
+      return nil unless end_time
+      (end_time - start_time).round(1)
+    end
+
+    def average_llm_latency_ms
+      generations = Observ::Generation.joins(trace: :observ_session)
+                                      .where(observ_sessions: { id: id })
+                                      .where.not(end_time: nil)
+      return 0 if generations.empty?
+
+      total_duration = generations.sum do |g|
+        ((g.end_time - g.start_time) * 1000).round(2)
+      end
+      (total_duration / generations.count).round(0)
+    end
+
+    def session_metrics
+      if end_time.nil?
+        {
+          session_id: session_id,
+          total_traces: traces.count,
+          total_llm_calls: generation_count,
+          total_tokens: traces.sum(:total_tokens),
+          total_cost: traces.sum(:total_cost).to_f,
+          total_llm_duration_ms: calculate_total_llm_duration,
+          average_llm_latency_ms: average_llm_latency_ms,
+          duration_s: duration_s
+        }
+      else
+        {
+          session_id: session_id,
+          total_traces: total_traces_count,
+          total_llm_calls: total_llm_calls_count,
+          total_tokens: total_tokens,
+          total_cost: total_cost.to_f,
+          total_llm_duration_ms: total_llm_duration_ms,
+          average_llm_latency_ms: average_llm_latency_ms,
+          duration_s: duration_s
+        }
+      end
+    end
+
+    def update_aggregated_metrics
+      update_columns(
+        total_traces_count: traces.count,
+        total_llm_calls_count: generation_count,
+        total_tokens: traces.sum(:total_tokens),
+        total_cost: traces.sum(:total_cost),
+        total_llm_duration_ms: calculate_total_llm_duration
+      )
+    end
+
+    def update_metadata(new_metadata)
+      self.metadata = (self.metadata || {}).merge(new_metadata)
+      save
+    end
+
+    def instrument_chat(chat_instance, context: {})
+      instrumenter = Observ::ChatInstrumenter.new(
+        self,
+        chat_instance,
+        context: context
+      )
+      instrumenter.instrument!
+      instrumenter
+    end
+
+    def chat
+      @chat ||= ::Chat.find_by(observability_session_id: session_id) if defined?(::Chat)
+    end
+
+    private
+
+    def set_session_id
+      self.session_id ||= SecureRandom.uuid
+    end
+
+    def set_start_time
+      self.start_time ||= Time.current
+    end
+
+    def generation_count
+      Observ::Generation.joins(:trace)
+                        .where(observ_traces: { observ_session_id: id })
+                        .count
+    end
+
+    def calculate_total_llm_duration
+      Observ::Generation.joins(:trace)
+                        .where(observ_traces: { observ_session_id: id })
+                        .where.not(end_time: nil)
+                        .sum do |g|
+        ((g.end_time - g.start_time) * 1000).round(2)
+      end || 0
+    end
+  end
+end

data/app/models/observ/span.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Observ
+  class Span < Observation
+    def finalize(output: nil, status_message: nil)
+      update!(
+        output: output.is_a?(String) ? output : output.to_json,
+        end_time: Time.current,
+        status_message: status_message
+      )
+    end
+  end
+end

data/app/models/observ/trace.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module Observ
+  class Trace < ApplicationRecord
+    self.table_name = "observ_traces"
+
+    belongs_to :observ_session, class_name: "Observ::Session", inverse_of: :traces
+    has_many :observations, class_name: "Observ::Observation",
+      foreign_key: :observ_trace_id, dependent: :destroy, inverse_of: :trace
+    belongs_to :message, optional: true
+    has_many :annotations, as: :annotatable, dependent: :destroy
+
+    validates :trace_id, presence: true, uniqueness: true
+    validates :start_time, presence: true
+
+    after_save :update_session_metrics, if: :saved_change_to_total_cost_or_total_tokens?
+
+    def create_generation(name: "llm_generation", model: nil, metadata: {}, **options)
+      observations.create!(
+        observation_id: SecureRandom.uuid,
+        type: "Observ::Generation",
+        name: name,
+        model: model,
+        metadata: metadata,
+        start_time: Time.current,
+        **options.slice(:model_parameters, :prompt_name, :prompt_version, :parent_observation_id)
+      )
+    end
+
+    def create_span(name:, input: nil, metadata: {}, parent_observation_id: nil)
+      observations.create!(
+        observation_id: SecureRandom.uuid,
+        type: "Observ::Span",
+        name: name,
+        input: input.is_a?(String) ? input : input.to_json,
+        metadata: metadata,
+        parent_observation_id: parent_observation_id,
+        start_time: Time.current
+      )
+    end
+
+    def finalize(output: nil, metadata: {})
+      merged_metadata = (self.metadata || {}).merge(metadata)
+      update!(
+        output: output.is_a?(String) ? output : output.to_json,
+        metadata: merged_metadata,
+        end_time: Time.current
+      )
+      update_aggregated_metrics
+    end
+
+    def finalize_with_response(response)
+      if response.is_a?(String)
+        finalize(output: response)
+        return response
+      end
+
+      response_metadata = extract_response_metadata(response)
+
+      finalize(
+        output: response.respond_to?(:content) ? response.content : response.to_s,
+        metadata: response_metadata
+      )
+
+      response.respond_to?(:content) ? response.content : response.to_s
+    end
+
+    def duration_ms
+      return nil unless end_time
+      ((end_time - start_time) * 1000).round(2)
+    end
+
+    def update_aggregated_metrics
+      new_total_cost = generations.sum(:cost_usd) || 0.0
+
+      # Database-agnostic token calculation
+      new_total_tokens = generations.sum do |gen|
+        gen.usage&.dig("total_tokens") || 0
+      end
+
+      update_columns(
+        total_cost: new_total_cost,
+        total_tokens: new_total_tokens
+      )
+    end
+
+    def generations
+      observations.where(type: "Observ::Generation")
+    end
+
+    def spans
+      observations.where(type: "Observ::Span")
+    end
+
+    def models_used
+      generations.where.not(model: nil).distinct.pluck(:model)
+    end
+
+    private
+
+    def extract_response_metadata(response)
+      metadata = {}
+
+      metadata[:model_id] = response.model_id if response.respond_to?(:model_id)
+      metadata[:input_tokens] = response.input_tokens if response.respond_to?(:input_tokens)
+
+      if response.respond_to?(:output_tokens)
+        metadata[:output_tokens] = response.output_tokens
+        metadata[:total_tokens] = (response.input_tokens || 0) + response.output_tokens
+      end
+
+      metadata[:role] = response.role if response.respond_to?(:role)
+
+      if response.respond_to?(:tool_calls) && response.tool_calls&.any?
+        metadata[:tool_calls_count] = response.tool_calls.count
+        metadata[:tool_calls] = response.tool_calls.map do |tc|
+          {
+            name: tc.respond_to?(:name) ? tc.name : nil,
+            arguments: tc.respond_to?(:arguments) ? tc.arguments : nil
+          }.compact
+        end
+      end
+
+      metadata
+    end
+
+    def saved_change_to_total_cost_or_total_tokens?
+      saved_change_to_total_cost? || saved_change_to_total_tokens?
+    end
+
+    def update_session_metrics
+      observ_session&.update_aggregated_metrics
+    end
+  end
+end

data/app/presenters/observ/agent_select_presenter.rb

@@ -0,0 +1,59 @@
+# Presenter for agent selection in the Observ domain
+#
+# This presenter receives agents via dependency injection and formats them
+# for display in select dropdowns. It has NO knowledge of:
+#   - How agents are discovered
+#   - Where agent files are located
+#   - The BaseAgent class hierarchy
+#
+# It only knows about the AgentSelectable interface that agents must implement.
+#
+# Usage with dependency injection (recommended):
+#   agents = Observ::AgentProvider.all_agents
+#   presenter = Observ::AgentSelectPresenter.new(agents: agents)
+#   presenter.options
+#   # => [["Default Agent", ""], ["Deep Research", "DeepResearchAgent"], ...]
+#
+# Usage with convenience class method:
+#   Observ::AgentSelectPresenter.options
+#   # => [["Default Agent", ""], ["Deep Research", "DeepResearchAgent"], ...]
+module Observ
+  class AgentSelectPresenter
+    attr_reader :agents
+
+    # Initialize with dependency injection
+    # @param agents [Array<Class>] array of agent classes implementing AgentSelectable
+    def initialize(agents:)
+      @agents = agents
+    end
+
+    # Returns formatted options for Rails select helper
+    # Format: [[display_name, identifier], ...]
+    # @return [Array<Array<String>>] options array for select dropdown
+    def options
+      [ default_option ] + agent_options
+    end
+
+    # Convenience class method that injects agents from Observ::AgentProvider
+    # Useful when you don't need to filter or transform agents
+    # @param agents [Array<Class>] optional array of agents (defaults to Observ::AgentProvider.all_agents)
+    # @return [Array<Array<String>>] options array for select dropdown
+    def self.options(agents: Observ::AgentProvider.all_agents)
+      new(agents: agents).options
+    end
+
+    private
+
+    # Default option for "no agent selected" state
+    # @return [Array<String>] the default option
+    def default_option
+      [ "Default Agent", "" ]
+    end
+
+    # Maps agents to [display_name, identifier] pairs
+    # @return [Array<Array<String>>] agent options
+    def agent_options
+      agents.map { |agent| [ agent.display_name, agent.agent_identifier ] }
+    end
+  end
+end

data/app/services/observ/agent_executor_service.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+module Observ
+  # Generic service for executing any agent against an input
+  #
+  # This service encapsulates the RubyLLM chat configuration and execution pattern,
+  # providing a unified way to run agents for dataset evaluations or other purposes.
+  #
+  # The service:
+  # - Creates a RubyLLM chat with the agent's model
+  # - Applies system prompt, schema, and model parameters
+  # - Optionally instruments the chat for observability
+  # - Handles both simple text input and structured context hashes
+  #
+  # Usage:
+  #   # Basic usage
+  #   executor = Observ::AgentExecutorService.new(LanguageDetectionAgent)
+  #   result = executor.call("Hello, how are you?")
+  #
+  #   # With observability
+  #   session = Observ::Session.create!(user_id: "user_123")
+  #   executor = Observ::AgentExecutorService.new(
+  #     LanguageDetectionAgent,
+  #     observability_session: session
+  #   )
+  #   result = executor.call("Bonjour!")
+  #
+  #   # With context hash (for agents that implement build_user_prompt)
+  #   executor = Observ::AgentExecutorService.new(CharacterGeneratorAgent)
+  #   result = executor.call(genre: "fantasy", title: "Dragon Quest")
+  #
+  class AgentExecutorService
+    class ExecutionError < StandardError; end
+    class RubyLLMNotAvailableError < ExecutionError; end
+
+    attr_reader :agent_class, :observability_session
+
+    # Initialize the executor
+    #
+    # @param agent_class [Class] The agent class to execute (must respond to model, system_prompt)
+    # @param observability_session [Observ::Session, nil] Optional session for tracing
+    # @param context [Hash] Additional context metadata for tracing
+    def initialize(agent_class, observability_session: nil, context: {})
+      @agent_class = agent_class
+      @observability_session = observability_session
+      @context = context
+
+      validate_ruby_llm_available!
+      validate_agent_class!
+    end
+
+    # Execute the agent with the given input
+    #
+    # @param input [String, Hash] The input text or context hash
+    # @return [Hash, String] The agent's response (structured if schema defined, string otherwise)
+    # @raise [ExecutionError] If the agent execution fails
+    def call(input)
+      chat = build_chat
+      configure_chat(chat)
+      instrument_chat(chat) if observability_session
+
+      user_prompt = build_user_prompt(input)
+      response = chat.ask(user_prompt)
+
+      normalize_response(response.content)
+    rescue StandardError => e
+      raise ExecutionError, "Agent execution failed: #{e.message}"
+    end
+
+    private
+
+    def validate_ruby_llm_available!
+      return if defined?(RubyLLM)
+
+      raise RubyLLMNotAvailableError,
+        "RubyLLM is not available. Please ensure the ruby_llm gem is installed and configured."
+    end
+
+    def validate_agent_class!
+      unless agent_class.respond_to?(:model) && agent_class.respond_to?(:system_prompt)
+        raise ArgumentError,
+          "Agent class must respond to :model and :system_prompt. Got: #{agent_class.name}"
+      end
+    end
+
+    def build_chat
+      RubyLLM.chat(model: agent_class.model)
+    end
+
+    def configure_chat(chat)
+      # Apply system prompt
+      chat.with_instructions(agent_class.system_prompt)
+
+      # Apply schema for structured output if agent defines one
+      if agent_class.respond_to?(:schema) && agent_class.schema
+        chat.with_schema(agent_class.schema)
+      end
+
+      # Apply model parameters (temperature, max_tokens, etc.)
+      if agent_class.respond_to?(:model_parameters)
+        params = agent_class.model_parameters
+        chat.with_params(**params) if params.any?
+      end
+
+      chat
+    end
+
+    def instrument_chat(chat)
+      return unless observability_session
+
+      instrumenter = Observ::ChatInstrumenter.new(
+        observability_session,
+        chat,
+        context: default_context.merge(@context)
+      )
+      instrumenter.instrument!
+    end
+
+    def default_context
+      {
+        service: "agent_executor",
+        agent_class: agent_class
+      }
+    end
+
+    # Build the user prompt from input
+    #
+    # If the agent implements build_user_prompt, use it with the input as context.
+    # Otherwise, extract text from the input directly.
+    def build_user_prompt(input)
+      if agent_class.respond_to?(:build_user_prompt)
+        context = input.is_a?(Hash) ? input : { text: input }
+        agent_class.build_user_prompt(context)
+      else
+        extract_text_input(input)
+      end
+    end
+
+    def extract_text_input(input)
+      case input
+      when String
+        input
+      when Hash
+        # Try common keys for text content
+        input[:text] || input["text"] ||
+          input[:content] || input["content"] ||
+          input[:input] || input["input"] ||
+          input.to_json
+      else
+        input.to_s
+      end
+    end
+
+    def normalize_response(content)
+      case content
+      when Hash
+        # Symbolize keys for consistent access
+        deep_symbolize_keys(content)
+      when String
+        content
+      else
+        content.respond_to?(:to_h) ? deep_symbolize_keys(content.to_h) : content
+      end
+    end
+
+    def deep_symbolize_keys(hash)
+      hash.each_with_object({}) do |(key, value), result|
+        new_key = key.respond_to?(:to_sym) ? key.to_sym : key
+        new_value = value.is_a?(Hash) ? deep_symbolize_keys(value) : value
+        result[new_key] = new_value
+      end
+    end
+  end
+end

data/app/services/observ/agent_provider.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Observ
+  # Service for discovering and providing available agents
+  # This is the ONLY class that knows how to discover agents in the application
+  #
+  # Responsibilities:
+  #   - Loading agent files in development mode (via Zeitwerk)
+  #   - Discovering all agents that implement Observ::AgentSelectable
+  #   - Sorting and filtering agents
+  #
+  # The Observ domain queries this service to get available agents,
+  # maintaining clean separation between domains.
+  #
+  # Configuration:
+  #   You can customize the agent discovery path via configuration:
+  #
+  #   Observ.configure do |config|
+  #     config.agent_path = Rails.root.join("lib", "my_agents")
+  #   end
+  #
+  # Usage:
+  #   agents = Observ::AgentProvider.all_agents
+  #   # => [LanguageDetectionAgent, MoodDetectionAgent, ...]
+  class AgentProvider
+    class << self
+      # Returns all available agents that implement the Observ::AgentSelectable interface
+      # Agents are sorted alphabetically by display_name
+      #
+      # @return [Array<Class>] array of agent classes
+      def all_agents
+        ensure_agents_loaded
+
+        return [] unless defined?(::BaseAgent)
+
+        ::BaseAgent.descendants
+          .select { |agent_class| agent_class.included_modules.include?(Observ::AgentSelectable) }
+          .sort_by(&:display_name)
+      end
+
+      private
+
+      # Ensures all agent files are loaded in development mode
+      # Uses Zeitwerk's eager_load_dir for thread-safe, Rails-idiomatic loading
+      # In production, eager loading handles this automatically (this becomes a no-op)
+      def ensure_agents_loaded
+        return if Rails.application.config.eager_load
+
+        agent_path = Observ.config.agent_path || default_agent_path
+        Rails.autoloaders.main.eager_load_dir(agent_path) if agent_path.exist?
+      end
+
+      # Default path where agents are located
+      # @return [Pathname] the default agent path
+      def default_agent_path
+        Rails.root.join("app", "agents")
+      end
+    end
+  end
+end

data/app/services/observ/agent_selection_service.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Observ
+  # Service for providing agent selection options in the Observ domain
+  #
+  # This service encapsulates the workflow of:
+  #   1. Discovering available agents (via Observ::AgentProvider)
+  #   2. Formatting them for UI presentation (via AgentSelectPresenter)
+  #
+  # This service acts as the single entry point for agent selection,
+  # hiding the complexity of agent discovery and presentation from
+  # controllers and views.
+  #
+  # Usage in controllers:
+  #   @agent_select_options = Observ::AgentSelectionService.options
+  #
+  # Usage in helpers:
+  #   def agent_select_options
+  #     Observ::AgentSelectionService.options
+  #   end
+  #
+  # Example output:
+  #   [
+  #     ["Default Agent", ""],
+  #     ["Deep Research", "DeepResearchAgent"],
+  #     ["Simple Research", "ResearchAgent"]
+  #   ]
+  class AgentSelectionService
+    class << self
+      # Returns formatted select options for agent selection dropdown
+      #
+      # This method orchestrates the entire agent selection workflow:
+      # - Discovers all available agents
+      # - Formats them for use in Rails select helpers
+      #
+      # @return [Array<Array<String>>] options array for Rails select helper
+      #   Format: [["Display Name", "ClassName"], ...]
+      def options
+        AgentSelectPresenter.options(agents: Observ::AgentProvider.all_agents)
+      end
+
+      # Returns all available agents (pass-through to Observ::AgentProvider)
+      #
+      # Useful when you need the raw agent classes instead of formatted options.
+      # For most UI purposes, prefer using .options instead.
+      #
+      # @return [Array<Class>] array of agent classes implementing Observ::AgentSelectable
+      def all_agents
+        Observ::AgentProvider.all_agents
+      end
+    end
+  end
+end