rubyllm-observ 0.5.0

data/app/models/concerns/observ/observability_instrumentation.rb

@@ -0,0 +1,124 @@
+module Observ
+  module ObservabilityInstrumentation
+    extend ActiveSupport::Concern
+
+  included do
+    belongs_to :observ_session, class_name: "Observ::Session", foreign_key: :observability_session_id,
+               primary_key: :session_id, optional: true
+
+    after_create :initialize_observability_session
+    after_find :ensure_instrumented_if_needed
+
+    attr_accessor :instrumenter
+  end
+
+  def ask_with_observability(message, **options)
+    ensure_instrumented!
+    ask(message, **options)
+  end
+
+  def complete_with_observability(&block)
+    ensure_instrumented!
+    complete(&block)
+  end
+
+  def update_observability_context(new_context)
+    return unless observ_session && @instrumenter
+
+    observ_session.update_metadata(new_context)
+    @instrumenter.instance_variable_get(:@context).merge!(new_context)
+  end
+
+  def finalize_observability_session
+    return unless observ_session
+
+    observ_session.finalize
+    Rails.logger.info "[Observability] Session finalized: #{observ_session.session_id}"
+  end
+
+  private
+
+  # Hook method for building observability session metadata
+  # Override this in your model or concerns to add custom metadata
+  # @return [Hash] metadata hash
+  def observability_metadata
+    {
+      agent_type: agent_class_name || "standard",
+      chat_id: id
+    }
+  end
+
+  # Hook method for building observability context
+  # Override this in your model or concerns to add custom context
+  # @return [Hash] context hash
+  def observability_context
+    context = {
+      agent_type: agent_class_name || "standard",
+      chat_id: id
+    }
+
+    # Include agent_class if available for prompt metadata extraction
+    if respond_to?(:agent_class)
+      context[:agent_class] = agent_class
+    end
+
+    # Include prompt version override if specified
+    if respond_to?(:prompt_version) && prompt_version.present?
+      context[:prompt_version_override] = prompt_version
+    end
+
+    context
+  end
+
+  def initialize_observability_session
+    return unless Rails.configuration.observability.enabled
+
+    session = Observ::Session.create!(
+      user_id: "chat_#{id}",
+      metadata: observability_metadata
+    )
+
+    update_column(:observability_session_id, session.session_id)
+
+    instrument_rubyllm_chat if Rails.configuration.observability.auto_instrument_chats
+  rescue StandardError => e
+    Rails.logger.error "[Observability] Failed to initialize session: #{e.message}"
+  end
+
+  def instrument_rubyllm_chat
+    return unless observ_session
+    return if @instrumenter
+
+    @instrumenter = Observ::ChatInstrumenter.new(
+      observ_session,
+      self,
+      context: observability_context
+    )
+    @instrumenter.instrument!
+
+  rescue StandardError => e
+    Rails.logger.error "[Observability] Failed to instrument chat: #{e.message}"
+  end
+
+  def ensure_instrumented!
+    return if @instrumenter
+
+    reload_observ_session if observability_session_id && !observ_session
+    instrument_rubyllm_chat if observ_session
+  end
+
+  def reload_observ_session
+    self.observ_session = Observ::Session.find_by(session_id: observability_session_id)
+  end
+
+  def ensure_instrumented_if_needed
+    return unless Rails.configuration.observability.enabled
+    return if @instrumenter
+    return unless observability_session_id
+
+    ensure_instrumented!
+  rescue StandardError => e
+    Rails.logger.error "[Observability] Failed to auto-instrument on find: #{e.message}"
+  end
+  end
+end

data/app/models/concerns/observ/prompt_management.rb

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+
+module Observ
+  # Concern for agents that want to use the prompt management system
+  # Provides functionality to fetch prompts from the database with fallback support,
+  # caching, variable interpolation, and model configuration from prompt metadata.
+  #
+  # Usage:
+  #   class MyAgent < BaseAgent
+  #     include Observ::PromptManagement
+  #
+  #     FALLBACK_PROMPT = "You are a helpful assistant."
+  #
+  #     use_prompt_management(
+  #       prompt_name: "my-agent-system-prompt",
+  #       fallback: FALLBACK_PROMPT
+  #     )
+  #
+  #     def self.default_model
+  #       "gpt-4.1-nano"
+  #     end
+  #   end
+  module PromptManagement
+    extend ActiveSupport::Concern
+
+    included do
+      class_attribute :prompt_config, default: {}
+      # Cache the fetched prompt template object for metadata access
+      class_attribute :cached_prompt_template, default: nil
+      # Instance variable for version override (set per instance, not shared across class)
+      attr_accessor :prompt_version_override
+    end
+
+    class_methods do
+      # Enable/disable prompt management per agent
+      def use_prompt_management(enabled: true, prompt_name: nil, fallback: nil)
+        self.prompt_config = {
+          enabled: enabled,
+          prompt_name: prompt_name || default_prompt_name,
+          fallback: fallback || default_fallback_prompt
+        }
+      end
+
+      # Default prompt name based on agent class name
+      def default_prompt_name
+        name.underscore.tr("_/", "--") + "-system-prompt"
+        # Example: ResearchAgent => 'research-agent-system-prompt'
+      end
+
+      # Override in subclasses if needed
+      def default_fallback_prompt
+        "You are a helpful AI assistant."
+      end
+
+      # Check if prompt management is enabled for this agent
+      def prompt_management_enabled?
+        Observ.config.prompt_management_enabled &&
+        prompt_config[:enabled] != false
+      end
+
+      # Fetch prompt with fallback
+      # Supports version override via thread-local storage or parameter
+      def fetch_prompt(variables: {}, version: nil)
+        # Ensure defaults if prompt_config was never initialized
+        config = prompt_config.presence || {}
+        fallback = config[:fallback] || default_fallback_prompt
+        prompt_name = config[:prompt_name] || default_prompt_name
+
+        return fallback unless prompt_management_enabled?
+
+        start_time = Time.current
+
+        begin
+          # Check for version override from thread-local storage or parameter
+          version_to_use = Thread.current[:observ_prompt_version_override] || version
+
+          # Fetch prompt with version or state
+          if version_to_use.present?
+            prompt_template = Observ::PromptManager.fetch(
+              name: prompt_name,
+              version: version_to_use,
+              fallback: fallback
+            )
+          else
+            prompt_template = Observ::PromptManager.fetch(
+              name: prompt_name,
+              state: :production,
+              fallback: fallback
+            )
+          end
+
+          # Cache the template for metadata access
+          @_prompt_template = prompt_template
+
+          # Log fetch result
+          duration_ms = ((Time.current - start_time) * 1000).round(2)
+          if prompt_template.version
+            version_info = version_to_use.present? ? "(version: #{prompt_template.version})" : "(production, version: #{prompt_template.version})"
+            Rails.logger.info(
+              "Prompt fetched for #{name}: #{prompt_name} " \
+              "#{version_info}, " \
+              "duration: #{duration_ms}ms)"
+            )
+          else
+            Rails.logger.info(
+              "Using fallback prompt for #{name}: prompt '#{prompt_name}' not found " \
+              "(duration: #{duration_ms}ms)"
+            )
+          end
+
+          # Compile with variables (works for both Prompt and NullPrompt)
+          if variables.any?
+            prompt_template.compile(variables)
+          else
+            prompt_template.prompt
+          end
+        rescue => e
+          Rails.logger.error(
+            "Failed to fetch prompt for #{name}: #{e.message}\n" \
+            "#{e.backtrace.first(5).join("\n")}"
+          )
+          # Clear cached template on error
+          @_prompt_template = nil
+          fallback
+        end
+      end
+
+      # Override this in subclasses to provide dynamic variables
+      def prompt_variables
+        {
+          current_date: Time.current.strftime("%B %d, %Y"),
+          current_time: Time.current.strftime("%I:%M %p %Z")
+        }
+      end
+
+      # Override system_prompt to use prompt management
+      def system_prompt
+        @_system_prompt ||= fetch_prompt(variables: prompt_variables)
+      end
+
+      # Get the cached prompt template (with name and version)
+      def current_prompt_template
+        @_prompt_template
+      end
+
+      # Get prompt metadata for observability
+      # @return [Hash] Hash with :prompt_name and :prompt_version keys
+      def prompt_metadata
+        template = current_prompt_template
+        return {} unless template
+
+        {
+          prompt_name: template.respond_to?(:name) ? template.name : nil,
+          prompt_version: template.respond_to?(:version) ? template.version : nil
+        }.compact
+      end
+
+      # Clear cached prompt (useful for tests or when prompt is updated)
+      def reset_prompt_cache!
+        @_system_prompt = nil
+        @_prompt_template = nil
+      end
+
+      # Override model to check prompt metadata first
+      # Priority:
+      # 1. Prompt metadata (config['model']) - if prompt management is enabled
+      # 2. Agent's default_model - fallback
+      def model
+        # Check if prompt has model in metadata
+        if prompt_management_enabled?
+          model_from_prompt = fetch_model_from_prompt
+          return model_from_prompt if model_from_prompt.present?
+        end
+
+        # Fallback to agent's default
+        default_model
+      end
+
+      # Fetch model from prompt metadata
+      # @return [String, nil] The model from prompt config, or nil
+      def fetch_model_from_prompt(version: nil)
+        return nil unless prompt_management_enabled?
+
+        config = prompt_config.presence || {}
+        fallback = config[:fallback] || default_fallback_prompt
+        prompt_name = config[:prompt_name] || default_prompt_name
+
+        begin
+          # Check for version override from thread-local storage or parameter
+          version_to_use = Thread.current[:observ_prompt_version_override] || version
+
+          # Fetch prompt with version or state
+          if version_to_use.present?
+            prompt_template = Observ::PromptManager.fetch(
+              name: prompt_name,
+              version: version_to_use,
+              fallback: fallback
+            )
+          else
+            prompt_template = Observ::PromptManager.fetch(
+              name: prompt_name,
+              state: :production,
+              fallback: fallback
+            )
+          end
+
+          # If we got a real Prompt object (not NullPrompt), check its config
+          if prompt_template.respond_to?(:config) && prompt_template.config.is_a?(Hash)
+            prompt_template.config["model"]
+          end
+        rescue => e
+          Rails.logger.debug(
+            "Could not fetch model from prompt #{prompt_name}: #{e.message}"
+          )
+          nil
+        end
+      end
+
+      # Override model_parameters to check prompt metadata first
+      # Priority:
+      # 1. Prompt metadata (config['temperature'], config['max_tokens'], etc.)
+      # 2. Agent's default_model_parameters - fallback
+      # @return [Hash] The model parameters to use
+      def model_parameters
+        # Check if prompt has parameters in metadata
+        if prompt_management_enabled?
+          params_from_prompt = fetch_model_parameters_from_prompt
+          return params_from_prompt if params_from_prompt.present?
+        end
+
+        # Fallback to agent's defaults
+        default_model_parameters
+      end
+
+      # Fetch model parameters from prompt metadata
+      # @return [Hash] The model parameters from prompt config
+      def fetch_model_parameters_from_prompt(version: nil)
+        return {} unless prompt_management_enabled?
+
+        config = prompt_config.presence || {}
+        fallback = config[:fallback] || default_fallback_prompt
+        prompt_name = config[:prompt_name] || default_prompt_name
+
+        begin
+          # Check for version override from thread-local storage or parameter
+          version_to_use = Thread.current[:observ_prompt_version_override] || version
+
+          # Fetch prompt with version or state
+          if version_to_use.present?
+            prompt_template = Observ::PromptManager.fetch(
+              name: prompt_name,
+              version: version_to_use,
+              fallback: fallback
+            )
+          else
+            prompt_template = Observ::PromptManager.fetch(
+              name: prompt_name,
+              state: :production,
+              fallback: fallback
+            )
+          end
+
+          # If we got a real Prompt object (not NullPrompt), extract parameters
+          if prompt_template.respond_to?(:config) && prompt_template.config.is_a?(Hash)
+            extract_llm_parameters(prompt_template.config)
+          else
+            {}
+          end
+        rescue => e
+          Rails.logger.debug(
+            "Could not fetch parameters from prompt #{prompt_name}: #{e.message}"
+          )
+          {}
+        end
+      end
+
+      private
+
+      # Extract LLM parameters from config hash
+      # @param config [Hash] The prompt config
+      # @return [Hash] Extracted parameters (temperature, max_tokens, etc.)
+      def extract_llm_parameters(config)
+        params = config.slice(
+          "temperature",
+          "max_tokens",
+          "top_p",
+          "frequency_penalty",
+          "presence_penalty",
+          "stop",
+          "response_format",
+          "seed"
+        ).transform_keys(&:to_sym).compact
+
+        # Convert string numbers to proper types (JSON returns strings)
+        params.transform_values do |value|
+          convert_to_numeric_if_needed(value)
+        end
+      end
+
+      # Convert string numbers to proper numeric types
+      # @param value [Object] The value to convert
+      # @return [Object] Converted value (or original if not a numeric string)
+      def convert_to_numeric_if_needed(value)
+        case value
+        when String
+          # Check if it's a numeric string (integer or float)
+          if value.match?(/\A-?\d+\.\d+\z/)
+            value.to_f
+          elsif value.match?(/\A-?\d+\z/)
+            value.to_i
+          else
+            value
+          end
+        else
+          value
+        end
+      end
+    end
+  end
+end

data/app/models/concerns/observ/trace_association.rb

@@ -0,0 +1,9 @@
+module Observ
+  module TraceAssociation
+    extend ActiveSupport::Concern
+
+    included do
+      has_many :traces, class_name: "Observ::Trace", dependent: :nullify
+    end
+  end
+end

data/app/models/observ/annotation.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Observ
+  class Annotation < ApplicationRecord
+    belongs_to :annotatable, polymorphic: true
+
+    validates :content, presence: true
+
+    # Serialize tags as JSON for SQLite compatibility
+    serialize :tags, coder: JSON
+
+    scope :recent, -> { order(created_at: :desc) }
+
+    # Ensure tags is always an array
+    after_initialize :ensure_tags_array
+
+    private
+
+    def ensure_tags_array
+      self.tags ||= []
+    end
+  end
+end

data/app/models/observ/application_record.rb

@@ -0,0 +1,5 @@
+module Observ
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/observ/dataset.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Observ
+  class Dataset < ApplicationRecord
+    self.table_name = "observ_datasets"
+
+    has_many :items, class_name: "Observ::DatasetItem",
+             foreign_key: :dataset_id, dependent: :destroy, inverse_of: :dataset
+    has_many :runs, class_name: "Observ::DatasetRun",
+             foreign_key: :dataset_id, dependent: :destroy, inverse_of: :dataset
+
+    validates :name, presence: true, uniqueness: true
+    validates :agent_class, presence: true
+    validate :agent_class_exists, if: -> { agent_class.present? }
+
+    # Returns the agent class constant
+    def agent
+      agent_class.constantize
+    end
+
+    # Returns only active items for running evaluations
+    def active_items
+      items.active
+    end
+
+    # Count helpers for UI
+    def items_count
+      items.count
+    end
+
+    def active_items_count
+      items.active.count
+    end
+
+    def runs_count
+      runs.count
+    end
+
+    def last_run
+      runs.order(created_at: :desc).first
+    end
+
+    private
+
+    def agent_class_exists
+      agent_class.constantize
+    rescue NameError
+      errors.add(:agent_class, "must be a valid agent class")
+    end
+  end
+end

data/app/models/observ/dataset_item.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Observ
+  class DatasetItem < ApplicationRecord
+    self.table_name = "observ_dataset_items"
+
+    belongs_to :dataset, class_name: "Observ::Dataset", inverse_of: :items
+    belongs_to :source_trace, class_name: "Observ::Trace", optional: true
+    has_many :run_items, class_name: "Observ::DatasetRunItem",
+             foreign_key: :dataset_item_id, dependent: :destroy, inverse_of: :dataset_item
+
+    enum :status, { active: 0, archived: 1 }
+
+    validates :input, presence: true
+
+    scope :active, -> { where(status: :active) }
+    scope :archived, -> { where(status: :archived) }
+
+    # Preview helpers for UI display
+    def input_preview(max_length: 100)
+      return nil if input.blank?
+      text = input.is_a?(Hash) ? input.to_json : input.to_s
+      text.length > max_length ? "#{text[0...max_length]}..." : text
+    end
+
+    def expected_output_preview(max_length: 100)
+      return nil if expected_output.blank?
+      text = expected_output.is_a?(Hash) ? expected_output.to_json : expected_output.to_s
+      text.length > max_length ? "#{text[0...max_length]}..." : text
+    end
+
+    # Check if this item has been run
+    def run_count
+      run_items.count
+    end
+
+    def last_run_item
+      run_items.order(created_at: :desc).first
+    end
+  end
+end

data/app/models/observ/dataset_run.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Observ
+  class DatasetRun < ApplicationRecord
+    self.table_name = "observ_dataset_runs"
+
+    belongs_to :dataset, class_name: "Observ::Dataset", inverse_of: :runs
+    has_many :run_items, class_name: "Observ::DatasetRunItem",
+             foreign_key: :dataset_run_id, dependent: :destroy, inverse_of: :dataset_run
+    has_many :items, through: :run_items, source: :dataset_item
+    has_many :scores, through: :run_items
+
+    enum :status, { pending: 0, running: 1, completed: 2, failed: 3 }
+
+    validates :name, presence: true, uniqueness: { scope: :dataset_id }
+
+    # Progress tracking
+    def progress_percentage
+      return 0 if total_items.zero?
+      ((completed_items + failed_items).to_f / total_items * 100).round(1)
+    end
+
+    def finished?
+      completed? || failed?
+    end
+
+    def in_progress?
+      pending? || running?
+    end
+
+    # Update aggregate metrics from run items
+    def update_metrics!
+      completed = run_items.where.not(trace_id: nil).where(error: nil).count
+      failed = run_items.where.not(error: nil).count
+
+      # Calculate cost and tokens from associated traces
+      trace_ids = run_items.where.not(trace_id: nil).pluck(:trace_id)
+      traces = Observ::Trace.where(id: trace_ids)
+
+      update!(
+        completed_items: completed,
+        failed_items: failed,
+        total_cost: traces.sum(:total_cost) || 0,
+        total_tokens: traces.sum(:total_tokens) || 0
+      )
+    end
+
+    # Initialize run items for all active dataset items
+    def initialize_run_items!
+      dataset.active_items.find_each do |item|
+        run_items.find_or_create_by!(dataset_item: item)
+      end
+      update!(total_items: run_items.count)
+    end
+
+    # Summary helpers for UI
+    def success_rate
+      return 0 if total_items.zero?
+      (completed_items.to_f / total_items * 100).round(1)
+    end
+
+    def failure_rate
+      return 0 if total_items.zero?
+      (failed_items.to_f / total_items * 100).round(1)
+    end
+
+    def pending_items_count
+      total_items - completed_items - failed_items
+    end
+
+    def duration_seconds
+      return nil unless finished? && run_items.any?
+      first_item = run_items.order(created_at: :asc).first
+      last_item = run_items.order(updated_at: :desc).first
+      (last_item.updated_at - first_item.created_at).round(1)
+    end
+
+    # Score aggregation
+    def average_score(name)
+      relevant_scores = scores.where(name: name)
+      return nil if relevant_scores.empty?
+      relevant_scores.average(:value)&.round(4)
+    end
+
+    def score_summary
+      scores.group(:name).average(:value).transform_values { |v| v.round(4) }
+    end
+
+    def pass_rate(score_name = nil)
+      scope = scores
+      scope = scope.where(name: score_name) if score_name
+      return nil if scope.empty?
+      (scope.where("value >= 0.5").count.to_f / scope.count * 100).round(1)
+    end
+
+    def items_with_scores_count
+      run_items.joins(:scores).distinct.count
+    end
+
+    def items_without_scores_count
+      total_items - items_with_scores_count
+    end
+  end
+end