rubyllm-observ 0.5.0

data/lib/generators/observ/install_chat/install_chat_generator.rb

@@ -0,0 +1,313 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module Observ
+  module Generators
+    # Generator for installing Observ chat/agent testing feature
+    #
+    # This generator enhances RubyLLM infrastructure with Observ-specific
+    # agent capabilities and observability features.
+    #
+    # Prerequisites:
+    #   - RubyLLM gem installed (gem 'ruby_llm')
+    #   - rails generate ruby_llm:install (run first)
+    #   - rails db:migrate
+    #   - rails ruby_llm:load_models
+    #
+    # Usage:
+    #   rails generate observ:install:chat
+    #   rails generate observ:install:chat --skip-tools
+    #   rails generate observ:install:chat --skip-migrations
+    class InstallChatGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      class_option :skip_tools,
+                   type: :boolean,
+                   desc: "Skip tool class generation",
+                   default: false
+
+      class_option :skip_migrations,
+                   type: :boolean,
+                   desc: "Skip migration generation",
+                   default: false
+
+      class_option :skip_job,
+                   type: :boolean,
+                   desc: "Skip ChatResponseJob generation",
+                   default: false
+
+      class_option :with_phase_tracking,
+                   type: :boolean,
+                   desc: "Include phase tracking for multi-phase agents",
+                   default: false
+
+      def check_prerequisites
+        say "\n"
+        say "=" * 80, :cyan
+        say "Observ Chat Feature Installation", :cyan
+        say "=" * 80, :cyan
+        say "\n"
+
+        check_ruby_llm_gem
+        check_ruby_llm_models_installed
+      end
+
+      def create_migrations
+        return if options[:skip_migrations]
+
+        say "Creating Observ-specific migrations...", :cyan
+        say "-" * 80, :cyan
+
+        migration_template "migrations/add_agent_class_name.rb.tt",
+                          "db/migrate/add_agent_class_name_to_chats.rb"
+
+        migration_template "migrations/add_observability_session_id.rb.tt",
+                          "db/migrate/add_observability_session_id_to_chats.rb"
+
+        say "  ✓ Created agent_class_name migration", :green
+        say "  ✓ Created observability_session_id migration", :green
+        say "\n"
+      end
+
+      def enhance_models
+        say "Enhancing RubyLLM models with Observ functionality...", :cyan
+        say "-" * 80, :cyan
+
+        enhance_chat_model
+        enhance_message_model
+
+        say "\n"
+      end
+
+      def create_agent_infrastructure
+        say "Creating agent infrastructure...", :cyan
+        say "-" * 80, :cyan
+
+        template "agents/base_agent.rb.tt", "app/agents/base_agent.rb"
+
+        say "  ✓ Created BaseAgent", :green
+        say "  ℹ AgentProvider is now available as Observ::AgentProvider (no generation needed)", :yellow
+        say "  ℹ AgentSelectable is now available as Observ::AgentSelectable (no generation needed)", :yellow
+        say "  ℹ PromptManagement is now available as Observ::PromptManagement (no generation needed)", :yellow
+        say "\n"
+      end
+
+      def create_example_agent
+        say "Creating example agent...", :cyan
+        say "-" * 80, :cyan
+
+        template "agents/simple_agent.rb.tt", "app/agents/simple_agent.rb"
+
+        say "  ✓ Created SimpleAgent", :green
+        say "\n"
+      end
+
+      def create_job
+        return if options[:skip_job]
+
+        say "Creating ChatResponseJob...", :cyan
+        say "-" * 80, :cyan
+
+        template "jobs/chat_response_job.rb.tt", "app/jobs/chat_response_job.rb"
+
+        say "  ✓ Created ChatResponseJob", :green
+        say "\n"
+      end
+
+      def create_tools
+        return if options[:skip_tools]
+
+        say "Creating tool classes...", :cyan
+        say "-" * 80, :cyan
+
+        template "tools/think_tool.rb.tt", "app/tools/think_tool.rb"
+
+        say "  ✓ Created ThinkTool (basic example)", :green
+        say "  ℹ For advanced tools (web search, etc.), see documentation", :yellow
+        say "\n"
+      end
+
+      def create_view_partials
+        say "Creating view partials...", :cyan
+        say "-" * 80, :cyan
+
+        template "views/messages/_content.html.erb.tt", "app/views/messages/_content.html.erb"
+
+        say "  ✓ Created messages/_content partial", :green
+        say "\n"
+      end
+
+      def create_initializer
+        say "Creating observability initializer...", :cyan
+        say "-" * 80, :cyan
+
+        template "initializers/observability.rb.tt", "config/initializers/observability.rb"
+
+        say "  ✓ Created observability initializer (debug logging enabled)", :green
+        say "\n"
+      end
+
+      def add_phase_tracking
+        return unless options[:with_phase_tracking]
+
+        say "Adding phase tracking support...", :cyan
+        say "-" * 80, :cyan
+
+        # Call the add_phase_tracking generator
+        generate "observ:add_phase_tracking"
+
+        say "\n"
+      end
+
+      def show_post_install_instructions
+        say "\n"
+        say "=" * 80, :green
+        say "Observ Chat Feature Installation Complete!", :green
+        say "=" * 80, :green
+        say "\n"
+
+        say "Next steps:", :cyan
+        say "\n"
+
+        say "1. Run migrations:", :cyan
+        say "   rails db:migrate", :white
+        say "\n"
+
+        say "2. Start your Rails server and visit:", :cyan
+        say "   http://localhost:3000/observ/chats", :white
+        say "\n"
+
+        say "3. Create your first agent by extending BaseAgent:", :cyan
+        say "   See app/agents/simple_agent.rb for an example", :white
+        say "\n"
+
+        unless options[:with_phase_tracking]
+          say "Optional: Add phase tracking for multi-phase agents:", :cyan
+          say "   rails generate observ:add_phase_tracking", :white
+          say "\n"
+        end
+
+        say "Documentation:", :cyan
+        say "  • Agent development: observ/docs/AGENT_DEVELOPMENT.md", :white
+        say "  • Tool development: observ/docs/TOOL_DEVELOPMENT.md", :white
+        say "\n"
+      end
+
+      private
+
+      def check_ruby_llm_gem
+        unless gem_installed?("ruby_llm")
+          raise Thor::Error, <<~ERROR
+            RubyLLM gem not found!
+
+            This generator requires RubyLLM to be installed first.
+
+            Please run:
+              1. Add to Gemfile: gem 'ruby_llm'
+              2. bundle install
+              3. rails generate ruby_llm:install
+              4. rails db:migrate
+              5. rails ruby_llm:load_models
+            #{'  '}
+            Then run this generator again.
+          ERROR
+        end
+        say "  ✓ RubyLLM gem found", :green
+      end
+
+      def check_ruby_llm_models_installed
+        models_to_check = %w[Chat Message ToolCall Model]
+        missing_models = []
+
+        models_to_check.each do |model_name|
+          model_path = Rails.root.join("app/models/#{model_name.underscore}.rb")
+          unless File.exist?(model_path)
+            missing_models << model_name
+          end
+        end
+
+        if missing_models.any?
+          raise Thor::Error, <<~ERROR
+            RubyLLM models not found: #{missing_models.join(', ')}
+
+            This generator requires ruby_llm:install to be run first.
+
+            Please run:
+              1. rails generate ruby_llm:install
+              2. rails db:migrate
+              3. rails ruby_llm:load_models
+            #{'  '}
+            Then run this generator again.
+          ERROR
+        end
+
+        say "  ✓ RubyLLM models found (Chat, Message, ToolCall, Model)", :green
+      end
+
+      def enhance_chat_model
+        # Include the concern from gem in Chat model if not already included
+        chat_content = File.read(Rails.root.join("app/models/chat.rb"))
+
+        unless chat_content.include?("Observ::ChatEnhancements")
+          inject_into_file "app/models/chat.rb", after: /class Chat < ApplicationRecord\n/ do
+            "  include Observ::ChatEnhancements\n\n"
+          end
+          say "  ✓ Included Observ::ChatEnhancements in Chat model", :green
+        else
+          say "  ⚠ Chat model already includes Observ::ChatEnhancements", :yellow
+        end
+
+        # Add agent_class method if agent_class_name column exists
+        unless chat_content.include?("def agent_class")
+          inject_into_file "app/models/chat.rb", before: /^end\s*$/ do
+            <<~RUBY
+
+                # Return the agent class for this chat
+                # Override this method if you need custom agent class resolution
+                def agent_class
+                  return BaseAgent if agent_class_name.blank?
+
+                  agent_class_name.constantize
+                rescue NameError
+                  Rails.logger.warn "Agent class \#{agent_class_name} not found, using BaseAgent"
+                  BaseAgent
+                end
+            RUBY
+          end
+          say "  ✓ Added agent_class method to Chat model", :green
+        else
+          say "  ⚠ Chat model already has agent_class method", :yellow
+        end
+      end
+
+      def enhance_message_model
+        # Include the concern from gem in Message model if not already included
+        message_content = File.read(Rails.root.join("app/models/message.rb"))
+
+        unless message_content.include?("Observ::MessageEnhancements")
+          inject_into_file "app/models/message.rb", after: /class Message < ApplicationRecord\n/ do
+            "  include Observ::MessageEnhancements\n\n"
+          end
+          say "  ✓ Included Observ::MessageEnhancements in Message model", :green
+        else
+          say "  ⚠ Message model already includes Observ::MessageEnhancements", :yellow
+        end
+      end
+
+      def gem_installed?(gem_name)
+        Gem::Specification.find_all_by_name(gem_name).any?
+      rescue Gem::LoadError
+        false
+      end
+
+      # Helper for migration timestamps
+      def migration_version
+        "[#{Rails::VERSION::MAJOR}.#{Rails::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/generators/observ/install_chat/templates/agents/base_agent.rb.tt

@@ -0,0 +1,147 @@
+# Base interface for all agents
+# Defines the contract that all agents must implement
+#
+# Responsibilities:
+#   - Define required interface methods (system_prompt, default_model)
+#   - Define optional interface methods (tools, initial_greeting)
+#   - Provide setup methods that work with the interface
+#
+# Usage:
+#   class MyAgent < BaseAgent
+#     def self.system_prompt
+#       "You are a helpful assistant."
+#     end
+#
+#     def self.default_model
+#       "gpt-4o-mini"
+#     end
+#
+#     def self.tools
+#       [MyTool]
+#     end
+#   end
+#
+# For agents that want prompt management, include the Observ::PromptManagement concern:
+#   class MyAgent < BaseAgent
+#     include Observ::PromptManagement
+#     # ...
+#   end
+class BaseAgent
+  # ============================================
+  # INTERFACE METHODS - Override in subclasses
+  # ============================================
+
+  # System prompt for the agent
+  # @return [String] The system prompt
+  def self.system_prompt
+    raise NotImplementedError, "#{name} must implement .system_prompt"
+  end
+
+  # Default model for this agent
+  # @return [String] The default model identifier
+  def self.default_model
+    raise NotImplementedError, "#{name} must implement .default_model"
+  end
+
+  # Tools available to this agent
+  # @return [Array<Class>] Array of tool classes
+  def self.tools
+    []
+  end
+
+  # Initial greeting message when chat starts
+  # @return [String, nil] The greeting message or nil
+  def self.initial_greeting
+    nil
+  end
+
+  # Model to use for this agent
+  # Can be overridden by concerns (e.g., Observ::PromptManagement)
+  # @return [String] The model identifier to use
+  def self.model
+    default_model
+  end
+
+  # Default model parameters for this agent (temperature, max_tokens, etc.)
+  # @return [Hash] Hash of model parameters
+  def self.default_model_parameters
+    {}  # Override in subclasses for custom defaults
+  end
+
+  # Model parameters to use for this agent
+  # Can be overridden by concerns (e.g., Observ::PromptManagement)
+  # @return [Hash] The model parameters to use
+  def self.model_parameters
+    default_model_parameters
+  end
+
+  # ============================================
+  # SETUP METHODS - Work with the interface
+  # ============================================
+
+  # Setup instructions for the chat session
+  # @param chat [Chat] The chat session
+  # @return [Chat] The configured chat session
+  def self.setup_instructions(chat)
+    chat.with_instructions(system_prompt) if system_prompt.present?
+    chat
+  end
+
+  # Setup tools for the chat session
+  # @param chat [Chat] The chat session
+  # @return [Chat] The configured chat session
+  def self.setup_tools(chat)
+    if tools.any?
+      instantiated_tools = tools.map do |tool|
+        if tool.instance_of?(Class)
+          observ_session = chat.observ_session if chat.respond_to?(:observ_session)
+          tool.new(observ_session)
+        else
+          if tool.respond_to?(:observability=) && chat.respond_to?(:observ_session)
+            tool.observability = chat.observ_session
+          end
+          tool
+        end
+      end
+      chat.with_tools(*instantiated_tools)
+    end
+    chat
+  end
+
+  # Setup model parameters for the chat session
+  # @param chat [Chat] The chat session
+  # @return [Chat] The configured chat session
+  def self.setup_parameters(chat)
+    params = model_parameters
+    
+    # Convert string numeric values to proper types for API compatibility
+    # This is necessary because prompt configs may return string values
+    normalized_params = params.transform_values do |value|
+      case value
+      when String
+        # Convert numeric strings to numbers
+        if value.match?(/\A-?\d+\.?\d*\z/)
+          value.include?('.') ? value.to_f : value.to_i
+        else
+          value
+        end
+      else
+        value
+      end
+    end
+    
+    chat.with_params(**normalized_params) if normalized_params.any?
+    chat
+  end
+
+  # Send initial greeting message
+  # @param chat [Chat] The chat session
+  def self.send_initial_greeting(chat)
+    return unless initial_greeting
+
+    chat.messages.create!(
+      role: :assistant,
+      content: initial_greeting
+    )
+  end
+end

data/lib/generators/observ/install_chat/templates/agents/simple_agent.rb.tt

@@ -0,0 +1,55 @@
+# SimpleAgent - A basic agent example
+#
+# This agent demonstrates the minimum required implementation
+# for a working agent in the Observ chat UI.
+#
+# To create your own agent:
+# 1. Extend BaseAgent
+# 2. Include Observ::AgentSelectable to make it appear in the UI
+# 3. Implement required methods: system_prompt, default_model, display_name
+# 4. Optionally add tools and initial_greeting
+class SimpleAgent < BaseAgent
+  include Observ::AgentSelectable
+
+  # Display name shown in the Observ UI
+  def self.display_name
+    "Simple Agent"
+  end
+
+  # Description shown in agent selection
+  def self.description
+    "A basic conversational agent"
+  end
+
+  # System prompt that defines the agent's behavior
+  def self.system_prompt
+    <<~PROMPT
+      You are a helpful AI assistant.
+      
+      Your role is to:
+      - Answer questions clearly and concisely
+      - Be friendly and professional
+      - Admit when you don't know something
+      
+      Current date: #{Time.current.strftime("%B %d, %Y")}
+    PROMPT
+  end
+
+  # Default model to use for this agent
+  def self.default_model
+    "gpt-4o-mini"
+  end
+
+  # Optional: Greeting message when chat starts
+  def self.initial_greeting
+    <<~GREETING
+      Hello! I'm a simple AI assistant. How can I help you today?
+    GREETING
+  end
+
+  # Optional: Tools available to this agent
+  # Uncomment and add tool classes as needed:
+  # def self.tools
+  #   [ThinkTool]
+  # end
+end

data/lib/generators/observ/install_chat/templates/concerns/observ_chat_enhancements.rb.tt

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module ObservChatEnhancements
+  extend ActiveSupport::Concern
+
+  included do
+    include Observ::ObservabilityInstrumentation
+    
+    # Consolidated callback for agent initialization to reduce redundant queries
+    after_create :initialize_agent, if: -> { agent_class_name.present? }
+  end
+
+  def agent_class
+    return BaseAgent if agent_class_name.blank?
+
+    agent_class_name.constantize
+  rescue NameError
+    Rails.logger.warn "Agent class #{agent_class_name} not found, using BaseAgent"
+    BaseAgent
+  end
+
+  def setup_tools
+    agent_class.setup_tools(self)
+  end
+
+  private
+
+  def initialize_agent
+    # Execute both agent setup steps in one consolidated callback
+    # This prevents redundant association loading between callbacks
+    agent_class.setup_instructions(self)
+    agent_class.send_initial_greeting(self)
+  end
+end

data/lib/generators/observ/install_chat/templates/concerns/observ_message_enhancements.rb.tt

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module ObservMessageEnhancements
+  extend ActiveSupport::Concern
+
+  included do
+    include Observ::TraceAssociation
+    
+    broadcasts_to ->(message) { "chat_#{message.chat_id}" }, partial: "observ/messages/message"
+  end
+
+  def broadcast_append_chunk(content)
+    broadcast_append_to "chat_#{chat_id}",
+      target: "message_#{id}_content",
+      partial: "observ/messages/content",
+      locals: { content: content }
+  end
+end

data/lib/generators/observ/install_chat/templates/initializers/observability.rb.tt

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Observability configuration
+# This file is optional and is used to configure observability settings
+
+Rails.application.configure do
+  config.observability = ActiveSupport::OrderedOptions.new
+  
+  # Enable observability instrumentation
+  # When enabled, sessions, traces, and observations are automatically tracked
+  config.observability.enabled = true
+  
+  # Automatically instrument RubyLLM chats with observability
+  # When enabled, LLM calls, tool usage, and metrics are tracked
+  config.observability.auto_instrument_chats = true
+  
+  # Enable debug logging for observability metrics
+  # When enabled, job completion metrics (tokens, cost) will be logged
+  config.observability.debug = Rails.env.development?
+end

data/lib/generators/observ/install_chat/templates/jobs/chat_response_job.rb.tt

@@ -0,0 +1,56 @@
+class ChatResponseJob < ApplicationJob
+  retry_on RubyLLM::BadRequestError, wait: 2.seconds, attempts: 1
+
+  def perform(chat_id, content)
+    chat = Chat.find(chat_id)
+
+    # Observability is automatically enabled via after_find callback
+    # All LLM calls, tool calls, and metrics are tracked automatically
+
+    chat.setup_tools
+
+    begin
+      # Model parameters (temperature, max_tokens, etc.) are automatically configured
+      # via the initialize_agent callback when the chat is created
+      chat.ask(content) do |chunk|
+        if chunk.content && !chunk.content.blank?
+          message = chat.messages.last
+          message.broadcast_append_chunk(chunk.content)
+        end
+      end
+    rescue RubyLLM::BadRequestError => e
+      Rails.logger.error "[ChatResponseJob] BadRequestError: #{e.message}"
+
+      error_message = chat.messages.create!(
+        role: :assistant,
+        content: "I apologize, but I encountered an error while processing your request. This might be due to a tool call issue. Please try rephrasing your question or try again."
+      )
+
+      error_message.broadcast_replace_to(
+        "chat_#{chat.id}",
+        target: "messages",
+        partial: "observ/messages/message",
+        locals: { message: error_message }
+      )
+
+      raise
+    end
+
+    chat.finalize_observability_session if chat.observ_session
+
+    if observability_debug_enabled? && chat.observ_session
+      metrics = chat.observ_session.session_metrics
+      Rails.logger.info "[Observability] Job completed - Tokens: #{metrics[:total_tokens]}, Cost: $#{metrics[:total_cost]}"
+    end
+  end
+
+  private
+
+  def observability_debug_enabled?
+    Rails.configuration.respond_to?(:observability) &&
+      Rails.configuration.observability.respond_to?(:debug) &&
+      Rails.configuration.observability.debug
+  rescue NoMethodError
+    false
+  end
+end

data/lib/generators/observ/install_chat/templates/migrations/add_agent_class_name.rb.tt

@@ -0,0 +1,6 @@
+class AddAgentClassNameToChats < ActiveRecord::Migration<%= migration_version %>
+  def change
+    add_column :chats, :agent_class_name, :string
+    add_index :chats, :agent_class_name
+  end
+end

data/lib/generators/observ/install_chat/templates/migrations/add_observability_session_id.rb.tt

@@ -0,0 +1,6 @@
+class AddObservabilitySessionIdToChats < ActiveRecord::Migration<%= migration_version %>
+  def change
+    add_column :chats, :observability_session_id, :string
+    add_index :chats, :observability_session_id
+  end
+end

data/lib/generators/observ/install_chat/templates/tools/think_tool.rb.tt

@@ -0,0 +1,29 @@
+require "ruby_llm"
+
+# ThinkTool - A basic tool for agent reflection
+#
+# This tool allows agents to "think out loud" and reflect on their
+# reasoning process. Useful for debugging and understanding agent behavior.
+#
+# To create your own tools:
+# 1. Extend RubyLLM::Tool
+# 2. Add description and params
+# 3. Implement execute method
+# 4. Add tool to agent's tools array
+class ThinkTool < RubyLLM::Tool
+  description "Reflect on current progress and plan next steps. Use this to organize your thoughts."
+
+  param :reflection,
+        desc: "Your thoughts on what you've learned so far and what to do next",
+        type: :string
+
+  attr_accessor :observability
+
+  def initialize(observability = nil)
+    @observability = observability
+  end
+
+  def execute(reflection:)
+    "Reflection noted: #{reflection}"
+  end
+end

data/lib/generators/observ/install_chat/templates/views/messages/_content.html.erb.tt

@@ -0,0 +1 @@
+<%%= content %>