ruby_llm-agents 0.1.0

data/app/models/ruby_llm/agents/execution/analytics.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    class Execution
+      # Analytics concern for advanced reporting and analysis
+      #
+      # Provides class methods for:
+      # - Daily reports with key metrics
+      # - Cost breakdown by agent type
+      # - Performance stats for specific agents
+      # - Version comparison
+      # - Trend analysis over time
+      #
+      module Analytics
+        extend ActiveSupport::Concern
+
+        class_methods do
+          # Daily report with key metrics
+          def daily_report
+            scope = today
+
+            {
+              date: Date.current,
+              total_executions: scope.count,
+              successful: scope.successful.count,
+              failed: scope.failed.count,
+              total_cost: scope.total_cost_sum || 0,
+              total_tokens: scope.total_tokens_sum || 0,
+              avg_duration_ms: scope.avg_duration&.round || 0,
+              avg_tokens: scope.avg_tokens&.round || 0,
+              error_rate: calculate_error_rate(scope),
+              by_agent: scope.group(:agent_type).count,
+              top_errors: scope.errors.group(:error_class).count.sort_by { |_, v| -v }.first(5).to_h
+            }
+          end
+
+          # Cost breakdown by agent type
+          def cost_by_agent(period: :today)
+            public_send(period)
+              .group(:agent_type)
+              .sum(:total_cost)
+              .sort_by { |_, cost| -(cost || 0) }
+              .to_h
+          end
+
+          # Performance stats for specific agent
+          def stats_for(agent_type, period: :today)
+            scope = by_agent(agent_type).public_send(period)
+            count = scope.count
+            total_cost = scope.total_cost_sum || 0
+
+            {
+              agent_type: agent_type,
+              period: period,
+              count: count,
+              total_cost: total_cost,
+              avg_cost: count > 0 ? (total_cost / count).round(6) : 0,
+              total_tokens: scope.total_tokens_sum || 0,
+              avg_tokens: scope.avg_tokens&.round || 0,
+              avg_duration_ms: scope.avg_duration&.round || 0,
+              success_rate: calculate_success_rate(scope),
+              error_rate: calculate_error_rate(scope)
+            }
+          end
+
+          # Compare versions of the same agent
+          def compare_versions(agent_type, version1, version2, period: :this_week)
+            base_scope = by_agent(agent_type).public_send(period)
+
+            v1_stats = stats_for_scope(base_scope.by_version(version1))
+            v2_stats = stats_for_scope(base_scope.by_version(version2))
+
+            {
+              agent_type: agent_type,
+              period: period,
+              version1: { version: version1, **v1_stats },
+              version2: { version: version2, **v2_stats },
+              improvements: {
+                cost_change_pct: percent_change(v1_stats[:avg_cost], v2_stats[:avg_cost]),
+                token_change_pct: percent_change(v1_stats[:avg_tokens], v2_stats[:avg_tokens]),
+                speed_change_pct: percent_change(v1_stats[:avg_duration_ms], v2_stats[:avg_duration_ms])
+              }
+            }
+          end
+
+          # Trend analysis over time
+          def trend_analysis(agent_type: nil, days: 7)
+            scope = agent_type ? by_agent(agent_type) : all
+
+            (0...days).map do |days_ago|
+              date = days_ago.days.ago.to_date
+              day_scope = scope.where(created_at: date.beginning_of_day..date.end_of_day)
+
+              {
+                date: date,
+                count: day_scope.count,
+                total_cost: day_scope.total_cost_sum || 0,
+                avg_duration_ms: day_scope.avg_duration&.round || 0,
+                error_count: day_scope.failed.count
+              }
+            end.reverse
+          end
+
+          # Chart data: Hourly activity chart for today showing success/failed
+          def hourly_activity_chart
+            success_data = {}
+            failed_data = {}
+
+            # Create entries for each hour of the day (0-23)
+            (0..23).each do |hour|
+              time_label = format("%02d:00", hour)
+              start_time = Time.current.beginning_of_day + hour.hours
+              end_time = start_time + 1.hour
+
+              hour_scope = where(created_at: start_time...end_time)
+              total = hour_scope.count
+              failed = hour_scope.failed.count
+
+              success_data[time_label] = total - failed
+              failed_data[time_label] = failed
+            end
+
+            [
+              { name: "Success", data: success_data },
+              { name: "Failed", data: failed_data }
+            ]
+          end
+
+          private
+
+          def calculate_success_rate(scope)
+            total = scope.count
+            return 0.0 if total.zero?
+            (scope.successful.count.to_f / total * 100).round(2)
+          end
+
+          def calculate_error_rate(scope)
+            total = scope.count
+            return 0.0 if total.zero?
+            (scope.failed.count.to_f / total * 100).round(2)
+          end
+
+          def stats_for_scope(scope)
+            count = scope.count
+            total_cost = scope.total_cost_sum || 0
+
+            {
+              count: count,
+              total_cost: total_cost,
+              avg_cost: count > 0 ? (total_cost / count).round(6) : 0,
+              avg_tokens: scope.avg_tokens&.round || 0,
+              avg_duration_ms: scope.avg_duration&.round || 0,
+              success_rate: calculate_success_rate(scope)
+            }
+          end
+
+          def percent_change(old_value, new_value)
+            return 0.0 if old_value.nil? || old_value.zero?
+            ((new_value - old_value).to_f / old_value * 100).round(2)
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/ruby_llm/agents/execution/metrics.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    class Execution
+      # Metrics concern for cost calculations and performance metrics
+      #
+      # Provides methods for:
+      # - Calculating costs from token usage via RubyLLM pricing
+      # - Human-readable duration formatting
+      # - Performance metrics (tokens/second, cost per 1K tokens)
+      # - Formatted cost display helpers
+      #
+      module Metrics
+        extend ActiveSupport::Concern
+
+        # Calculate costs from token usage and model pricing
+        def calculate_costs!(model_info = nil)
+          return unless input_tokens && output_tokens
+
+          model_info ||= resolve_model_info
+          return unless model_info
+
+          # Get pricing from RubyLLM (prices per million tokens in dollars)
+          input_price_per_million = model_info.pricing&.text_tokens&.input || 0
+          output_price_per_million = model_info.pricing&.text_tokens&.output || 0
+
+          # Calculate costs in dollars (with 6 decimal precision for micro-dollars)
+          self.input_cost = ((input_tokens / 1_000_000.0) * input_price_per_million).round(6)
+          self.output_cost = ((output_tokens / 1_000_000.0) * output_price_per_million).round(6)
+        end
+
+        # Human-readable duration
+        def duration_seconds
+          duration_ms ? (duration_ms / 1000.0).round(2) : nil
+        end
+
+        # Tokens per second
+        def tokens_per_second
+          return nil unless duration_ms && duration_ms > 0 && total_tokens
+          (total_tokens / duration_seconds.to_f).round(2)
+        end
+
+        # Cost per 1K tokens (for comparison, in dollars)
+        def cost_per_1k_tokens
+          return nil unless total_tokens && total_tokens > 0 && total_cost
+          (total_cost / total_tokens.to_f * 1000).round(6)
+        end
+
+        # ==============================================================================
+        # Cost Display Helpers
+        # ==============================================================================
+        #
+        # Format cost as currency string
+        # Example: format_cost(0.000045) => "$0.000045"
+        #
+
+        def formatted_input_cost
+          format_cost(input_cost)
+        end
+
+        def formatted_output_cost
+          format_cost(output_cost)
+        end
+
+        def formatted_total_cost
+          format_cost(total_cost)
+        end
+
+        private
+
+        def resolve_model_info
+          return nil unless model_id
+
+          model, _provider = RubyLLM::Models.resolve(model_id)
+          model
+        rescue RubyLLM::ModelNotFoundError
+          Rails.logger.warn("[RubyLLM::Agents] Model not found for pricing: #{model_id}")
+          nil
+        end
+
+        def format_cost(cost)
+          return nil unless cost
+          format("$%.6f", cost)
+        end
+      end
+    end
+  end
+end

data/app/models/ruby_llm/agents/execution/scopes.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    class Execution
+      # Scopes concern for common query patterns
+      #
+      # Provides chainable scopes for:
+      # - Time-based filtering (today, this_week, last_n_days)
+      # - Agent-based filtering (by_agent, by_version, by_model)
+      # - Status filtering (successful, failed, errors, timeouts)
+      # - Performance filtering (expensive, slow, high_token)
+      # - JSONB parameter queries
+      # - Aggregations (total_cost_sum, avg_duration)
+      #
+      module Scopes
+        extend ActiveSupport::Concern
+
+        included do
+          # Time-based scopes
+          scope :recent, ->(limit = 100) { order(created_at: :desc).limit(limit) }
+          scope :oldest, ->(limit = 100) { order(created_at: :asc).limit(limit) }
+          scope :all_time, -> { all }  # Explicit scope for all-time queries (used by analytics)
+          scope :today, -> { where("created_at >= ?", Time.current.beginning_of_day) }
+          scope :yesterday, -> { where(created_at: 1.day.ago.beginning_of_day..1.day.ago.end_of_day) }
+          scope :this_week, -> { where("created_at >= ?", Time.current.beginning_of_week) }
+          scope :this_month, -> { where("created_at >= ?", Time.current.beginning_of_month) }
+          scope :last_n_days, ->(n) { where("created_at >= ?", n.days.ago) }
+
+          # Agent-based scopes
+          scope :by_agent, ->(agent_type) { where(agent_type: agent_type.to_s) }
+          scope :by_version, ->(version) { where(agent_version: version.to_s) }
+          scope :by_model, ->(model_id) { where(model_id: model_id.to_s) }
+
+          # Status scopes
+          scope :running, -> { where(status: "running") }
+          scope :in_progress, -> { running }  # alias
+          scope :completed, -> { where.not(status: "running") }
+          scope :successful, -> { where(status: "success") }
+          scope :failed, -> { where(status: %w[error timeout]) }
+          scope :errors, -> { where(status: "error") }
+          scope :timeouts, -> { where(status: "timeout") }
+
+          # Performance scopes
+          scope :expensive, ->(threshold_dollars = 1.00) { where("total_cost >= ?", threshold_dollars) }
+          scope :slow, ->(threshold_ms = 5000) { where("duration_ms >= ?", threshold_ms) }
+          scope :high_token, ->(threshold = 10_000) { where("total_tokens >= ?", threshold) }
+
+          # Parameter-based scopes (JSONB queries)
+          scope :with_parameter, ->(key, value = nil) do
+            if value
+              where("parameters @> ?", { key => value }.to_json)
+            else
+              where("parameters ? :key", key: key.to_s)
+            end
+          end
+
+        end
+
+        # Aggregation methods (not scopes - these return values, not relations)
+        class_methods do
+          def total_cost_sum
+            sum(:total_cost)
+          end
+
+          def total_tokens_sum
+            sum(:total_tokens)
+          end
+
+          def avg_duration
+            average(:duration_ms)
+          end
+
+          def avg_tokens
+            average(:total_tokens)
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/ruby_llm/agents/execution.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    # Execution model for tracking agent executions
+    #
+    # Stores all agent execution data including:
+    # - Agent identification (type, version)
+    # - Model configuration (model_id, temperature)
+    # - Timing (started_at, completed_at, duration_ms)
+    # - Token usage (input_tokens, output_tokens, cached_tokens)
+    # - Costs (input_cost, output_cost, total_cost in dollars)
+    # - Status (success, error, timeout)
+    # - Parameters and metadata (JSONB)
+    # - Error tracking (error_class, error_message)
+    #
+    class Execution < ::ActiveRecord::Base
+      self.table_name = "ruby_llm_agents_executions"
+
+      include Execution::Metrics
+      include Execution::Scopes
+      include Execution::Analytics
+
+      # Status enum
+      # - running: execution in progress
+      # - success: completed successfully
+      # - error: completed with error
+      # - timeout: completed due to timeout
+      enum :status, %w[running success error timeout].index_by(&:itself), prefix: true
+
+      # Validations
+      validates :agent_type, :model_id, :started_at, presence: true
+      validates :status, inclusion: { in: statuses.keys }
+      validates :agent_version, presence: true
+      validates :temperature, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 2 }, allow_nil: true
+      validates :input_tokens, :output_tokens, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true
+      validates :duration_ms, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true
+      validates :input_cost, :output_cost, :total_cost, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true
+
+      # Callbacks
+      before_save :calculate_total_tokens, if: -> { input_tokens_changed? || output_tokens_changed? }
+      before_save :calculate_total_cost, if: -> { input_cost_changed? || output_cost_changed? }
+      after_commit :broadcast_execution, on: %i[create update]
+
+      # Broadcast execution changes via ActionCable
+      def broadcast_execution
+        ActionCable.server.broadcast(
+          "ruby_llm_agents:executions",
+          {
+            action: previously_new_record? ? "created" : "updated",
+            id: id,
+            status: status,
+            html: render_execution_html
+          }
+        )
+      rescue StandardError => e
+        Rails.logger.error("[RubyLLM::Agents] Failed to broadcast execution: #{e.message}")
+      end
+
+      private
+
+      def render_execution_html
+        ApplicationController.render(
+          partial: "rubyllm/agents/dashboard/execution_item",
+          locals: { execution: self }
+        )
+      rescue StandardError
+        # Partial may not exist in all contexts
+        nil
+      end
+
+      def calculate_total_tokens
+        self.total_tokens = (input_tokens || 0) + (output_tokens || 0)
+      end
+
+      def calculate_total_cost
+        self.total_cost = (input_cost || 0) + (output_cost || 0)
+      end
+    end
+  end
+end

data/app/services/ruby_llm/agents/agent_registry.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    # Service for discovering and listing available agents
+    #
+    # Combines two sources:
+    # 1. File system - Classes inheriting from ApplicationAgent in app/agents/
+    # 2. Execution history - Agent types that have execution records
+    #
+    # This ensures all agents are visible, including:
+    # - Agents that have never been executed
+    # - Deleted agents that still have execution history
+    #
+    class AgentRegistry
+      class << self
+        # Returns all unique agent type names (sorted)
+        def all
+          (file_system_agents + execution_agents).uniq.sort
+        end
+
+        # Returns agent class if it exists, nil if only in execution history
+        def find(agent_type)
+          agent_type.safe_constantize
+        end
+
+        # Check if an agent class is currently defined
+        def exists?(agent_type)
+          find(agent_type).present?
+        end
+
+        # Get detailed info about all agents
+        def all_with_details
+          all.map do |agent_type|
+            build_agent_info(agent_type)
+          end
+        end
+
+        private
+
+        # Find agent classes defined in the file system
+        def file_system_agents
+          # Ensure all agent classes are loaded
+          eager_load_agents!
+
+          # Find all descendants of the base class
+          base_class = RubyLLM::Agents::Base
+          base_class.descendants.map(&:name).compact
+        rescue StandardError => e
+          Rails.logger.error("[RubyLLM::Agents] Error loading agents from file system: #{e.message}")
+          []
+        end
+
+        # Find agent types from execution history
+        def execution_agents
+          Execution.distinct.pluck(:agent_type).compact
+        rescue StandardError => e
+          Rails.logger.error("[RubyLLM::Agents] Error loading agents from executions: #{e.message}")
+          []
+        end
+
+        # Eager load all agent files to ensure descendants are registered
+        def eager_load_agents!
+          agents_path = Rails.root.join("app", "agents")
+          return unless agents_path.exist?
+
+          Dir.glob(agents_path.join("**", "*.rb")).each do |file|
+            require_dependency file
+          end
+        end
+
+        # Build detailed info hash for an agent
+        def build_agent_info(agent_type)
+          agent_class = find(agent_type)
+          stats = fetch_stats(agent_type)
+
+          {
+            name: agent_type,
+            class: agent_class,
+            active: agent_class.present?,
+            version: agent_class&.version || "N/A",
+            model: agent_class&.model || "N/A",
+            temperature: agent_class&.temperature,
+            timeout: agent_class&.timeout,
+            cache_enabled: agent_class&.cache_enabled? || false,
+            cache_ttl: agent_class&.cache_ttl,
+            params: agent_class&.params || {},
+            execution_count: stats[:count],
+            total_cost: stats[:total_cost],
+            total_tokens: stats[:total_tokens],
+            avg_duration_ms: stats[:avg_duration_ms],
+            success_rate: stats[:success_rate],
+            error_rate: stats[:error_rate],
+            last_executed: last_execution_time(agent_type)
+          }
+        end
+
+        def fetch_stats(agent_type)
+          Execution.stats_for(agent_type, period: :all_time)
+        rescue StandardError
+          { count: 0, total_cost: 0, total_tokens: 0, avg_duration_ms: 0, success_rate: 0, error_rate: 0 }
+        end
+
+        def last_execution_time(agent_type)
+          Execution.by_agent(agent_type).order(created_at: :desc).first&.created_at
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end