ruby_llm-agents 0.2.4 → 0.3.1

data/app/models/ruby_llm/agents/execution/analytics.rb

@@ -5,18 +5,28 @@ module RubyLLM
     class Execution
       # Analytics concern for advanced reporting and analysis
       #
-      # Provides class methods for:
-      # - Daily reports with key metrics
-      # - Cost breakdown by agent type
-      # - Performance stats for specific agents
-      # - Version comparison
-      # - Trend analysis over time
+      # Provides class methods for generating reports, analyzing trends,
+      # comparing versions, and building chart data.
       #
+      # @see RubyLLM::Agents::Execution::Scopes
+      # @api public
       module Analytics
         extend ActiveSupport::Concern
 
         class_methods do
-          # Daily report with key metrics
+          # Generates a daily report with key metrics for today
+          #
+          # @return [Hash] Report containing:
+          #   - :date [Date] Current date
+          #   - :total_executions [Integer] Total execution count
+          #   - :successful [Integer] Successful execution count
+          #   - :failed [Integer] Failed execution count
+          #   - :total_cost [Float] Sum of all costs
+          #   - :total_tokens [Integer] Sum of all tokens
+          #   - :avg_duration_ms [Integer] Average duration
+          #   - :error_rate [Float] Percentage of failures
+          #   - :by_agent [Hash] Counts grouped by agent type
+          #   - :top_errors [Hash] Top 5 error classes
           def daily_report
             scope = today
 
@@ -35,7 +45,10 @@ module RubyLLM
             }
           end
 
-          # Cost breakdown by agent type
+          # Returns cost breakdown grouped by agent type
+          #
+          # @param period [Symbol] Time scope (:today, :this_week, :this_month, :all_time)
+          # @return [Hash{String => Float}] Agent types mapped to total cost, sorted descending
           def cost_by_agent(period: :today)
             public_send(period)
               .group(:agent_type)
@@ -44,7 +57,11 @@ module RubyLLM
               .to_h
           end
 
-          # Performance stats for specific agent
+          # Returns performance statistics for a specific agent
+          #
+          # @param agent_type [String] The agent class name
+          # @param period [Symbol] Time scope (:today, :this_week, :this_month, :all_time)
+          # @return [Hash] Statistics including count, costs, tokens, duration, rates
           def stats_for(agent_type, period: :today)
             scope = by_agent(agent_type).public_send(period)
             count = scope.count
@@ -64,7 +81,13 @@ module RubyLLM
             }
           end
 
-          # Compare versions of the same agent
+          # Compares performance between two agent versions
+          #
+          # @param agent_type [String] The agent class name
+          # @param version1 [String] First version to compare (baseline)
+          # @param version2 [String] Second version to compare
+          # @param period [Symbol] Time scope for comparison
+          # @return [Hash] Comparison data with stats for each version and improvement percentages
           def compare_versions(agent_type, version1, version2, period: :this_week)
             base_scope = by_agent(agent_type).public_send(period)
 
@@ -84,7 +107,38 @@ module RubyLLM
             }
           end
 
-          # Trend analysis over time
+          # Returns daily trend data for a specific agent version
+          #
+          # Used for sparkline charts in version comparison.
+          #
+          # @param agent_type [String] The agent class name
+          # @param version [String] The version to analyze
+          # @param days [Integer] Number of days to analyze
+          # @return [Array<Hash>] Daily metrics sorted oldest to newest
+          def version_trend_data(agent_type, version, days: 14)
+            scope = by_agent(agent_type).by_version(version)
+
+            (0...days).map do |days_ago|
+              date = days_ago.days.ago.to_date
+              day_scope = scope.where(created_at: date.beginning_of_day..date.end_of_day)
+              count = day_scope.count
+
+              {
+                date: date,
+                count: count,
+                success_rate: calculate_success_rate(day_scope),
+                avg_cost: count > 0 ? ((day_scope.total_cost_sum || 0) / count).round(6) : 0,
+                avg_duration_ms: day_scope.avg_duration&.round || 0,
+                avg_tokens: day_scope.avg_tokens&.round || 0
+              }
+            end.reverse
+          end
+
+          # Analyzes trends over a time period
+          #
+          # @param agent_type [String, nil] Filter to specific agent, or nil for all
+          # @param days [Integer] Number of days to analyze
+          # @return [Array<Hash>] Daily metrics sorted oldest to newest
           def trend_analysis(agent_type: nil, days: 7)
             scope = agent_type ? by_agent(agent_type) : all
 
@@ -102,11 +156,96 @@ module RubyLLM
             end.reverse
           end
 
-          # Chart data: Hourly activity chart for today showing success/failed
+          # Builds hourly activity chart data for today
+          #
+          # Cached for 5 minutes to reduce database load.
+          #
+          # @return [Array<Hash>] Chart series with success and failed counts per hour
           def hourly_activity_chart
+            # No caching - always fresh data based on latest execution
+            build_hourly_activity_data
+          end
+
+          # Returns chart data as arrays for Highcharts live updates
+          # Format: { categories: [...], series: [...] }
+          def hourly_activity_chart_json
+            # Always use current time as reference so chart shows "now" on the right
+            reference_time = Time.current.beginning_of_hour
+
+            categories = []
+            success_data = []
+            failed_data = []
+
+            # Create entries for the last 24 hours ending at current hour
+            (23.downto(0)).each do |hours_ago|
+              start_time = reference_time - hours_ago.hours
+              end_time = start_time + 1.hour
+              categories << start_time.in_time_zone.strftime("%H:%M")
+
+              hour_scope = where(created_at: start_time...end_time)
+              success_data << hour_scope.successful.count
+              failed_data << hour_scope.failed.count
+            end
+
+            {
+              categories: categories,
+              series: [
+                { name: "Success", data: success_data },
+                { name: "Failed", data: failed_data }
+              ]
+            }
+          end
+
+          # Builds the hourly activity data structure
+          # Shows the last 24 hours with current hour on the right
+          #
+          # @return [Array<Hash>] Success and failed series data
+          # @api private
+          def build_hourly_activity_data
             success_data = {}
             failed_data = {}
 
+            # Use current time as reference so chart shows "now" on the right
+            reference_time = Time.current.beginning_of_hour
+
+            # Create entries for the last 24 hours ending at current hour
+            (23.downto(0)).each do |hours_ago|
+              start_time = reference_time - hours_ago.hours
+              end_time = start_time + 1.hour
+              time_label = start_time.in_time_zone.strftime("%H:%M")
+
+              hour_scope = where(created_at: start_time...end_time)
+              success_data[time_label] = hour_scope.successful.count
+              failed_data[time_label] = hour_scope.failed.count
+            end
+
+            [
+              { name: "Success", data: success_data },
+              { name: "Failed", data: failed_data }
+            ]
+          end
+
+          # Retrieves hourly cost data for chart display
+          #
+          # Returns two series (input cost and output cost) with hourly breakdowns
+          # for the current day. Results are cached for 5 minutes.
+          #
+          # @return [Array<Hash>] Chart series with input and output cost per hour
+          def hourly_cost_chart
+            cache_key = "ruby_llm_agents/hourly_cost/#{Date.current}"
+            Rails.cache.fetch(cache_key, expires_in: 5.minutes) do
+              build_hourly_cost_data
+            end
+          end
+
+          # Builds the hourly cost data structure (uncached)
+          #
+          # @return [Array<Hash>] Input and output cost series data
+          # @api private
+          def build_hourly_cost_data
+            input_cost_data = {}
+            output_cost_data = {}
+
             # Create entries for each hour of the day (0-23)
             (0..23).each do |hour|
               time_label = format("%02d:00", hour)
@@ -114,33 +253,93 @@ module RubyLLM
               end_time = start_time + 1.hour
 
               hour_scope = where(created_at: start_time...end_time)
-              total = hour_scope.count
-              failed = hour_scope.failed.count
-
-              success_data[time_label] = total - failed
-              failed_data[time_label] = failed
+              input_cost_data[time_label] = (hour_scope.sum(:input_cost) || 0).round(6)
+              output_cost_data[time_label] = (hour_scope.sum(:output_cost) || 0).round(6)
             end
 
             [
-              { name: "Success", data: success_data },
-              { name: "Failed", data: failed_data }
+              { name: "Input Cost", data: input_cost_data },
+              { name: "Output Cost", data: output_cost_data }
             ]
           end
 
-          private
+          # Cache hit rate percentage
+          #
+          # @return [Float] Percentage of executions that were cache hits (0.0-100.0)
+          def cache_hit_rate
+            total = count
+            return 0.0 if total.zero?
+
+            (cached.count.to_f / total * 100).round(1)
+          end
+
+          # Streaming execution rate percentage
+          #
+          # @return [Float] Percentage of executions that used streaming (0.0-100.0)
+          def streaming_rate
+            total = count
+            return 0.0 if total.zero?
+
+            (streaming.count.to_f / total * 100).round(1)
+          end
+
+          # Average time to first token for streaming executions
+          #
+          # @return [Integer, nil] Average TTFT in milliseconds, or nil if no data
+          def avg_time_to_first_token
+            streaming.where.not(time_to_first_token_ms: nil).average(:time_to_first_token_ms)&.round(0)
+          end
+
+          # Finish reason distribution
+          #
+          # @return [Hash{String => Integer}] Counts grouped by finish reason, sorted descending
+          def finish_reason_distribution
+            group(:finish_reason).count.sort_by { |_, v| -v }.to_h
+          end
+
+          # Rate limited execution count
+          #
+          # @return [Integer] Number of executions that were rate limited
+          def rate_limited_count
+            where(rate_limited: true).count
+          end
+
+          # Rate limited rate percentage
+          #
+          # @return [Float] Percentage of executions that were rate limited (0.0-100.0)
+          def rate_limited_rate
+            total = count
+            return 0.0 if total.zero?
+
+            (rate_limited_count.to_f / total * 100).round(1)
+          end
+
+        private
 
+          # Calculates success rate percentage for a scope
+          #
+          # @param scope [ActiveRecord::Relation] The scope to calculate from
+          # @return [Float] Success rate as percentage (0.0-100.0)
           def calculate_success_rate(scope)
             total = scope.count
             return 0.0 if total.zero?
             (scope.successful.count.to_f / total * 100).round(2)
           end
 
+          # Calculates error rate percentage for a scope
+          #
+          # @param scope [ActiveRecord::Relation] The scope to calculate from
+          # @return [Float] Error rate as percentage (0.0-100.0)
           def calculate_error_rate(scope)
             total = scope.count
             return 0.0 if total.zero?
             (scope.failed.count.to_f / total * 100).round(2)
           end
 
+          # Calculates statistics for an arbitrary scope
+          #
+          # @param scope [ActiveRecord::Relation] The scope to analyze
+          # @return [Hash] Statistics hash
           def stats_for_scope(scope)
             count = scope.count
             total_cost = scope.total_cost_sum || 0
@@ -155,6 +354,11 @@ module RubyLLM
             }
           end
 
+          # Calculates percentage change between two values
+          #
+          # @param old_value [Numeric, nil] Baseline value
+          # @param new_value [Numeric] New value
+          # @return [Float] Percentage change (negative = improvement for costs/duration)
           def percent_change(old_value, new_value)
             return 0.0 if old_value.nil? || old_value.zero?
             ((new_value - old_value).to_f / old_value * 100).round(2)

data/app/models/ruby_llm/agents/execution/metrics.rb

@@ -5,16 +5,22 @@ module RubyLLM
     class Execution
       # Metrics concern for cost calculations and performance metrics
       #
-      # Provides methods for:
-      # - Calculating costs from token usage via RubyLLM pricing
-      # - Human-readable duration formatting
-      # - Performance metrics (tokens/second, cost per 1K tokens)
-      # - Formatted cost display helpers
+      # Provides instance methods for calculating costs from token usage,
+      # formatting durations, and computing performance metrics.
       #
+      # @see RubyLLM::Agents::Execution::Analytics
+      # @api public
       module Metrics
         extend ActiveSupport::Concern
 
-        # Calculate costs from token usage and model pricing
+        # Calculates and sets input/output costs from token usage
+        #
+        # Uses RubyLLM's built-in pricing data to calculate costs.
+        # Sets input_cost and output_cost attributes (total_cost is calculated by callback).
+        #
+        # @param model_info [RubyLLM::Model, nil] Optional pre-resolved model info
+        # @return [void]
+        # @note Requires input_tokens and output_tokens to be set
         def calculate_costs!(model_info = nil)
           return unless input_tokens && output_tokens
 
@@ -30,55 +36,65 @@ module RubyLLM
           self.output_cost = ((output_tokens / 1_000_000.0) * output_price_per_million).round(6)
         end
 
-        # Human-readable duration
+        # Returns execution duration in seconds
+        #
+        # @return [Float, nil] Duration in seconds with 2 decimal places, or nil
         def duration_seconds
           duration_ms ? (duration_ms / 1000.0).round(2) : nil
         end
 
-        # Tokens per second
+        # Calculates throughput as tokens processed per second
+        #
+        # @return [Float, nil] Tokens per second, or nil if data unavailable
         def tokens_per_second
           return nil unless duration_ms && duration_ms > 0 && total_tokens
           (total_tokens / duration_seconds.to_f).round(2)
         end
 
-        # Cost per 1K tokens (for comparison, in dollars)
+        # Calculates cost efficiency as cost per 1,000 tokens
+        #
+        # Useful for comparing cost efficiency across different models.
+        #
+        # @return [Float, nil] Cost per 1K tokens in USD, or nil if data unavailable
         def cost_per_1k_tokens
           return nil unless total_tokens && total_tokens > 0 && total_cost
           (total_cost / total_tokens.to_f * 1000).round(6)
         end
 
-        # ==============================================================================
-        # Cost Display Helpers
-        # ==============================================================================
-        #
-        # Format cost as currency string
-        # Example: format_cost(0.000045) => "$0.000045"
-        #
+        # @!group Cost Display Helpers
 
+        # Returns input_cost formatted as currency
+        #
+        # @return [String, nil] Formatted cost (e.g., "$0.000045") or nil
         def formatted_input_cost
           format_cost(input_cost)
         end
 
+        # Returns output_cost formatted as currency
+        #
+        # @return [String, nil] Formatted cost (e.g., "$0.000045") or nil
         def formatted_output_cost
           format_cost(output_cost)
         end
 
+        # Returns total_cost formatted as currency
+        #
+        # @return [String, nil] Formatted cost (e.g., "$0.000045") or nil
         def formatted_total_cost
           format_cost(total_cost)
         end
 
-        private
+        # @!endgroup
 
-        def resolve_model_info
-          return nil unless model_id
+        private
 
-          model, _provider = RubyLLM::Models.resolve(model_id)
-          model
-        rescue RubyLLM::ModelNotFoundError
-          Rails.logger.warn("[RubyLLM::Agents] Model not found for pricing: #{model_id}")
-          nil
-        end
+        # NOTE: resolve_model_info is defined in Execution class (execution.rb)
+        # It accepts an optional model_id parameter, defaulting to self.model_id
 
+        # Formats a cost value as currency string
+        #
+        # @param cost [Float, nil] Cost in USD
+        # @return [String, nil] Formatted string or nil
         def format_cost(cost)
           return nil unless cost
           format("$%.6f", cost)