ruby_llm-agents 0.3.1 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2a8bd149077abc08185f8bc5c59d03323ba6adce25f1feed23dfc35d17376de
-  data.tar.gz: 4e9d466a76aa4565a6936a8d9cc7499b4b18aa6efb1e9c40baa0a28c35ca656d
+  metadata.gz: 26167fa0299a5be7e45f5742829e8d30b6b5eb6c96ce341af7895358897b5e88
+  data.tar.gz: ea883ffe277ac2595b2866cdc427a95876bf0fcb4f3b6f5d9678b7c1e54279cb
 SHA512:
-  metadata.gz: 3758ac407012134aab9fcf89f0ad7895b3cb38dd9b385772bb8102cea93a4d9247e9b3083e5eaa7e7f2d1969c4079d1b4286168c00353976af84effa6272ec2b
-  data.tar.gz: 3787b8665b33714ac6f4221e7a79753563d4b4553b8868a8dba2d9a1b2d3e310c43c3f27064fb7e0aa1bf9addad865b26626127d7862ed52221637c6b81eaa91
+  metadata.gz: 8d2f5c7e95d86da7c22ec97e90b205ec2b61918cf5d9c7fedcfca86275515d76409429ea4bdb5fe524dd58ae4691ef7d5602edee6233d77ad243a1e1ae9778e6
+  data.tar.gz: b32a708e9576ee8378a298febd4657cf72cccac481e5b31e929ef160c71be45a32a02093be06d7d62403cc03f08bb48465c8a217cd3f808f406149fa4ba6a92a

data/README.md

@@ -16,6 +16,7 @@ A powerful Rails engine for building, managing, and monitoring LLM-powered agent
 - **🎯 Type Safety** - Structured output with RubyLLM::Schema integration
 - **⚡ Real-time Streaming** - Stream LLM responses with time-to-first-token tracking
 - **📎 Attachments** - Send images, PDFs, and files to vision-capable models
+- **📋 Rich Results** - Access token counts, costs, timing, and model info from every execution
 - **🔄 Reliability** - Automatic retries, model fallbacks, and circuit breakers for resilient agents
 - **💵 Budget Controls** - Daily/monthly spending limits with hard and soft enforcement
 - **🔔 Alerts** - Slack, webhook, and custom notifications for budget and circuit breaker events
@@ -271,6 +272,93 @@ VisionAgent.call(question: "test", with: "image.png", dry_run: true)
 # => { ..., attachments: "image.png", ... }
 ```
 
+### Execution Results
+
+Every agent call returns a `Result` object with full execution metadata:
+
+```ruby
+result = SearchAgent.call(query: "red dress")
+
+# Access the processed response
+result.content            # => { refined_query: "red dress", ... }
+
+# Token usage
+result.input_tokens       # => 150
+result.output_tokens      # => 50
+result.total_tokens       # => 200
+result.cached_tokens      # => 0
+
+# Cost calculation
+result.input_cost         # => 0.000150
+result.output_cost        # => 0.000100
+result.total_cost         # => 0.000250
+
+# Model info
+result.model_id           # => "gpt-4o"
+result.chosen_model_id    # => "gpt-4o" (may differ if fallback used)
+result.temperature        # => 0.0
+
+# Timing
+result.duration_ms        # => 1234
+result.started_at         # => 2025-11-27 10:30:00 UTC
+result.completed_at       # => 2025-11-27 10:30:01 UTC
+result.time_to_first_token_ms # => 245 (streaming only)
+
+# Status
+result.finish_reason      # => "stop", "length", "tool_calls", etc.
+result.streaming?         # => false
+result.success?           # => true
+result.truncated?         # => false (true if hit max_tokens)
+
+# Tool calls (for agents with tools)
+result.tool_calls         # => [{ "id" => "call_abc", "name" => "search", "arguments" => {...} }]
+result.tool_calls_count   # => 1
+result.has_tool_calls?    # => true
+
+# Reliability info
+result.attempts_count     # => 1
+result.used_fallback?     # => false
+```
+
+#### Backward Compatibility
+
+The Result object delegates hash methods to content, so existing code continues to work:
+
+```ruby
+# Old style (still works)
+result[:refined_query]
+result.dig(:nested, :key)
+
+# New style (access metadata)
+result.content[:refined_query]
+result.total_cost
+```
+
+#### Full Metadata Hash
+
+```ruby
+result.to_h
+# => {
+#   content: { refined_query: "red dress", ... },
+#   input_tokens: 150,
+#   output_tokens: 50,
+#   total_tokens: 200,
+#   cached_tokens: 0,
+#   input_cost: 0.000150,
+#   output_cost: 0.000100,
+#   total_cost: 0.000250,
+#   model_id: "gpt-4o",
+#   chosen_model_id: "gpt-4o",
+#   temperature: 0.0,
+#   duration_ms: 1234,
+#   finish_reason: "stop",
+#   streaming: false,
+#   tool_calls: [...],
+#   tool_calls_count: 0,
+#   ...
+# }
+```
+
 ## Usage Guide
 
 ### Agent DSL

data/app/controllers/ruby_llm/agents/dashboard_controller.rb

@@ -14,25 +14,89 @@ module RubyLLM
       # Renders the main dashboard view
       #
       # Loads now strip data, critical alerts, hourly activity,
-      # and recent executions for real-time monitoring.
+      # recent executions, agent comparison, and top errors.
       #
       # @return [void]
       def index
-        @now_strip = Execution.now_strip_data
+        @selected_range = params[:range].presence || "today"
+        @days = range_to_days(@selected_range)
+        @now_strip = Execution.now_strip_data(range: @selected_range)
         @critical_alerts = load_critical_alerts
         @hourly_activity = Execution.hourly_activity_chart
         @recent_executions = Execution.recent(10)
+        @agent_stats = build_agent_comparison
+        @top_errors = build_top_errors
       end
 
       # Returns chart data as JSON for live updates
       #
-      # @return [JSON] Chart data with categories and series
+      # @param range [String] Time range: "today", "7d", or "30d"
+      # @return [JSON] Chart data with series
       def chart_data
-        render json: Execution.hourly_activity_chart_json
+        range = params[:range].presence || "today"
+        render json: Execution.activity_chart_json(range: range)
       end
 
       private
 
+      # Converts range parameter to number of days
+      #
+      # @param range [String] Range parameter (today, 7d, 30d)
+      # @return [Integer] Number of days
+      def range_to_days(range)
+        case range
+        when "today" then 1
+        when "7d" then 7
+        when "30d" then 30
+        else 1
+        end
+      end
+
+      # Builds per-agent comparison statistics
+      #
+      # @return [Array<Hash>] Array of agent stats sorted by cost descending
+      def build_agent_comparison
+        scope = Execution.last_n_days(@days)
+        agent_types = scope.distinct.pluck(:agent_type)
+
+        agent_types.map do |agent_type|
+          agent_scope = scope.where(agent_type: agent_type)
+          count = agent_scope.count
+          total_cost = agent_scope.sum(:total_cost) || 0
+          successful = agent_scope.successful.count
+
+          {
+            agent_type: agent_type,
+            executions: count,
+            total_cost: total_cost,
+            avg_cost: count > 0 ? (total_cost / count).round(6) : 0,
+            avg_duration_ms: agent_scope.average(:duration_ms)&.round || 0,
+            success_rate: count > 0 ? (successful.to_f / count * 100).round(1) : 0
+          }
+        end.sort_by { |a| -(a[:total_cost] || 0) }
+      end
+
+      # Builds top errors list
+      #
+      # @return [Array<Hash>] Top 5 error classes with counts
+      def build_top_errors
+        scope = Execution.last_n_days(@days).where(status: "error")
+        total_errors = scope.count
+
+        scope.group(:error_class)
+             .select("error_class, COUNT(*) as count, MAX(created_at) as last_seen")
+             .order("count DESC")
+             .limit(5)
+             .map do |row|
+          {
+            error_class: row.error_class || "Unknown Error",
+            count: row.count,
+            percentage: total_errors > 0 ? (row.count.to_f / total_errors * 100).round(1) : 0,
+            last_seen: row.last_seen
+          }
+        end
+      end
+
       # Fetches cached daily statistics for the dashboard
       #
       # Results are cached for 1 minute to reduce database load while

data/app/models/ruby_llm/agents/execution/analytics.rb

@@ -167,35 +167,136 @@ module RubyLLM
           end
 
           # Returns chart data as arrays for Highcharts live updates
-          # Format: { categories: [...], series: [...] }
+          # Format: { categories: [...], series: [...], range: ... }
+          #
+          # @param range [String] Time range: "today" (hourly), "7d" or "30d" (daily)
+          def activity_chart_json(range: "today")
+            case range
+            when "7d"
+              build_daily_chart_data(7)
+            when "30d"
+              build_daily_chart_data(30)
+            else
+              build_hourly_chart_data
+            end
+          end
+
+          # Alias for backwards compatibility
           def hourly_activity_chart_json
-            # Always use current time as reference so chart shows "now" on the right
-            reference_time = Time.current.beginning_of_hour
+            activity_chart_json(range: "today")
+          end
 
-            categories = []
+          private
+
+          # Builds hourly chart data for last 24 hours
+          # Optimized: Single GROUP BY query instead of 72 individual queries
+          def build_hourly_chart_data
+            reference_time = Time.current.beginning_of_hour
+            start_time = reference_time - 23.hours
+
+            # Single query with GROUP BY - reduces 72 queries to 1
+            results = where(created_at: start_time..(reference_time + 1.hour))
+              .group(Arel.sql("DATE_TRUNC('hour', created_at)"))
+              .select(
+                Arel.sql("DATE_TRUNC('hour', created_at) as time_bucket"),
+                Arel.sql("COUNT(*) FILTER (WHERE status = 'success') as success_count"),
+                Arel.sql("COUNT(*) FILTER (WHERE status IN ('error', 'timeout')) as failed_count"),
+                Arel.sql("COALESCE(SUM(total_cost), 0) as total_cost")
+              )
+              .index_by { |r| r.time_bucket.to_time.beginning_of_hour }
+
+            # Build arrays for all 24 hours (fill missing with zeros)
             success_data = []
             failed_data = []
+            cost_data = []
+            total_success = 0
+            total_failed = 0
+            total_cost = 0.0
 
-            # Create entries for the last 24 hours ending at current hour
             (23.downto(0)).each do |hours_ago|
-              start_time = reference_time - hours_ago.hours
-              end_time = start_time + 1.hour
-              categories << start_time.in_time_zone.strftime("%H:%M")
+              bucket_time = (reference_time - hours_ago.hours).beginning_of_hour
+              row = results[bucket_time]
 
-              hour_scope = where(created_at: start_time...end_time)
-              success_data << hour_scope.successful.count
-              failed_data << hour_scope.failed.count
+              s = row&.success_count.to_i
+              f = row&.failed_count.to_i
+              c = row&.total_cost.to_f
+
+              success_data << s
+              failed_data << f
+              cost_data << c.round(4)
+
+              total_success += s
+              total_failed += f
+              total_cost += c
             end
 
             {
-              categories: categories,
+              range: "today",
+              totals: { success: total_success, failed: total_failed, cost: total_cost.round(4) },
               series: [
                 { name: "Success", data: success_data },
-                { name: "Failed", data: failed_data }
+                { name: "Failed", data: failed_data },
+                { name: "Cost", data: cost_data }
               ]
             }
           end
 
+          # Builds daily chart data for specified number of days
+          # Optimized: Single GROUP BY query instead of 3*days individual queries
+          def build_daily_chart_data(days)
+            end_date = Date.current
+            start_date = (days - 1).days.ago.to_date
+
+            # Single query with GROUP BY - reduces 3*days queries to 1
+            results = where(created_at: start_date.beginning_of_day..end_date.end_of_day)
+              .group(Arel.sql("DATE_TRUNC('day', created_at)"))
+              .select(
+                Arel.sql("DATE_TRUNC('day', created_at) as time_bucket"),
+                Arel.sql("COUNT(*) FILTER (WHERE status = 'success') as success_count"),
+                Arel.sql("COUNT(*) FILTER (WHERE status IN ('error', 'timeout')) as failed_count"),
+                Arel.sql("COALESCE(SUM(total_cost), 0) as total_cost")
+              )
+              .index_by { |r| r.time_bucket.to_date }
+
+            # Build arrays for all days (fill missing with zeros)
+            success_data = []
+            failed_data = []
+            cost_data = []
+            total_success = 0
+            total_failed = 0
+            total_cost = 0.0
+
+            (days - 1).downto(0).each do |days_ago|
+              date = days_ago.days.ago.to_date
+              row = results[date]
+
+              s = row&.success_count.to_i
+              f = row&.failed_count.to_i
+              c = row&.total_cost.to_f
+
+              success_data << s
+              failed_data << f
+              cost_data << c.round(4)
+
+              total_success += s
+              total_failed += f
+              total_cost += c
+            end
+
+            {
+              range: "#{days}d",
+              days: days,
+              totals: { success: total_success, failed: total_failed, cost: total_cost.round(4) },
+              series: [
+                { name: "Success", data: success_data },
+                { name: "Failed", data: failed_data },
+                { name: "Cost", data: cost_data }
+              ]
+            }
+          end
+
+          public
+
           # Builds the hourly activity data structure
           # Shows the last 24 hours with current hour on the right
           #

data/app/models/ruby_llm/agents/execution/scopes.rb

@@ -256,6 +256,16 @@ module RubyLLM
           scope :content_filtered, -> { where(finish_reason: "content_filter") }
           scope :tool_calls, -> { where(finish_reason: "tool_calls") }
 
+          # @!method with_tool_calls
+          #   Returns executions that made tool calls
+          #   @return [ActiveRecord::Relation]
+
+          # @!method without_tool_calls
+          #   Returns executions that did not make tool calls
+          #   @return [ActiveRecord::Relation]
+          scope :with_tool_calls, -> { where("tool_calls_count > 0") }
+          scope :without_tool_calls, -> { where(tool_calls_count: 0) }
+
           # @!endgroup
         end
 

data/app/models/ruby_llm/agents/execution.rb

@@ -85,7 +85,6 @@ module RubyLLM
 
       before_save :calculate_total_tokens, if: -> { input_tokens_changed? || output_tokens_changed? }
       before_save :calculate_total_cost, if: -> { input_cost_changed? || output_cost_changed? }
-      after_commit :broadcast_turbo_streams, on: %i[create update]
 
       # Aggregates costs from all attempts using each attempt's model pricing
       #
@@ -219,79 +218,48 @@ module RubyLLM
         finish_reason == "content_filter"
       end
 
+      # Returns whether this execution made tool calls
+      #
+      # @return [Boolean] true if tool calls were made
+      def has_tool_calls?
+        tool_calls_count.to_i > 0
+      end
+
       # Returns real-time dashboard data for the Now Strip
       #
+      # @param range [String] Time range: "today", "7d", or "30d"
       # @return [Hash] Now strip metrics
-      def self.now_strip_data
-        today_scope = today
+      def self.now_strip_data(range: "today")
+        scope = case range
+                when "7d" then last_n_days(7)
+                when "30d" then last_n_days(30)
+                else today
+                end
+
         {
           running: running.count,
-          success_today: today_scope.status_success.count,
-          errors_today: today_scope.status_error.count,
-          timeouts_today: today_scope.status_timeout.count,
-          cost_today: today_scope.sum(:total_cost) || 0,
-          executions_today: today_scope.count,
-          success_rate: calculate_today_success_rate
+          success_today: scope.status_success.count,
+          errors_today: scope.status_error.count,
+          timeouts_today: scope.status_timeout.count,
+          cost_today: scope.sum(:total_cost) || 0,
+          executions_today: scope.count,
+          success_rate: calculate_period_success_rate(scope)
         }
       end
 
-      # Calculates today's success rate
+      # Calculates success rate for a given scope
       #
+      # @param scope [ActiveRecord::Relation] The execution scope
       # @return [Float] Success rate as percentage
-      def self.calculate_today_success_rate
-        total = today.count
+      def self.calculate_period_success_rate(scope)
+        total = scope.count
         return 0.0 if total.zero?
 
-        (today.successful.count.to_f / total * 100).round(1)
-      end
-
-      # Broadcasts execution changes via ActionCable for real-time dashboard updates
-      #
-      # Sends JSON with action, id, status, and rendered HTML partials.
-      # The JavaScript client handles DOM updates based on the action type.
-      #
-      # @return [void]
-      def broadcast_turbo_streams
-        ActionCable.server.broadcast(
-          "ruby_llm_agents:executions",
-          {
-            action: previously_new_record? ? "created" : "updated",
-            id: id,
-            status: status,
-            html: render_execution_html,
-            now_strip_html: render_now_strip_html
-          }
-        )
-      rescue StandardError => e
-        Rails.logger.error("[RubyLLM::Agents] Failed to broadcast execution: #{e.message}")
+        (scope.successful.count.to_f / total * 100).round(1)
       end
 
       private
 
-      # Renders the execution item partial for broadcast
-      #
-      # @return [String, nil] HTML string or nil if rendering fails
-      def render_execution_html
-        ApplicationController.render(
-          partial: "rubyllm/agents/dashboard/execution_item",
-          locals: { execution: self }
-        )
-      rescue StandardError
-        nil
-      end
-
-      # Renders the Now Strip values partial for broadcast
-      #
-      # @return [String, nil] HTML string or nil if rendering fails
-      def render_now_strip_html
-        ApplicationController.render(
-          partial: "rubyllm/agents/dashboard/now_strip_values",
-          locals: { now_strip: self.class.now_strip_data }
-        )
-      rescue StandardError
-        nil
-      end
-
       # Calculates and sets total_tokens from input and output
       #
       # @return [Integer] The calculated total