ruby_llm-agents 3.15.1 → 3.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/app/controllers/concerns/ruby_llm/agents/paginatable.rb +29 -8
  3. data/app/controllers/ruby_llm/agents/agents_controller.rb +48 -44
  4. data/app/controllers/ruby_llm/agents/analytics_controller.rb +38 -72
  5. data/app/controllers/ruby_llm/agents/dashboard_controller.rb +69 -56
  6. data/app/controllers/ruby_llm/agents/executions_controller.rb +25 -41
  7. data/app/controllers/ruby_llm/agents/requests_controller.rb +117 -45
  8. data/app/controllers/ruby_llm/agents/tenants_controller.rb +67 -47
  9. data/app/models/ruby_llm/agents/execution/analytics.rb +174 -133
  10. data/app/models/ruby_llm/agents/execution/breakdown.rb +243 -0
  11. data/app/models/ruby_llm/agents/execution/scopes.rb +39 -3
  12. data/app/models/ruby_llm/agents/execution.rb +23 -5
  13. data/app/services/ruby_llm/agents/agent_registry.rb +75 -12
  14. data/app/views/ruby_llm/agents/agents/index.html.erb +24 -7
  15. data/app/views/ruby_llm/agents/agents/show.html.erb +3 -2
  16. data/app/views/ruby_llm/agents/executions/_filters.html.erb +12 -7
  17. data/app/views/ruby_llm/agents/executions/_list.html.erb +1 -52
  18. data/app/views/ruby_llm/agents/requests/index.html.erb +8 -57
  19. data/app/views/ruby_llm/agents/shared/_executions_table.html.erb +1 -52
  20. data/app/views/ruby_llm/agents/shared/_pagination.html.erb +66 -0
  21. data/app/views/ruby_llm/agents/shared/query_timeout.html.erb +18 -0
  22. data/app/views/ruby_llm/agents/system_config/show.html.erb +4 -0
  23. data/app/views/ruby_llm/agents/tenants/index.html.erb +1 -1
  24. data/app/views/ruby_llm/agents/tenants/show.html.erb +6 -5
  25. data/config/routes.rb +2 -2
  26. data/lib/generators/ruby_llm_agents/templates/add_executions_analytics_index_migration.rb.tt +50 -0
  27. data/lib/generators/ruby_llm_agents/templates/add_root_execution_list_index_migration.rb.tt +16 -0
  28. data/lib/generators/ruby_llm_agents/templates/initializer.rb.tt +5 -0
  29. data/lib/generators/ruby_llm_agents/templates/migration.rb.tt +26 -0
  30. data/lib/generators/ruby_llm_agents/upgrade_generator.rb +44 -0
  31. data/lib/ruby_llm/agents/core/configuration.rb +20 -0
  32. data/lib/ruby_llm/agents/core/version.rb +1 -1
  33. data/lib/ruby_llm/agents/pipeline/middleware/instrumentation.rb +39 -1
  34. data/lib/ruby_llm/agents/rails/engine.rb +87 -7
  35. metadata +7 -2
@@ -78,24 +78,142 @@ module RubyLLM
78
78
  # @param period [Symbol] Time scope (:today, :this_week, :this_month, :all_time)
79
79
  # @return [Hash] Statistics including count, costs, tokens, duration, rates
80
80
  def stats_for(agent_type, period: :today)
81
- scope = by_agent(agent_type).public_send(period)
82
- count = scope.count
83
- total_cost = scope.total_cost_sum || 0
81
+ {agent_type: agent_type, period: period}
82
+ .merge(by_agent(agent_type).public_send(period).usage_summary)
83
+ end
84
+
85
+ # Usage and reliability figures for the current scope, in one query
86
+ #
87
+ # @example An agent's last 30 days
88
+ # Execution.by_agent("SearchAgent").last_n_days(30).usage_summary
89
+ #
90
+ # @return [Hash] :count, :total_cost, :avg_cost, :total_tokens,
91
+ # :avg_tokens, :avg_duration_ms, :success_rate, :error_rate,
92
+ # :cache_hit_rate, :streaming_rate
93
+ def usage_summary
94
+ count, cost, tokens, avg_tok, avg_dur, successful, failed, cache_hits, streamed = pick(
95
+ Arel.sql("COUNT(*)"),
96
+ Arel.sql("COALESCE(SUM(total_cost), 0)"),
97
+ Arel.sql("COALESCE(SUM(total_tokens), 0)"),
98
+ Arel.sql("AVG(total_tokens)"),
99
+ Arel.sql("AVG(duration_ms)"),
100
+ Arel.sql("SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END)"),
101
+ Arel.sql("SUM(CASE WHEN status IN ('error', 'timeout') THEN 1 ELSE 0 END)"),
102
+ Arel.sql("SUM(CASE WHEN #{true_condition("cache_hit")} THEN 1 ELSE 0 END)"),
103
+ Arel.sql("SUM(CASE WHEN #{true_condition("streaming")} THEN 1 ELSE 0 END)")
104
+ )
105
+
106
+ count = count.to_i
107
+ total_cost = (cost || 0).to_d.round(6)
108
+ percent = ->(part, digits) { (count > 0) ? (part.to_f / count * 100).round(digits) : 0.0 }
84
109
 
85
110
  {
86
- agent_type: agent_type,
87
- period: period,
88
111
  count: count,
89
112
  total_cost: total_cost,
90
113
  avg_cost: (count > 0) ? (total_cost / count).round(6) : 0,
91
- total_tokens: scope.total_tokens_sum || 0,
92
- avg_tokens: scope.avg_tokens&.round || 0,
93
- avg_duration_ms: scope.avg_duration&.round || 0,
94
- success_rate: calculate_success_rate(scope),
95
- error_rate: calculate_error_rate(scope)
114
+ total_tokens: tokens.to_i,
115
+ avg_tokens: avg_tok&.round || 0,
116
+ avg_duration_ms: avg_dur&.round || 0,
117
+ success_rate: percent.call(successful, 2),
118
+ error_rate: percent.call(failed, 2),
119
+ cache_hit_rate: percent.call(cache_hits, 1),
120
+ streaming_rate: percent.call(streamed, 1)
96
121
  }
97
122
  end
98
123
 
124
+ # Count, cost and token totals for the current scope in one query
125
+ #
126
+ # @return [Hash] :total_count, :total_cost (BigDecimal), :total_tokens
127
+ def totals
128
+ count, cost, tokens = pick(
129
+ Arel.sql("COUNT(*)"),
130
+ Arel.sql("COALESCE(SUM(total_cost), 0)"),
131
+ Arel.sql("COALESCE(SUM(total_tokens), 0)")
132
+ )
133
+
134
+ {total_count: count.to_i, total_cost: (cost || 0).to_d.round(6), total_tokens: tokens.to_i}
135
+ end
136
+
137
+ # Everything the dashboard shows about the current scope, from one scan
138
+ #
139
+ # Reads the scope once, grouped by agent, model, status and error
140
+ # class. The returned {Breakdown} derives totals, per-agent and
141
+ # per-model stats, top errors, error cost and cache savings from
142
+ # those groups, so a page that shows all of them costs one pass over
143
+ # the time window instead of one per section.
144
+ #
145
+ # @return [Execution::Breakdown]
146
+ def breakdown
147
+ # Spend belongs to the model that actually ran, not the one that
148
+ # was configured — a fallback's cost must not be billed to the
149
+ # primary. Older rows predate chosen_model_id.
150
+ billed_model = Arel.sql("COALESCE(chosen_model_id, model_id)")
151
+ hit = cache_hit_condition
152
+
153
+ Breakdown.new(group(:agent_type, :model_id, billed_model, :status, :error_class).pluck(
154
+ :agent_type, :model_id, billed_model, :status, :error_class,
155
+ Arel.sql("COUNT(*)"),
156
+ Arel.sql("COALESCE(SUM(total_cost), 0)"),
157
+ Arel.sql("COALESCE(SUM(total_tokens), 0)"),
158
+ Arel.sql("COALESCE(SUM(duration_ms), 0)"),
159
+ Arel.sql("COUNT(duration_ms)"),
160
+ Arel.sql("SUM(CASE WHEN #{hit} THEN 1 ELSE 0 END)"),
161
+ Arel.sql("COALESCE(SUM(CASE WHEN #{hit} THEN 0 ELSE total_cost END), 0)"),
162
+ Arel.sql("MAX(created_at)")
163
+ ))
164
+ end
165
+
166
+ # Runs a block with the database's statement timeout lowered
167
+ #
168
+ # Dashboard queries aggregate over whatever window the user picked,
169
+ # so on a large table a wide window can run for minutes — long after
170
+ # the web server has given up on the request, and with the database
171
+ # still working on it. Under a statement timeout the database
172
+ # cancels the query instead and the block raises
173
+ # ActiveRecord::QueryCanceled.
174
+ #
175
+ # PostgreSQL only; elsewhere the block simply runs. A stricter
176
+ # timeout already in force is left alone. Uses SET LOCAL inside a
177
+ # (sub)transaction, so the setting never outlives the block and is
178
+ # safe behind a transaction-pooling proxy.
179
+ #
180
+ # @param seconds [Numeric, nil] Timeout; nil runs the block unguarded
181
+ # @yield Queries to guard. They must run inside the block — load
182
+ # relations before returning them.
183
+ # @return [Object] The block's value
184
+ # @raise [ActiveRecord::QueryCanceled] When a statement is cancelled
185
+ def with_statement_timeout(seconds)
186
+ return yield unless seconds && connection.adapter_name.downcase.include?("postg")
187
+
188
+ transaction(requires_new: true) do
189
+ current = connection.select_value("SELECT setting FROM pg_settings WHERE name = 'statement_timeout'").to_i
190
+ wanted = (seconds * 1000).ceil
191
+ next yield if current.positive? && current <= wanted
192
+
193
+ connection.execute("SET LOCAL statement_timeout = #{wanted}")
194
+ result = yield
195
+ # Releasing a savepoint keeps its SET LOCAL, so restore by hand;
196
+ # after a cancellation the rollback restores it instead.
197
+ connection.execute("SET LOCAL statement_timeout = #{current}")
198
+ result
199
+ end
200
+ end
201
+
202
+ # Runs a block of queries that the page can live without
203
+ #
204
+ # For figures that are nice to have next to the real content (the
205
+ # totals strip above a list, say): if the database cannot answer
206
+ # within the timeout the caller gets nil and renders a placeholder,
207
+ # rather than the whole page waiting on an aggregate.
208
+ #
209
+ # @param timeout [Numeric] Seconds to allow (PostgreSQL only)
210
+ # @return [Object, nil] The block's value, or nil if it was cancelled
211
+ def best_effort(timeout:, &block)
212
+ with_statement_timeout(timeout, &block)
213
+ rescue ::ActiveRecord::QueryCanceled
214
+ nil
215
+ end
216
+
99
217
  # Compares performance between two agent versions
100
218
  # Analyzes trends over a time period
101
219
  #
@@ -470,36 +588,33 @@ module RubyLLM
470
588
  #
471
589
  # @return [Float] Percentage of executions that were cache hits (0.0-100.0)
472
590
  def cache_hit_rate
473
- total = count
474
- return 0.0 if total.zero?
475
-
476
- (cached.count.to_f / total * 100).round(1)
591
+ boolean_rate("cache_hit")
477
592
  end
478
593
 
479
594
  # Streaming execution rate percentage
480
595
  #
481
596
  # @return [Float] Percentage of executions that used streaming (0.0-100.0)
482
597
  def streaming_rate
483
- total = count
484
- return 0.0 if total.zero?
485
-
486
- (streaming.count.to_f / total * 100).round(1)
598
+ boolean_rate("streaming")
487
599
  end
488
600
 
489
601
  # Average time to first token for streaming executions
490
602
  #
491
- # time_to_first_token_ms is stored in metadata JSON, so we use
492
- # Ruby-level calculation instead of SQL aggregation.
603
+ # time_to_first_token_ms lives in the metadata JSON column, so the
604
+ # average is computed from a JSON extract in SQL rather than by
605
+ # loading every streaming execution's metadata into Ruby.
493
606
  #
494
607
  # @return [Integer, nil] Average TTFT in milliseconds, or nil if no data
495
608
  def avg_time_to_first_token
496
- ttft_values = streaming
497
- .where("metadata IS NOT NULL")
498
- .pluck(:metadata)
499
- .filter_map { |m| m&.dig("time_to_first_token_ms") }
500
- return nil if ttft_values.empty?
609
+ ttft = if connection.adapter_name.downcase.include?("sqlite")
610
+ "json_extract(metadata, '$.time_to_first_token_ms')"
611
+ else
612
+ "(metadata->>'time_to_first_token_ms')::numeric"
613
+ end
501
614
 
502
- (ttft_values.sum.to_f / ttft_values.size).round(0)
615
+ streaming.metadata_present("time_to_first_token_ms")
616
+ .pick(Arel.sql("AVG(#{ttft})"))
617
+ &.round
503
618
  end
504
619
 
505
620
  # Finish reason distribution
@@ -529,47 +644,11 @@ module RubyLLM
529
644
  end
530
645
 
531
646
  # Builds per-model statistics for model comparison
532
- # Optimized: Single SQL GROUP BY with conditional aggregation
533
647
  #
534
648
  # @param scope [ActiveRecord::Relation] Pre-filtered scope
535
649
  # @return [Array<Hash>] Model stats sorted by total cost descending
536
650
  def model_stats(scope: all)
537
- # Attribute spend to the model that actually ran, not the one that
538
- # was configured — otherwise a fallback's cost is billed to the
539
- # primary in the comparison table. Older rows predate
540
- # chosen_model_id, so fall back to model_id for those.
541
- billed_model = Arel.sql("COALESCE(chosen_model_id, model_id)")
542
-
543
- rows = scope.where.not(model_id: nil)
544
- .select(
545
- Arel.sql("#{billed_model} AS billed_model_id"),
546
- Arel.sql("COUNT(*) AS exec_count"),
547
- Arel.sql("COALESCE(SUM(total_cost), 0) AS sum_cost"),
548
- Arel.sql("COALESCE(SUM(total_tokens), 0) AS sum_tokens"),
549
- Arel.sql("AVG(duration_ms) AS avg_dur"),
550
- Arel.sql("SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS success_cnt")
551
- )
552
- .group(billed_model)
553
-
554
- total_cost = rows.sum { |r| r["sum_cost"].to_f }
555
-
556
- rows.map do |row|
557
- count = row["exec_count"].to_i
558
- model_cost = row["sum_cost"].to_f
559
- model_tokens = row["sum_tokens"].to_i
560
- successful = row["success_cnt"].to_i
561
-
562
- {
563
- model_id: row["billed_model_id"],
564
- executions: count,
565
- total_cost: model_cost,
566
- total_tokens: model_tokens,
567
- avg_duration_ms: row["avg_dur"].to_i,
568
- success_rate: (count > 0) ? (successful.to_f / count * 100).round(1) : 0,
569
- cost_per_1k_tokens: (model_tokens > 0) ? (model_cost / model_tokens * 1000).round(4) : 0,
570
- cost_percentage: (total_cost > 0) ? (model_cost / total_cost * 100).round(1) : 0
571
- }
572
- end.sort_by { |m| -(m[:total_cost] || 0) }
651
+ scope.breakdown.model_stats
573
652
  end
574
653
 
575
654
  # Builds top errors list from error executions
@@ -578,87 +657,23 @@ module RubyLLM
578
657
  # @param limit [Integer] Max errors to return
579
658
  # @return [Array<Hash>] Top error classes with counts
580
659
  def top_errors(scope: all, limit: 5)
581
- error_scope = scope.where(status: "error")
582
- total_errors = error_scope.count
583
-
584
- error_scope.group(:error_class)
585
- .select("error_class, COUNT(*) as count, MAX(created_at) as last_seen")
586
- .order("count DESC")
587
- .limit(limit)
588
- .map do |row|
589
- {
590
- error_class: row.error_class || "Unknown Error",
591
- count: row.count,
592
- percentage: (total_errors > 0) ? (row.count.to_f / total_errors * 100).round(1) : 0,
593
- last_seen: row.last_seen
594
- }
595
- end
660
+ scope.breakdown.top_errors(limit: limit)
596
661
  end
597
662
 
598
663
  # Builds cache savings statistics
599
- # Optimized: Single SQL query with conditional aggregation
600
664
  #
601
665
  # @param scope [ActiveRecord::Relation] Pre-filtered scope
602
666
  # @return [Hash] Cache savings data
603
667
  def cache_savings(scope: all)
604
- cond = cache_hit_condition
605
- total_count, cache_count, miss_count, miss_cost = scope.pick(
606
- Arel.sql("COUNT(*)"),
607
- Arel.sql("SUM(CASE WHEN #{cond} THEN 1 ELSE 0 END)"),
608
- Arel.sql("SUM(CASE WHEN #{cond} THEN 0 ELSE 1 END)"),
609
- Arel.sql("COALESCE(SUM(CASE WHEN #{cond} THEN 0 ELSE total_cost END), 0)")
610
- )
611
-
612
- total_count = total_count.to_i
613
- cache_count = cache_count.to_i
614
- miss_count = miss_count.to_i
615
-
616
- return {count: 0, estimated_savings: 0, hit_rate: 0, total_executions: 0} if total_count.zero?
617
-
618
- # Savings are the cost AVOIDED, not the cost recorded: a cache hit
619
- # makes no API call, so its own total_cost is always 0 and summing
620
- # those rows reported $0.00 saved forever. Estimate each hit at the
621
- # mean cost of the misses in this same scope.
622
- #
623
- # ponytail: scope-wide mean, not per-agent. Group by agent_type if
624
- # a mixed dashboard scope ever skews this enough to matter.
625
- avg_miss_cost = miss_count.positive? ? (miss_cost.to_f / miss_count) : 0.0
626
-
627
- {
628
- count: cache_count,
629
- estimated_savings: (cache_count * avg_miss_cost).round(6),
630
- hit_rate: (cache_count.to_f / total_count * 100).round(1),
631
- total_executions: total_count
632
- }
668
+ scope.breakdown.cache_savings
633
669
  end
634
670
 
635
671
  # Batch fetches execution stats grouped by agent type
636
- # Optimized: Single SQL GROUP BY with conditional aggregation
637
672
  #
638
673
  # @param scope [ActiveRecord::Relation] Pre-filtered scope
639
674
  # @return [Hash<String, Hash>] Agent type => stats hash
640
675
  def batch_agent_stats(scope: all)
641
- rows = scope.select(
642
- :agent_type,
643
- Arel.sql("COUNT(*) AS exec_count"),
644
- Arel.sql("COALESCE(SUM(total_cost), 0) AS sum_cost"),
645
- Arel.sql("AVG(duration_ms) AS avg_dur"),
646
- Arel.sql("SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS success_cnt")
647
- ).group(:agent_type)
648
-
649
- rows.each_with_object({}) do |row, hash|
650
- count = row["exec_count"].to_i
651
- total_cost = row["sum_cost"].to_f
652
- successful = row["success_cnt"].to_i
653
-
654
- hash[row.agent_type] = {
655
- count: count,
656
- total_cost: total_cost,
657
- avg_cost: (count > 0) ? (total_cost / count).round(6) : 0,
658
- avg_duration_ms: row["avg_dur"].to_i,
659
- success_rate: (count > 0) ? (successful.to_f / count * 100).round(1) : 0
660
- }
661
- end
676
+ scope.breakdown.agent_stats
662
677
  end
663
678
 
664
679
  # Cached daily statistics for dashboard
@@ -790,16 +805,42 @@ module RubyLLM
790
805
 
791
806
  # SQL condition for boolean cache_hit column
792
807
  #
808
+ # @return [String] SQL condition fragment
809
+ def cache_hit_condition
810
+ true_condition("cache_hit")
811
+ end
812
+
813
+ # SQL condition for a boolean column being true
814
+ #
793
815
  # SQLite stores booleans as 1/0, PostgreSQL as TRUE/FALSE.
794
816
  #
817
+ # @param column [String] Column name (internal literal, never user input)
795
818
  # @return [String] SQL condition fragment
796
- def cache_hit_condition
819
+ def true_condition(column)
797
820
  if connection.adapter_name.downcase.include?("sqlite")
798
- "cache_hit = 1"
821
+ "#{column} = 1"
799
822
  else
800
- "cache_hit = TRUE"
823
+ "#{column} = TRUE"
801
824
  end
802
825
  end
826
+
827
+ # Percentage of the current scope where a boolean column is true
828
+ #
829
+ # One query (count plus conditional sum) rather than a count of the
830
+ # scope followed by a count of the filtered scope.
831
+ #
832
+ # @param column [String] Column name (internal literal, never user input)
833
+ # @return [Float] 0.0-100.0
834
+ def boolean_rate(column)
835
+ total, hits = pick(
836
+ Arel.sql("COUNT(*)"),
837
+ Arel.sql("SUM(CASE WHEN #{true_condition(column)} THEN 1 ELSE 0 END)")
838
+ )
839
+ total = total.to_i
840
+ return 0.0 if total.zero?
841
+
842
+ (hits.to_f / total * 100).round(1)
843
+ end
803
844
  end
804
845
  end
805
846
  end
@@ -0,0 +1,243 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyLLM
4
+ module Agents
5
+ class Execution
6
+ # Every dashboard figure for a set of executions, from one scan
7
+ #
8
+ # The dashboard and analytics pages each show half a dozen sections over
9
+ # the same time window: totals, per-agent stats, per-model stats, top
10
+ # errors, cache savings, error cost. Computed separately, that is one
11
+ # pass over the window per section — and on a table with millions of
12
+ # executions each pass takes seconds. A Breakdown reads the window once,
13
+ # grouped by the few low-cardinality columns the sections slice on, and
14
+ # derives each section from those groups in Ruby.
15
+ #
16
+ # @example
17
+ # breakdown = Execution.last_n_days(30).breakdown
18
+ # breakdown.totals #=> { total: 1200, cost: 4.2, success_rate: 97.1, ... }
19
+ # breakdown.model_stats #=> [{ model_id: "gpt-4o", executions: 800, ... }]
20
+ # breakdown.top_errors #=> [{ error_class: "Timeout::Error", count: 12, ... }]
21
+ #
22
+ # @see Execution::Analytics::ClassMethods#breakdown
23
+ # @api public
24
+ class Breakdown
25
+ Group = Struct.new(
26
+ :agent_type, :model_id, :billed_model_id, :status, :error_class,
27
+ :count, :cost, :tokens, :duration_sum, :duration_count,
28
+ :cache_hits, :miss_cost, :last_seen
29
+ )
30
+
31
+ # Raw grouped rows, as plain arrays so a breakdown can be cached and
32
+ # rebuilt with {.new}.
33
+ #
34
+ # @return [Array<Array>]
35
+ attr_reader :rows
36
+
37
+ # @param rows [Array<Array>] One array per group, in {Group} member
38
+ # order, as selected by {Execution::Analytics::ClassMethods#breakdown}
39
+ def initialize(rows)
40
+ @rows = rows
41
+ @groups = rows.map do |row|
42
+ Group.new(*row[0, 5], row[5].to_i, row[6].to_f, row[7].to_i, row[8].to_i,
43
+ row[9].to_i, row[10].to_i, row[11].to_f, parse_time(row[12]))
44
+ end
45
+ end
46
+
47
+ # A breakdown narrowed to some agents
48
+ #
49
+ # @param agent_types [Array<String>] Agent class names to keep
50
+ # @return [Breakdown]
51
+ def for_agents(agent_types)
52
+ names = Array(agent_types)
53
+ self.class.new(@rows.select { |row| names.include?(row[0]) })
54
+ end
55
+
56
+ # Headline figures across the whole scope
57
+ #
58
+ # @return [Hash] :total, :success, :errors, :timeouts, :cost, :tokens,
59
+ # :avg_duration_ms, :success_rate, :error_rate, :last_seen
60
+ def totals
61
+ total = sum(@groups, :count)
62
+ success = sum(with_status("success"), :count)
63
+ errors = sum(with_status("error"), :count)
64
+ timeouts = sum(with_status("timeout"), :count)
65
+
66
+ {
67
+ total: total,
68
+ success: success,
69
+ errors: errors,
70
+ timeouts: timeouts,
71
+ cost: sum(@groups, :cost),
72
+ tokens: sum(@groups, :tokens),
73
+ avg_duration_ms: avg_duration(@groups),
74
+ success_rate: rate(success, total),
75
+ error_rate: rate(errors + timeouts, total),
76
+ last_seen: @groups.filter_map(&:last_seen).max
77
+ }
78
+ end
79
+
80
+ # Per-agent stats
81
+ #
82
+ # @return [Hash{String => Hash}] Agent type => :count, :total_cost,
83
+ # :total_tokens, :avg_cost, :avg_duration_ms, :success_rate
84
+ def agent_stats
85
+ @groups.group_by(&:agent_type).transform_values do |groups|
86
+ count = sum(groups, :count)
87
+ cost = sum(groups, :cost)
88
+
89
+ {
90
+ count: count,
91
+ total_cost: cost,
92
+ total_tokens: sum(groups, :tokens),
93
+ avg_cost: (count > 0) ? (cost / count).round(6) : 0,
94
+ avg_duration_ms: avg_duration(groups),
95
+ success_rate: rate(sum(groups.select { |g| g.status == "success" }, :count), count)
96
+ }
97
+ end
98
+ end
99
+
100
+ # Per-model stats, attributed to the model that actually ran
101
+ #
102
+ # @return [Array<Hash>] Sorted by total cost, most expensive first
103
+ def model_stats
104
+ overall_cost = sum(@groups, :cost)
105
+
106
+ @groups.group_by(&:billed_model_id).map do |model_id, groups|
107
+ count = sum(groups, :count)
108
+ cost = sum(groups, :cost)
109
+ tokens = sum(groups, :tokens)
110
+
111
+ {
112
+ model_id: model_id,
113
+ executions: count,
114
+ total_cost: cost,
115
+ total_tokens: tokens,
116
+ avg_duration_ms: avg_duration(groups),
117
+ success_rate: rate(sum(groups.select { |g| g.status == "success" }, :count), count),
118
+ cost_per_1k_tokens: (tokens > 0) ? (cost / tokens * 1000).round(4) : 0,
119
+ cost_percentage: (overall_cost > 0) ? (cost / overall_cost * 100).round(1) : 0
120
+ }
121
+ end.sort_by { |m| -m[:total_cost] }
122
+ end
123
+
124
+ # Usage per configured model, most expensive first
125
+ #
126
+ # Unlike {#model_stats} this groups on the model the agent asked for,
127
+ # which is what the "move these runs to a cheaper model" comparison
128
+ # needs.
129
+ #
130
+ # @return [Array<Hash>] :model_id, :runs, :cost, :tokens, :cost_per_run,
131
+ # :cost_per_1k_tokens
132
+ def configured_model_usage
133
+ @groups.group_by(&:model_id).map do |model_id, groups|
134
+ count = sum(groups, :count)
135
+ cost = sum(groups, :cost)
136
+ tokens = sum(groups, :tokens)
137
+
138
+ {
139
+ model_id: model_id,
140
+ runs: count,
141
+ cost: cost,
142
+ tokens: tokens,
143
+ cost_per_run: (count > 0) ? (cost / count) : 0,
144
+ cost_per_1k_tokens: (tokens > 0) ? (cost / tokens * 1000) : 0
145
+ }
146
+ end.sort_by { |m| -m[:cost] }
147
+ end
148
+
149
+ # Most frequent error classes
150
+ #
151
+ # @param limit [Integer] Max error classes to return
152
+ # @return [Array<Hash>] :error_class, :count, :percentage, :last_seen
153
+ def top_errors(limit: 5)
154
+ errors = with_status("error")
155
+ total = sum(errors, :count)
156
+
157
+ errors.group_by(&:error_class).map do |error_class, groups|
158
+ count = sum(groups, :count)
159
+
160
+ {
161
+ error_class: error_class || "Unknown Error",
162
+ count: count,
163
+ percentage: rate(count, total),
164
+ last_seen: groups.filter_map(&:last_seen).max
165
+ }
166
+ end.sort_by { |e| -e[:count] }.first(limit)
167
+ end
168
+
169
+ # Money spent on executions that ended in an error
170
+ #
171
+ # @param limit [Integer] Max (error class, agent) pairs to return
172
+ # @return [Hash] :total_cost, :total_count, and :breakdown — the
173
+ # costliest (error class, agent) pairs
174
+ def error_cost(limit: 10)
175
+ errors = with_status("error")
176
+
177
+ pairs = errors.group_by { |g| [g.error_class, g.agent_type] }.map do |(error_class, agent_type), groups|
178
+ {
179
+ error_class: error_class || "Unknown",
180
+ agent_type: agent_type,
181
+ count: sum(groups, :count),
182
+ cost: sum(groups, :cost),
183
+ last_seen: groups.filter_map(&:last_seen).max
184
+ }
185
+ end
186
+
187
+ {
188
+ total_cost: sum(errors, :cost),
189
+ total_count: sum(errors, :count),
190
+ breakdown: pairs.sort_by { |e| -e[:cost] }.first(limit)
191
+ }
192
+ end
193
+
194
+ # Cache hit rate and the spend those hits avoided
195
+ #
196
+ # A cache hit makes no API call, so its own cost is always zero; the
197
+ # saving is estimated at the mean cost of the misses in the same scope.
198
+ #
199
+ # @return [Hash] :count, :estimated_savings, :hit_rate, :total_executions
200
+ def cache_savings
201
+ total = sum(@groups, :count)
202
+ return {count: 0, estimated_savings: 0, hit_rate: 0, total_executions: 0} if total.zero?
203
+
204
+ hits = sum(@groups, :cache_hits)
205
+ misses = total - hits
206
+ avg_miss_cost = misses.positive? ? (sum(@groups, :miss_cost) / misses) : 0.0
207
+
208
+ {
209
+ count: hits,
210
+ estimated_savings: (hits * avg_miss_cost).round(6),
211
+ hit_rate: rate(hits, total),
212
+ total_executions: total
213
+ }
214
+ end
215
+
216
+ private
217
+
218
+ def with_status(status)
219
+ @groups.select { |g| g.status == status }
220
+ end
221
+
222
+ def sum(groups, field)
223
+ groups.sum { |g| g[field] }
224
+ end
225
+
226
+ def rate(part, whole)
227
+ (whole > 0) ? (part.to_f / whole * 100).round(1) : 0.0
228
+ end
229
+
230
+ def avg_duration(groups)
231
+ samples = sum(groups, :duration_count)
232
+ (samples > 0) ? (sum(groups, :duration_sum) / samples) : 0
233
+ end
234
+
235
+ # MAX(created_at) comes back as a Time on PostgreSQL and as a UTC
236
+ # string on SQLite.
237
+ def parse_time(value)
238
+ value.is_a?(String) ? ActiveSupport::TimeZone["UTC"].parse(value) : value
239
+ end
240
+ end
241
+ end
242
+ end
243
+ end