ruby_llm-agents 3.15.1 → 3.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/controllers/concerns/ruby_llm/agents/paginatable.rb +29 -8
- data/app/controllers/ruby_llm/agents/agents_controller.rb +48 -44
- data/app/controllers/ruby_llm/agents/analytics_controller.rb +38 -72
- data/app/controllers/ruby_llm/agents/dashboard_controller.rb +69 -56
- data/app/controllers/ruby_llm/agents/executions_controller.rb +25 -41
- data/app/controllers/ruby_llm/agents/requests_controller.rb +117 -45
- data/app/controllers/ruby_llm/agents/tenants_controller.rb +67 -47
- data/app/models/ruby_llm/agents/execution/analytics.rb +174 -133
- data/app/models/ruby_llm/agents/execution/breakdown.rb +243 -0
- data/app/models/ruby_llm/agents/execution/scopes.rb +39 -3
- data/app/models/ruby_llm/agents/execution.rb +23 -5
- data/app/services/ruby_llm/agents/agent_registry.rb +75 -12
- data/app/views/ruby_llm/agents/agents/index.html.erb +24 -7
- data/app/views/ruby_llm/agents/agents/show.html.erb +3 -2
- data/app/views/ruby_llm/agents/executions/_filters.html.erb +12 -7
- data/app/views/ruby_llm/agents/executions/_list.html.erb +1 -52
- data/app/views/ruby_llm/agents/requests/index.html.erb +8 -57
- data/app/views/ruby_llm/agents/shared/_executions_table.html.erb +1 -52
- data/app/views/ruby_llm/agents/shared/_pagination.html.erb +66 -0
- data/app/views/ruby_llm/agents/shared/query_timeout.html.erb +18 -0
- data/app/views/ruby_llm/agents/system_config/show.html.erb +4 -0
- data/app/views/ruby_llm/agents/tenants/index.html.erb +1 -1
- data/app/views/ruby_llm/agents/tenants/show.html.erb +6 -5
- data/config/routes.rb +2 -2
- data/lib/generators/ruby_llm_agents/templates/add_executions_analytics_index_migration.rb.tt +50 -0
- data/lib/generators/ruby_llm_agents/templates/add_root_execution_list_index_migration.rb.tt +16 -0
- data/lib/generators/ruby_llm_agents/templates/initializer.rb.tt +5 -0
- data/lib/generators/ruby_llm_agents/templates/migration.rb.tt +26 -0
- data/lib/generators/ruby_llm_agents/upgrade_generator.rb +44 -0
- data/lib/ruby_llm/agents/core/configuration.rb +20 -0
- data/lib/ruby_llm/agents/core/version.rb +1 -1
- data/lib/ruby_llm/agents/pipeline/middleware/instrumentation.rb +39 -1
- data/lib/ruby_llm/agents/rails/engine.rb +87 -7
- metadata +7 -2
|
@@ -78,24 +78,142 @@ module RubyLLM
|
|
|
78
78
|
# @param period [Symbol] Time scope (:today, :this_week, :this_month, :all_time)
|
|
79
79
|
# @return [Hash] Statistics including count, costs, tokens, duration, rates
|
|
80
80
|
def stats_for(agent_type, period: :today)
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
81
|
+
{agent_type: agent_type, period: period}
|
|
82
|
+
.merge(by_agent(agent_type).public_send(period).usage_summary)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Usage and reliability figures for the current scope, in one query
|
|
86
|
+
#
|
|
87
|
+
# @example An agent's last 30 days
|
|
88
|
+
# Execution.by_agent("SearchAgent").last_n_days(30).usage_summary
|
|
89
|
+
#
|
|
90
|
+
# @return [Hash] :count, :total_cost, :avg_cost, :total_tokens,
|
|
91
|
+
# :avg_tokens, :avg_duration_ms, :success_rate, :error_rate,
|
|
92
|
+
# :cache_hit_rate, :streaming_rate
|
|
93
|
+
def usage_summary
|
|
94
|
+
count, cost, tokens, avg_tok, avg_dur, successful, failed, cache_hits, streamed = pick(
|
|
95
|
+
Arel.sql("COUNT(*)"),
|
|
96
|
+
Arel.sql("COALESCE(SUM(total_cost), 0)"),
|
|
97
|
+
Arel.sql("COALESCE(SUM(total_tokens), 0)"),
|
|
98
|
+
Arel.sql("AVG(total_tokens)"),
|
|
99
|
+
Arel.sql("AVG(duration_ms)"),
|
|
100
|
+
Arel.sql("SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END)"),
|
|
101
|
+
Arel.sql("SUM(CASE WHEN status IN ('error', 'timeout') THEN 1 ELSE 0 END)"),
|
|
102
|
+
Arel.sql("SUM(CASE WHEN #{true_condition("cache_hit")} THEN 1 ELSE 0 END)"),
|
|
103
|
+
Arel.sql("SUM(CASE WHEN #{true_condition("streaming")} THEN 1 ELSE 0 END)")
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
count = count.to_i
|
|
107
|
+
total_cost = (cost || 0).to_d.round(6)
|
|
108
|
+
percent = ->(part, digits) { (count > 0) ? (part.to_f / count * 100).round(digits) : 0.0 }
|
|
84
109
|
|
|
85
110
|
{
|
|
86
|
-
agent_type: agent_type,
|
|
87
|
-
period: period,
|
|
88
111
|
count: count,
|
|
89
112
|
total_cost: total_cost,
|
|
90
113
|
avg_cost: (count > 0) ? (total_cost / count).round(6) : 0,
|
|
91
|
-
total_tokens:
|
|
92
|
-
avg_tokens:
|
|
93
|
-
avg_duration_ms:
|
|
94
|
-
success_rate:
|
|
95
|
-
error_rate:
|
|
114
|
+
total_tokens: tokens.to_i,
|
|
115
|
+
avg_tokens: avg_tok&.round || 0,
|
|
116
|
+
avg_duration_ms: avg_dur&.round || 0,
|
|
117
|
+
success_rate: percent.call(successful, 2),
|
|
118
|
+
error_rate: percent.call(failed, 2),
|
|
119
|
+
cache_hit_rate: percent.call(cache_hits, 1),
|
|
120
|
+
streaming_rate: percent.call(streamed, 1)
|
|
96
121
|
}
|
|
97
122
|
end
|
|
98
123
|
|
|
124
|
+
# Count, cost and token totals for the current scope in one query
|
|
125
|
+
#
|
|
126
|
+
# @return [Hash] :total_count, :total_cost (BigDecimal), :total_tokens
|
|
127
|
+
def totals
|
|
128
|
+
count, cost, tokens = pick(
|
|
129
|
+
Arel.sql("COUNT(*)"),
|
|
130
|
+
Arel.sql("COALESCE(SUM(total_cost), 0)"),
|
|
131
|
+
Arel.sql("COALESCE(SUM(total_tokens), 0)")
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
{total_count: count.to_i, total_cost: (cost || 0).to_d.round(6), total_tokens: tokens.to_i}
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Everything the dashboard shows about the current scope, from one scan
|
|
138
|
+
#
|
|
139
|
+
# Reads the scope once, grouped by agent, model, status and error
|
|
140
|
+
# class. The returned {Breakdown} derives totals, per-agent and
|
|
141
|
+
# per-model stats, top errors, error cost and cache savings from
|
|
142
|
+
# those groups, so a page that shows all of them costs one pass over
|
|
143
|
+
# the time window instead of one per section.
|
|
144
|
+
#
|
|
145
|
+
# @return [Execution::Breakdown]
|
|
146
|
+
def breakdown
|
|
147
|
+
# Spend belongs to the model that actually ran, not the one that
|
|
148
|
+
# was configured — a fallback's cost must not be billed to the
|
|
149
|
+
# primary. Older rows predate chosen_model_id.
|
|
150
|
+
billed_model = Arel.sql("COALESCE(chosen_model_id, model_id)")
|
|
151
|
+
hit = cache_hit_condition
|
|
152
|
+
|
|
153
|
+
Breakdown.new(group(:agent_type, :model_id, billed_model, :status, :error_class).pluck(
|
|
154
|
+
:agent_type, :model_id, billed_model, :status, :error_class,
|
|
155
|
+
Arel.sql("COUNT(*)"),
|
|
156
|
+
Arel.sql("COALESCE(SUM(total_cost), 0)"),
|
|
157
|
+
Arel.sql("COALESCE(SUM(total_tokens), 0)"),
|
|
158
|
+
Arel.sql("COALESCE(SUM(duration_ms), 0)"),
|
|
159
|
+
Arel.sql("COUNT(duration_ms)"),
|
|
160
|
+
Arel.sql("SUM(CASE WHEN #{hit} THEN 1 ELSE 0 END)"),
|
|
161
|
+
Arel.sql("COALESCE(SUM(CASE WHEN #{hit} THEN 0 ELSE total_cost END), 0)"),
|
|
162
|
+
Arel.sql("MAX(created_at)")
|
|
163
|
+
))
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# Runs a block with the database's statement timeout lowered
|
|
167
|
+
#
|
|
168
|
+
# Dashboard queries aggregate over whatever window the user picked,
|
|
169
|
+
# so on a large table a wide window can run for minutes — long after
|
|
170
|
+
# the web server has given up on the request, and with the database
|
|
171
|
+
# still working on it. Under a statement timeout the database
|
|
172
|
+
# cancels the query instead and the block raises
|
|
173
|
+
# ActiveRecord::QueryCanceled.
|
|
174
|
+
#
|
|
175
|
+
# PostgreSQL only; elsewhere the block simply runs. A stricter
|
|
176
|
+
# timeout already in force is left alone. Uses SET LOCAL inside a
|
|
177
|
+
# (sub)transaction, so the setting never outlives the block and is
|
|
178
|
+
# safe behind a transaction-pooling proxy.
|
|
179
|
+
#
|
|
180
|
+
# @param seconds [Numeric, nil] Timeout; nil runs the block unguarded
|
|
181
|
+
# @yield Queries to guard. They must run inside the block — load
|
|
182
|
+
# relations before returning them.
|
|
183
|
+
# @return [Object] The block's value
|
|
184
|
+
# @raise [ActiveRecord::QueryCanceled] When a statement is cancelled
|
|
185
|
+
def with_statement_timeout(seconds)
|
|
186
|
+
return yield unless seconds && connection.adapter_name.downcase.include?("postg")
|
|
187
|
+
|
|
188
|
+
transaction(requires_new: true) do
|
|
189
|
+
current = connection.select_value("SELECT setting FROM pg_settings WHERE name = 'statement_timeout'").to_i
|
|
190
|
+
wanted = (seconds * 1000).ceil
|
|
191
|
+
next yield if current.positive? && current <= wanted
|
|
192
|
+
|
|
193
|
+
connection.execute("SET LOCAL statement_timeout = #{wanted}")
|
|
194
|
+
result = yield
|
|
195
|
+
# Releasing a savepoint keeps its SET LOCAL, so restore by hand;
|
|
196
|
+
# after a cancellation the rollback restores it instead.
|
|
197
|
+
connection.execute("SET LOCAL statement_timeout = #{current}")
|
|
198
|
+
result
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# Runs a block of queries that the page can live without
|
|
203
|
+
#
|
|
204
|
+
# For figures that are nice to have next to the real content (the
|
|
205
|
+
# totals strip above a list, say): if the database cannot answer
|
|
206
|
+
# within the timeout the caller gets nil and renders a placeholder,
|
|
207
|
+
# rather than the whole page waiting on an aggregate.
|
|
208
|
+
#
|
|
209
|
+
# @param timeout [Numeric] Seconds to allow (PostgreSQL only)
|
|
210
|
+
# @return [Object, nil] The block's value, or nil if it was cancelled
|
|
211
|
+
def best_effort(timeout:, &block)
|
|
212
|
+
with_statement_timeout(timeout, &block)
|
|
213
|
+
rescue ::ActiveRecord::QueryCanceled
|
|
214
|
+
nil
|
|
215
|
+
end
|
|
216
|
+
|
|
99
217
|
# Compares performance between two agent versions
|
|
100
218
|
# Analyzes trends over a time period
|
|
101
219
|
#
|
|
@@ -470,36 +588,33 @@ module RubyLLM
|
|
|
470
588
|
#
|
|
471
589
|
# @return [Float] Percentage of executions that were cache hits (0.0-100.0)
|
|
472
590
|
def cache_hit_rate
|
|
473
|
-
|
|
474
|
-
return 0.0 if total.zero?
|
|
475
|
-
|
|
476
|
-
(cached.count.to_f / total * 100).round(1)
|
|
591
|
+
boolean_rate("cache_hit")
|
|
477
592
|
end
|
|
478
593
|
|
|
479
594
|
# Streaming execution rate percentage
|
|
480
595
|
#
|
|
481
596
|
# @return [Float] Percentage of executions that used streaming (0.0-100.0)
|
|
482
597
|
def streaming_rate
|
|
483
|
-
|
|
484
|
-
return 0.0 if total.zero?
|
|
485
|
-
|
|
486
|
-
(streaming.count.to_f / total * 100).round(1)
|
|
598
|
+
boolean_rate("streaming")
|
|
487
599
|
end
|
|
488
600
|
|
|
489
601
|
# Average time to first token for streaming executions
|
|
490
602
|
#
|
|
491
|
-
# time_to_first_token_ms
|
|
492
|
-
#
|
|
603
|
+
# time_to_first_token_ms lives in the metadata JSON column, so the
|
|
604
|
+
# average is computed from a JSON extract in SQL rather than by
|
|
605
|
+
# loading every streaming execution's metadata into Ruby.
|
|
493
606
|
#
|
|
494
607
|
# @return [Integer, nil] Average TTFT in milliseconds, or nil if no data
|
|
495
608
|
def avg_time_to_first_token
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
609
|
+
ttft = if connection.adapter_name.downcase.include?("sqlite")
|
|
610
|
+
"json_extract(metadata, '$.time_to_first_token_ms')"
|
|
611
|
+
else
|
|
612
|
+
"(metadata->>'time_to_first_token_ms')::numeric"
|
|
613
|
+
end
|
|
501
614
|
|
|
502
|
-
|
|
615
|
+
streaming.metadata_present("time_to_first_token_ms")
|
|
616
|
+
.pick(Arel.sql("AVG(#{ttft})"))
|
|
617
|
+
&.round
|
|
503
618
|
end
|
|
504
619
|
|
|
505
620
|
# Finish reason distribution
|
|
@@ -529,47 +644,11 @@ module RubyLLM
|
|
|
529
644
|
end
|
|
530
645
|
|
|
531
646
|
# Builds per-model statistics for model comparison
|
|
532
|
-
# Optimized: Single SQL GROUP BY with conditional aggregation
|
|
533
647
|
#
|
|
534
648
|
# @param scope [ActiveRecord::Relation] Pre-filtered scope
|
|
535
649
|
# @return [Array<Hash>] Model stats sorted by total cost descending
|
|
536
650
|
def model_stats(scope: all)
|
|
537
|
-
|
|
538
|
-
# was configured — otherwise a fallback's cost is billed to the
|
|
539
|
-
# primary in the comparison table. Older rows predate
|
|
540
|
-
# chosen_model_id, so fall back to model_id for those.
|
|
541
|
-
billed_model = Arel.sql("COALESCE(chosen_model_id, model_id)")
|
|
542
|
-
|
|
543
|
-
rows = scope.where.not(model_id: nil)
|
|
544
|
-
.select(
|
|
545
|
-
Arel.sql("#{billed_model} AS billed_model_id"),
|
|
546
|
-
Arel.sql("COUNT(*) AS exec_count"),
|
|
547
|
-
Arel.sql("COALESCE(SUM(total_cost), 0) AS sum_cost"),
|
|
548
|
-
Arel.sql("COALESCE(SUM(total_tokens), 0) AS sum_tokens"),
|
|
549
|
-
Arel.sql("AVG(duration_ms) AS avg_dur"),
|
|
550
|
-
Arel.sql("SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS success_cnt")
|
|
551
|
-
)
|
|
552
|
-
.group(billed_model)
|
|
553
|
-
|
|
554
|
-
total_cost = rows.sum { |r| r["sum_cost"].to_f }
|
|
555
|
-
|
|
556
|
-
rows.map do |row|
|
|
557
|
-
count = row["exec_count"].to_i
|
|
558
|
-
model_cost = row["sum_cost"].to_f
|
|
559
|
-
model_tokens = row["sum_tokens"].to_i
|
|
560
|
-
successful = row["success_cnt"].to_i
|
|
561
|
-
|
|
562
|
-
{
|
|
563
|
-
model_id: row["billed_model_id"],
|
|
564
|
-
executions: count,
|
|
565
|
-
total_cost: model_cost,
|
|
566
|
-
total_tokens: model_tokens,
|
|
567
|
-
avg_duration_ms: row["avg_dur"].to_i,
|
|
568
|
-
success_rate: (count > 0) ? (successful.to_f / count * 100).round(1) : 0,
|
|
569
|
-
cost_per_1k_tokens: (model_tokens > 0) ? (model_cost / model_tokens * 1000).round(4) : 0,
|
|
570
|
-
cost_percentage: (total_cost > 0) ? (model_cost / total_cost * 100).round(1) : 0
|
|
571
|
-
}
|
|
572
|
-
end.sort_by { |m| -(m[:total_cost] || 0) }
|
|
651
|
+
scope.breakdown.model_stats
|
|
573
652
|
end
|
|
574
653
|
|
|
575
654
|
# Builds top errors list from error executions
|
|
@@ -578,87 +657,23 @@ module RubyLLM
|
|
|
578
657
|
# @param limit [Integer] Max errors to return
|
|
579
658
|
# @return [Array<Hash>] Top error classes with counts
|
|
580
659
|
def top_errors(scope: all, limit: 5)
|
|
581
|
-
|
|
582
|
-
total_errors = error_scope.count
|
|
583
|
-
|
|
584
|
-
error_scope.group(:error_class)
|
|
585
|
-
.select("error_class, COUNT(*) as count, MAX(created_at) as last_seen")
|
|
586
|
-
.order("count DESC")
|
|
587
|
-
.limit(limit)
|
|
588
|
-
.map do |row|
|
|
589
|
-
{
|
|
590
|
-
error_class: row.error_class || "Unknown Error",
|
|
591
|
-
count: row.count,
|
|
592
|
-
percentage: (total_errors > 0) ? (row.count.to_f / total_errors * 100).round(1) : 0,
|
|
593
|
-
last_seen: row.last_seen
|
|
594
|
-
}
|
|
595
|
-
end
|
|
660
|
+
scope.breakdown.top_errors(limit: limit)
|
|
596
661
|
end
|
|
597
662
|
|
|
598
663
|
# Builds cache savings statistics
|
|
599
|
-
# Optimized: Single SQL query with conditional aggregation
|
|
600
664
|
#
|
|
601
665
|
# @param scope [ActiveRecord::Relation] Pre-filtered scope
|
|
602
666
|
# @return [Hash] Cache savings data
|
|
603
667
|
def cache_savings(scope: all)
|
|
604
|
-
|
|
605
|
-
total_count, cache_count, miss_count, miss_cost = scope.pick(
|
|
606
|
-
Arel.sql("COUNT(*)"),
|
|
607
|
-
Arel.sql("SUM(CASE WHEN #{cond} THEN 1 ELSE 0 END)"),
|
|
608
|
-
Arel.sql("SUM(CASE WHEN #{cond} THEN 0 ELSE 1 END)"),
|
|
609
|
-
Arel.sql("COALESCE(SUM(CASE WHEN #{cond} THEN 0 ELSE total_cost END), 0)")
|
|
610
|
-
)
|
|
611
|
-
|
|
612
|
-
total_count = total_count.to_i
|
|
613
|
-
cache_count = cache_count.to_i
|
|
614
|
-
miss_count = miss_count.to_i
|
|
615
|
-
|
|
616
|
-
return {count: 0, estimated_savings: 0, hit_rate: 0, total_executions: 0} if total_count.zero?
|
|
617
|
-
|
|
618
|
-
# Savings are the cost AVOIDED, not the cost recorded: a cache hit
|
|
619
|
-
# makes no API call, so its own total_cost is always 0 and summing
|
|
620
|
-
# those rows reported $0.00 saved forever. Estimate each hit at the
|
|
621
|
-
# mean cost of the misses in this same scope.
|
|
622
|
-
#
|
|
623
|
-
# ponytail: scope-wide mean, not per-agent. Group by agent_type if
|
|
624
|
-
# a mixed dashboard scope ever skews this enough to matter.
|
|
625
|
-
avg_miss_cost = miss_count.positive? ? (miss_cost.to_f / miss_count) : 0.0
|
|
626
|
-
|
|
627
|
-
{
|
|
628
|
-
count: cache_count,
|
|
629
|
-
estimated_savings: (cache_count * avg_miss_cost).round(6),
|
|
630
|
-
hit_rate: (cache_count.to_f / total_count * 100).round(1),
|
|
631
|
-
total_executions: total_count
|
|
632
|
-
}
|
|
668
|
+
scope.breakdown.cache_savings
|
|
633
669
|
end
|
|
634
670
|
|
|
635
671
|
# Batch fetches execution stats grouped by agent type
|
|
636
|
-
# Optimized: Single SQL GROUP BY with conditional aggregation
|
|
637
672
|
#
|
|
638
673
|
# @param scope [ActiveRecord::Relation] Pre-filtered scope
|
|
639
674
|
# @return [Hash<String, Hash>] Agent type => stats hash
|
|
640
675
|
def batch_agent_stats(scope: all)
|
|
641
|
-
|
|
642
|
-
:agent_type,
|
|
643
|
-
Arel.sql("COUNT(*) AS exec_count"),
|
|
644
|
-
Arel.sql("COALESCE(SUM(total_cost), 0) AS sum_cost"),
|
|
645
|
-
Arel.sql("AVG(duration_ms) AS avg_dur"),
|
|
646
|
-
Arel.sql("SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS success_cnt")
|
|
647
|
-
).group(:agent_type)
|
|
648
|
-
|
|
649
|
-
rows.each_with_object({}) do |row, hash|
|
|
650
|
-
count = row["exec_count"].to_i
|
|
651
|
-
total_cost = row["sum_cost"].to_f
|
|
652
|
-
successful = row["success_cnt"].to_i
|
|
653
|
-
|
|
654
|
-
hash[row.agent_type] = {
|
|
655
|
-
count: count,
|
|
656
|
-
total_cost: total_cost,
|
|
657
|
-
avg_cost: (count > 0) ? (total_cost / count).round(6) : 0,
|
|
658
|
-
avg_duration_ms: row["avg_dur"].to_i,
|
|
659
|
-
success_rate: (count > 0) ? (successful.to_f / count * 100).round(1) : 0
|
|
660
|
-
}
|
|
661
|
-
end
|
|
676
|
+
scope.breakdown.agent_stats
|
|
662
677
|
end
|
|
663
678
|
|
|
664
679
|
# Cached daily statistics for dashboard
|
|
@@ -790,16 +805,42 @@ module RubyLLM
|
|
|
790
805
|
|
|
791
806
|
# SQL condition for boolean cache_hit column
|
|
792
807
|
#
|
|
808
|
+
# @return [String] SQL condition fragment
|
|
809
|
+
def cache_hit_condition
|
|
810
|
+
true_condition("cache_hit")
|
|
811
|
+
end
|
|
812
|
+
|
|
813
|
+
# SQL condition for a boolean column being true
|
|
814
|
+
#
|
|
793
815
|
# SQLite stores booleans as 1/0, PostgreSQL as TRUE/FALSE.
|
|
794
816
|
#
|
|
817
|
+
# @param column [String] Column name (internal literal, never user input)
|
|
795
818
|
# @return [String] SQL condition fragment
|
|
796
|
-
def
|
|
819
|
+
def true_condition(column)
|
|
797
820
|
if connection.adapter_name.downcase.include?("sqlite")
|
|
798
|
-
"
|
|
821
|
+
"#{column} = 1"
|
|
799
822
|
else
|
|
800
|
-
"
|
|
823
|
+
"#{column} = TRUE"
|
|
801
824
|
end
|
|
802
825
|
end
|
|
826
|
+
|
|
827
|
+
# Percentage of the current scope where a boolean column is true
|
|
828
|
+
#
|
|
829
|
+
# One query (count plus conditional sum) rather than a count of the
|
|
830
|
+
# scope followed by a count of the filtered scope.
|
|
831
|
+
#
|
|
832
|
+
# @param column [String] Column name (internal literal, never user input)
|
|
833
|
+
# @return [Float] 0.0-100.0
|
|
834
|
+
def boolean_rate(column)
|
|
835
|
+
total, hits = pick(
|
|
836
|
+
Arel.sql("COUNT(*)"),
|
|
837
|
+
Arel.sql("SUM(CASE WHEN #{true_condition(column)} THEN 1 ELSE 0 END)")
|
|
838
|
+
)
|
|
839
|
+
total = total.to_i
|
|
840
|
+
return 0.0 if total.zero?
|
|
841
|
+
|
|
842
|
+
(hits.to_f / total * 100).round(1)
|
|
843
|
+
end
|
|
803
844
|
end
|
|
804
845
|
end
|
|
805
846
|
end
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyLLM
|
|
4
|
+
module Agents
|
|
5
|
+
class Execution
|
|
6
|
+
# Every dashboard figure for a set of executions, from one scan
|
|
7
|
+
#
|
|
8
|
+
# The dashboard and analytics pages each show half a dozen sections over
|
|
9
|
+
# the same time window: totals, per-agent stats, per-model stats, top
|
|
10
|
+
# errors, cache savings, error cost. Computed separately, that is one
|
|
11
|
+
# pass over the window per section — and on a table with millions of
|
|
12
|
+
# executions each pass takes seconds. A Breakdown reads the window once,
|
|
13
|
+
# grouped by the few low-cardinality columns the sections slice on, and
|
|
14
|
+
# derives each section from those groups in Ruby.
|
|
15
|
+
#
|
|
16
|
+
# @example
|
|
17
|
+
# breakdown = Execution.last_n_days(30).breakdown
|
|
18
|
+
# breakdown.totals #=> { total: 1200, cost: 4.2, success_rate: 97.1, ... }
|
|
19
|
+
# breakdown.model_stats #=> [{ model_id: "gpt-4o", executions: 800, ... }]
|
|
20
|
+
# breakdown.top_errors #=> [{ error_class: "Timeout::Error", count: 12, ... }]
|
|
21
|
+
#
|
|
22
|
+
# @see Execution::Analytics::ClassMethods#breakdown
|
|
23
|
+
# @api public
|
|
24
|
+
class Breakdown
|
|
25
|
+
Group = Struct.new(
|
|
26
|
+
:agent_type, :model_id, :billed_model_id, :status, :error_class,
|
|
27
|
+
:count, :cost, :tokens, :duration_sum, :duration_count,
|
|
28
|
+
:cache_hits, :miss_cost, :last_seen
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
# Raw grouped rows, as plain arrays so a breakdown can be cached and
|
|
32
|
+
# rebuilt with {.new}.
|
|
33
|
+
#
|
|
34
|
+
# @return [Array<Array>]
|
|
35
|
+
attr_reader :rows
|
|
36
|
+
|
|
37
|
+
# @param rows [Array<Array>] One array per group, in {Group} member
|
|
38
|
+
# order, as selected by {Execution::Analytics::ClassMethods#breakdown}
|
|
39
|
+
def initialize(rows)
|
|
40
|
+
@rows = rows
|
|
41
|
+
@groups = rows.map do |row|
|
|
42
|
+
Group.new(*row[0, 5], row[5].to_i, row[6].to_f, row[7].to_i, row[8].to_i,
|
|
43
|
+
row[9].to_i, row[10].to_i, row[11].to_f, parse_time(row[12]))
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# A breakdown narrowed to some agents
|
|
48
|
+
#
|
|
49
|
+
# @param agent_types [Array<String>] Agent class names to keep
|
|
50
|
+
# @return [Breakdown]
|
|
51
|
+
def for_agents(agent_types)
|
|
52
|
+
names = Array(agent_types)
|
|
53
|
+
self.class.new(@rows.select { |row| names.include?(row[0]) })
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Headline figures across the whole scope
|
|
57
|
+
#
|
|
58
|
+
# @return [Hash] :total, :success, :errors, :timeouts, :cost, :tokens,
|
|
59
|
+
# :avg_duration_ms, :success_rate, :error_rate, :last_seen
|
|
60
|
+
def totals
|
|
61
|
+
total = sum(@groups, :count)
|
|
62
|
+
success = sum(with_status("success"), :count)
|
|
63
|
+
errors = sum(with_status("error"), :count)
|
|
64
|
+
timeouts = sum(with_status("timeout"), :count)
|
|
65
|
+
|
|
66
|
+
{
|
|
67
|
+
total: total,
|
|
68
|
+
success: success,
|
|
69
|
+
errors: errors,
|
|
70
|
+
timeouts: timeouts,
|
|
71
|
+
cost: sum(@groups, :cost),
|
|
72
|
+
tokens: sum(@groups, :tokens),
|
|
73
|
+
avg_duration_ms: avg_duration(@groups),
|
|
74
|
+
success_rate: rate(success, total),
|
|
75
|
+
error_rate: rate(errors + timeouts, total),
|
|
76
|
+
last_seen: @groups.filter_map(&:last_seen).max
|
|
77
|
+
}
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Per-agent stats
|
|
81
|
+
#
|
|
82
|
+
# @return [Hash{String => Hash}] Agent type => :count, :total_cost,
|
|
83
|
+
# :total_tokens, :avg_cost, :avg_duration_ms, :success_rate
|
|
84
|
+
def agent_stats
|
|
85
|
+
@groups.group_by(&:agent_type).transform_values do |groups|
|
|
86
|
+
count = sum(groups, :count)
|
|
87
|
+
cost = sum(groups, :cost)
|
|
88
|
+
|
|
89
|
+
{
|
|
90
|
+
count: count,
|
|
91
|
+
total_cost: cost,
|
|
92
|
+
total_tokens: sum(groups, :tokens),
|
|
93
|
+
avg_cost: (count > 0) ? (cost / count).round(6) : 0,
|
|
94
|
+
avg_duration_ms: avg_duration(groups),
|
|
95
|
+
success_rate: rate(sum(groups.select { |g| g.status == "success" }, :count), count)
|
|
96
|
+
}
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Per-model stats, attributed to the model that actually ran
|
|
101
|
+
#
|
|
102
|
+
# @return [Array<Hash>] Sorted by total cost, most expensive first
|
|
103
|
+
def model_stats
|
|
104
|
+
overall_cost = sum(@groups, :cost)
|
|
105
|
+
|
|
106
|
+
@groups.group_by(&:billed_model_id).map do |model_id, groups|
|
|
107
|
+
count = sum(groups, :count)
|
|
108
|
+
cost = sum(groups, :cost)
|
|
109
|
+
tokens = sum(groups, :tokens)
|
|
110
|
+
|
|
111
|
+
{
|
|
112
|
+
model_id: model_id,
|
|
113
|
+
executions: count,
|
|
114
|
+
total_cost: cost,
|
|
115
|
+
total_tokens: tokens,
|
|
116
|
+
avg_duration_ms: avg_duration(groups),
|
|
117
|
+
success_rate: rate(sum(groups.select { |g| g.status == "success" }, :count), count),
|
|
118
|
+
cost_per_1k_tokens: (tokens > 0) ? (cost / tokens * 1000).round(4) : 0,
|
|
119
|
+
cost_percentage: (overall_cost > 0) ? (cost / overall_cost * 100).round(1) : 0
|
|
120
|
+
}
|
|
121
|
+
end.sort_by { |m| -m[:total_cost] }
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Usage per configured model, most expensive first
|
|
125
|
+
#
|
|
126
|
+
# Unlike {#model_stats} this groups on the model the agent asked for,
|
|
127
|
+
# which is what the "move these runs to a cheaper model" comparison
|
|
128
|
+
# needs.
|
|
129
|
+
#
|
|
130
|
+
# @return [Array<Hash>] :model_id, :runs, :cost, :tokens, :cost_per_run,
|
|
131
|
+
# :cost_per_1k_tokens
|
|
132
|
+
def configured_model_usage
|
|
133
|
+
@groups.group_by(&:model_id).map do |model_id, groups|
|
|
134
|
+
count = sum(groups, :count)
|
|
135
|
+
cost = sum(groups, :cost)
|
|
136
|
+
tokens = sum(groups, :tokens)
|
|
137
|
+
|
|
138
|
+
{
|
|
139
|
+
model_id: model_id,
|
|
140
|
+
runs: count,
|
|
141
|
+
cost: cost,
|
|
142
|
+
tokens: tokens,
|
|
143
|
+
cost_per_run: (count > 0) ? (cost / count) : 0,
|
|
144
|
+
cost_per_1k_tokens: (tokens > 0) ? (cost / tokens * 1000) : 0
|
|
145
|
+
}
|
|
146
|
+
end.sort_by { |m| -m[:cost] }
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Most frequent error classes
|
|
150
|
+
#
|
|
151
|
+
# @param limit [Integer] Max error classes to return
|
|
152
|
+
# @return [Array<Hash>] :error_class, :count, :percentage, :last_seen
|
|
153
|
+
def top_errors(limit: 5)
|
|
154
|
+
errors = with_status("error")
|
|
155
|
+
total = sum(errors, :count)
|
|
156
|
+
|
|
157
|
+
errors.group_by(&:error_class).map do |error_class, groups|
|
|
158
|
+
count = sum(groups, :count)
|
|
159
|
+
|
|
160
|
+
{
|
|
161
|
+
error_class: error_class || "Unknown Error",
|
|
162
|
+
count: count,
|
|
163
|
+
percentage: rate(count, total),
|
|
164
|
+
last_seen: groups.filter_map(&:last_seen).max
|
|
165
|
+
}
|
|
166
|
+
end.sort_by { |e| -e[:count] }.first(limit)
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Money spent on executions that ended in an error
|
|
170
|
+
#
|
|
171
|
+
# @param limit [Integer] Max (error class, agent) pairs to return
|
|
172
|
+
# @return [Hash] :total_cost, :total_count, and :breakdown — the
|
|
173
|
+
# costliest (error class, agent) pairs
|
|
174
|
+
def error_cost(limit: 10)
|
|
175
|
+
errors = with_status("error")
|
|
176
|
+
|
|
177
|
+
pairs = errors.group_by { |g| [g.error_class, g.agent_type] }.map do |(error_class, agent_type), groups|
|
|
178
|
+
{
|
|
179
|
+
error_class: error_class || "Unknown",
|
|
180
|
+
agent_type: agent_type,
|
|
181
|
+
count: sum(groups, :count),
|
|
182
|
+
cost: sum(groups, :cost),
|
|
183
|
+
last_seen: groups.filter_map(&:last_seen).max
|
|
184
|
+
}
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
{
|
|
188
|
+
total_cost: sum(errors, :cost),
|
|
189
|
+
total_count: sum(errors, :count),
|
|
190
|
+
breakdown: pairs.sort_by { |e| -e[:cost] }.first(limit)
|
|
191
|
+
}
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Cache hit rate and the spend those hits avoided
|
|
195
|
+
#
|
|
196
|
+
# A cache hit makes no API call, so its own cost is always zero; the
|
|
197
|
+
# saving is estimated at the mean cost of the misses in the same scope.
|
|
198
|
+
#
|
|
199
|
+
# @return [Hash] :count, :estimated_savings, :hit_rate, :total_executions
|
|
200
|
+
def cache_savings
|
|
201
|
+
total = sum(@groups, :count)
|
|
202
|
+
return {count: 0, estimated_savings: 0, hit_rate: 0, total_executions: 0} if total.zero?
|
|
203
|
+
|
|
204
|
+
hits = sum(@groups, :cache_hits)
|
|
205
|
+
misses = total - hits
|
|
206
|
+
avg_miss_cost = misses.positive? ? (sum(@groups, :miss_cost) / misses) : 0.0
|
|
207
|
+
|
|
208
|
+
{
|
|
209
|
+
count: hits,
|
|
210
|
+
estimated_savings: (hits * avg_miss_cost).round(6),
|
|
211
|
+
hit_rate: rate(hits, total),
|
|
212
|
+
total_executions: total
|
|
213
|
+
}
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
private
|
|
217
|
+
|
|
218
|
+
def with_status(status)
|
|
219
|
+
@groups.select { |g| g.status == status }
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
def sum(groups, field)
|
|
223
|
+
groups.sum { |g| g[field] }
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
def rate(part, whole)
|
|
227
|
+
(whole > 0) ? (part.to_f / whole * 100).round(1) : 0.0
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
def avg_duration(groups)
|
|
231
|
+
samples = sum(groups, :duration_count)
|
|
232
|
+
(samples > 0) ? (sum(groups, :duration_sum) / samples) : 0
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# MAX(created_at) comes back as a Time on PostgreSQL and as a UTC
|
|
236
|
+
# string on SQLite.
|
|
237
|
+
def parse_time(value)
|
|
238
|
+
value.is_a?(String) ? ActiveSupport::TimeZone["UTC"].parse(value) : value
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
end
|