ruby_llm-agents 3.11.0 → 3.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/controllers/ruby_llm/agents/agents_controller.rb +74 -0
- data/app/controllers/ruby_llm/agents/analytics_controller.rb +304 -0
- data/app/controllers/ruby_llm/agents/tenants_controller.rb +74 -2
- data/app/models/ruby_llm/agents/agent_override.rb +47 -0
- data/app/models/ruby_llm/agents/execution/analytics.rb +37 -16
- data/app/services/ruby_llm/agents/agent_registry.rb +8 -1
- data/app/views/layouts/ruby_llm/agents/application.html.erb +4 -2
- data/app/views/ruby_llm/agents/agents/_config_agent.html.erb +89 -4
- data/app/views/ruby_llm/agents/agents/show.html.erb +14 -0
- data/app/views/ruby_llm/agents/analytics/index.html.erb +398 -0
- data/app/views/ruby_llm/agents/tenants/index.html.erb +3 -2
- data/app/views/ruby_llm/agents/tenants/show.html.erb +225 -0
- data/config/routes.rb +12 -4
- data/lib/generators/ruby_llm_agents/templates/create_overrides_migration.rb.tt +28 -0
- data/lib/generators/ruby_llm_agents/templates/skills/AGENTS.md.tt +1 -1
- data/lib/generators/ruby_llm_agents/templates/skills/TOOLS.md.tt +1 -1
- data/lib/generators/ruby_llm_agents/upgrade_generator.rb +14 -0
- data/lib/ruby_llm/agents/base_agent.rb +90 -133
- data/lib/ruby_llm/agents/core/base.rb +9 -0
- data/lib/ruby_llm/agents/core/configuration.rb +5 -1
- data/lib/ruby_llm/agents/core/version.rb +1 -1
- data/lib/ruby_llm/agents/dsl/base.rb +131 -4
- data/lib/ruby_llm/agents/dsl/knowledge.rb +157 -0
- data/lib/ruby_llm/agents/dsl.rb +1 -1
- data/lib/ruby_llm/agents/pipeline/middleware/budget.rb +32 -20
- data/lib/ruby_llm/agents/pipeline/middleware/instrumentation.rb +22 -1
- data/lib/ruby_llm/agents/pipeline/middleware/reliability.rb +1 -1
- data/lib/ruby_llm/agents/stream_event.rb +2 -10
- data/lib/ruby_llm/agents/tool.rb +1 -1
- data/lib/ruby_llm/agents.rb +0 -3
- metadata +6 -3
- data/lib/ruby_llm/agents/agent_tool.rb +0 -143
- data/lib/ruby_llm/agents/dsl/agents.rb +0 -141
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5e29829dcedb6a67dd1d9e55c7a45ead560e5da2fc0f75ffd04c97026e8d5b90
|
|
4
|
+
data.tar.gz: 9c2981a6eed9459f2c7d56c47f6ff5ddcdd282bc90729ace04b8eca4f83e9e55
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 86ed72b1e799b64259b83588ac9c9111d04539eb7225e65d517332d22ed671436214255ec0127fdcff4ef65fe283683219e88b67a956c6270a7b6a0140937a27
|
|
7
|
+
data.tar.gz: b532f5a4a47814ed19a9929b06e506b56af651d8d1be65348c68b61aa78dc48f771c89c8d667dac06860d88283fee3a890b8d39afc4d11f5e6905ceca396c136
|
|
@@ -94,6 +94,62 @@ module RubyLLM
|
|
|
94
94
|
redirect_to ruby_llm_agents.agents_path, alert: "Error loading agent details"
|
|
95
95
|
end
|
|
96
96
|
|
|
97
|
+
# Saves dashboard overrides for an agent's overridable settings
|
|
98
|
+
#
|
|
99
|
+
# Only persists values for fields the agent has declared as
|
|
100
|
+
# `overridable: true` in its DSL. Ignores all other fields.
|
|
101
|
+
#
|
|
102
|
+
# @return [void]
|
|
103
|
+
def update
|
|
104
|
+
@agent_type = CGI.unescape(params[:id])
|
|
105
|
+
@agent_class = AgentRegistry.find(@agent_type)
|
|
106
|
+
|
|
107
|
+
unless @agent_class
|
|
108
|
+
redirect_to ruby_llm_agents.agents_path, alert: "Agent not found"
|
|
109
|
+
return
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
allowed = @agent_class.overridable_fields.map(&:to_s)
|
|
113
|
+
if allowed.empty?
|
|
114
|
+
redirect_to agent_path(@agent_type), alert: "This agent has no overridable fields"
|
|
115
|
+
return
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Build settings hash from permitted params, only for overridable fields
|
|
119
|
+
settings = {}
|
|
120
|
+
allowed.each do |field|
|
|
121
|
+
next unless params.dig(:override, field).present?
|
|
122
|
+
|
|
123
|
+
raw = params[:override][field]
|
|
124
|
+
settings[field] = coerce_override_value(field, raw)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
override = AgentOverride.find_or_initialize_by(agent_type: @agent_type)
|
|
128
|
+
|
|
129
|
+
if settings.empty?
|
|
130
|
+
# No overrides left — delete the record
|
|
131
|
+
override.destroy if override.persisted?
|
|
132
|
+
redirect_to agent_path(@agent_type), notice: "Overrides cleared"
|
|
133
|
+
else
|
|
134
|
+
override.settings = settings
|
|
135
|
+
if override.save
|
|
136
|
+
redirect_to agent_path(@agent_type), notice: "Overrides saved"
|
|
137
|
+
else
|
|
138
|
+
redirect_to agent_path(@agent_type), alert: "Failed to save overrides"
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Removes all dashboard overrides for an agent
|
|
144
|
+
#
|
|
145
|
+
# @return [void]
|
|
146
|
+
def reset_overrides
|
|
147
|
+
@agent_type = CGI.unescape(params[:id])
|
|
148
|
+
override = AgentOverride.find_by(agent_type: @agent_type)
|
|
149
|
+
override&.destroy
|
|
150
|
+
redirect_to agent_path(@agent_type), notice: "Overrides cleared"
|
|
151
|
+
end
|
|
152
|
+
|
|
97
153
|
private
|
|
98
154
|
|
|
99
155
|
# Loads all-time and today's statistics for the agent
|
|
@@ -294,6 +350,24 @@ module RubyLLM
|
|
|
294
350
|
|
|
295
351
|
(direction == "desc") ? sorted.reverse : sorted
|
|
296
352
|
end
|
|
353
|
+
|
|
354
|
+
# Coerces an override value from the form string to the appropriate Ruby type
|
|
355
|
+
#
|
|
356
|
+
# @param field [String] The field name
|
|
357
|
+
# @param raw [String] The raw string value from the form
|
|
358
|
+
# @return [Object] The coerced value
|
|
359
|
+
def coerce_override_value(field, raw)
|
|
360
|
+
case field
|
|
361
|
+
when "temperature"
|
|
362
|
+
raw.to_f
|
|
363
|
+
when "timeout"
|
|
364
|
+
raw.to_i
|
|
365
|
+
when "streaming"
|
|
366
|
+
ActiveModel::Type::Boolean.new.cast(raw)
|
|
367
|
+
else
|
|
368
|
+
raw.to_s
|
|
369
|
+
end
|
|
370
|
+
end
|
|
297
371
|
end
|
|
298
372
|
end
|
|
299
373
|
end
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyLLM
|
|
4
|
+
module Agents
|
|
5
|
+
# Controller for cost intelligence and analytics
|
|
6
|
+
#
|
|
7
|
+
# Provides interactive, filterable cost analysis across agents, models,
|
|
8
|
+
# and tenants. The filter bar (agent/model/tenant) applies to all sections,
|
|
9
|
+
# making this an exploration tool rather than a static report.
|
|
10
|
+
#
|
|
11
|
+
# @api private
|
|
12
|
+
class AnalyticsController < ApplicationController
|
|
13
|
+
VALID_RANGES = %w[7d 30d 90d custom].freeze
|
|
14
|
+
DEFAULT_RANGE = "30d"
|
|
15
|
+
|
|
16
|
+
def index
|
|
17
|
+
@selected_range = sanitize_range(params[:range])
|
|
18
|
+
@days = range_to_days(@selected_range)
|
|
19
|
+
parse_custom_dates if @selected_range == "custom"
|
|
20
|
+
|
|
21
|
+
load_filter_options
|
|
22
|
+
base = apply_filters(time_scoped(tenant_scoped_executions))
|
|
23
|
+
prior = apply_filters(prior_period_scope(tenant_scoped_executions))
|
|
24
|
+
|
|
25
|
+
load_summary(base, prior)
|
|
26
|
+
load_projection(base)
|
|
27
|
+
load_savings_opportunity(base)
|
|
28
|
+
load_efficiency(base)
|
|
29
|
+
load_error_cost(base)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Returns chart JSON: current period + prior period overlay
|
|
33
|
+
def chart_data
|
|
34
|
+
@selected_range = sanitize_range(params[:range])
|
|
35
|
+
@days = range_to_days(@selected_range)
|
|
36
|
+
parse_custom_dates if @selected_range == "custom"
|
|
37
|
+
|
|
38
|
+
current_scope = apply_filters(time_scoped(tenant_scoped_executions))
|
|
39
|
+
prior_scope = apply_filters(prior_period_scope(tenant_scoped_executions))
|
|
40
|
+
|
|
41
|
+
render json: build_overlay_chart_json(current_scope, prior_scope)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
# ── Time helpers ──────────────────────────
|
|
47
|
+
|
|
48
|
+
def sanitize_range(range)
|
|
49
|
+
VALID_RANGES.include?(range) ? range : DEFAULT_RANGE
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def range_to_days(range)
|
|
53
|
+
case range
|
|
54
|
+
when "7d" then 7
|
|
55
|
+
when "30d" then 30
|
|
56
|
+
when "90d" then 90
|
|
57
|
+
else 30
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def parse_date(value)
|
|
62
|
+
return nil if value.blank?
|
|
63
|
+
date = Date.parse(value)
|
|
64
|
+
return nil if date > Date.current
|
|
65
|
+
return nil if date < 1.year.ago.to_date
|
|
66
|
+
date
|
|
67
|
+
rescue ArgumentError
|
|
68
|
+
nil
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def parse_custom_dates
|
|
72
|
+
from = parse_date(params[:from])
|
|
73
|
+
to = parse_date(params[:to])
|
|
74
|
+
|
|
75
|
+
if from && to
|
|
76
|
+
from, to = [from, to].sort
|
|
77
|
+
@custom_from = from
|
|
78
|
+
@custom_to = [to, Date.current].min
|
|
79
|
+
@days = (@custom_to - @custom_from).to_i + 1
|
|
80
|
+
else
|
|
81
|
+
@selected_range = DEFAULT_RANGE
|
|
82
|
+
@days = 30
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def time_scoped(scope)
|
|
87
|
+
if @selected_range == "custom" && @custom_from && @custom_to
|
|
88
|
+
scope.where(created_at: @custom_from.beginning_of_day..@custom_to.end_of_day)
|
|
89
|
+
else
|
|
90
|
+
scope.last_n_days(@days)
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def prior_period_scope(scope)
|
|
95
|
+
if @selected_range == "custom" && @custom_from && @custom_to
|
|
96
|
+
duration = @custom_to - @custom_from + 1
|
|
97
|
+
prior_end = @custom_from - 1.day
|
|
98
|
+
prior_start = prior_end - duration.days + 1.day
|
|
99
|
+
scope.where(created_at: prior_start.beginning_of_day..prior_end.end_of_day)
|
|
100
|
+
else
|
|
101
|
+
scope.where(created_at: (@days * 2).days.ago...@days.days.ago)
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# ── Filters ──────────────────────────
|
|
106
|
+
|
|
107
|
+
def load_filter_options
|
|
108
|
+
base = tenant_scoped_executions
|
|
109
|
+
@available_agents = base.where.not(agent_type: nil).distinct.pluck(:agent_type).sort
|
|
110
|
+
@available_models = base.where.not(model_id: nil).distinct.pluck(:model_id).sort
|
|
111
|
+
@available_tenants = if tenant_filter_enabled? && Tenant.table_exists?
|
|
112
|
+
Tenant.pluck(:tenant_id, :name).map { |tid, name| [tid, name.presence || tid] }.sort_by(&:last)
|
|
113
|
+
else
|
|
114
|
+
[]
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
@filter_agent = params[:agent].presence
|
|
118
|
+
@filter_model = params[:model].presence
|
|
119
|
+
@filter_tenant = params[:filter_tenant].presence
|
|
120
|
+
@any_filter = @filter_agent || @filter_model || @filter_tenant
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def apply_filters(scope)
|
|
124
|
+
scope = scope.where(agent_type: @filter_agent) if @filter_agent.present?
|
|
125
|
+
scope = scope.where(model_id: @filter_model) if @filter_model.present?
|
|
126
|
+
scope = scope.where(tenant_id: @filter_tenant) if @filter_tenant.present?
|
|
127
|
+
scope
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# ── Data loaders ──────────────────────────
|
|
131
|
+
|
|
132
|
+
def load_summary(base, prior)
|
|
133
|
+
current_agg = aggregate(base)
|
|
134
|
+
prior_agg = aggregate(prior)
|
|
135
|
+
|
|
136
|
+
@summary = {
|
|
137
|
+
total_cost: current_agg[:cost],
|
|
138
|
+
total_runs: current_agg[:count],
|
|
139
|
+
total_tokens: current_agg[:tokens],
|
|
140
|
+
avg_cost: (current_agg[:count] > 0) ? (current_agg[:cost].to_f / current_agg[:count]) : 0,
|
|
141
|
+
avg_tokens: (current_agg[:count] > 0) ? (current_agg[:tokens].to_f / current_agg[:count]).round : 0,
|
|
142
|
+
cost_change: pct_change(prior_agg[:cost], current_agg[:cost]),
|
|
143
|
+
runs_change: pct_change(prior_agg[:count], current_agg[:count]),
|
|
144
|
+
prior_cost: prior_agg[:cost],
|
|
145
|
+
prior_runs: prior_agg[:count]
|
|
146
|
+
}
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def load_projection(base)
|
|
150
|
+
return @projection = nil if @days.nil? || @summary[:total_cost].zero?
|
|
151
|
+
|
|
152
|
+
# Calculate daily burn rate and project to end of month
|
|
153
|
+
daily_rate = @summary[:total_cost].to_f / @days
|
|
154
|
+
days_left = (Date.current.end_of_month - Date.current).to_i
|
|
155
|
+
month_so_far = base.where("created_at >= ?", Date.current.beginning_of_month.beginning_of_day)
|
|
156
|
+
.sum(:total_cost).to_f
|
|
157
|
+
|
|
158
|
+
@projection = {
|
|
159
|
+
daily_rate: daily_rate,
|
|
160
|
+
month_so_far: month_so_far,
|
|
161
|
+
projected_month: month_so_far + (daily_rate * days_left),
|
|
162
|
+
days_left: days_left
|
|
163
|
+
}
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def load_savings_opportunity(base)
|
|
167
|
+
# Find the most expensive model and suggest switching to the cheapest
|
|
168
|
+
models = base.where.not(model_id: nil)
|
|
169
|
+
.select(
|
|
170
|
+
:model_id,
|
|
171
|
+
Arel.sql("COUNT(*) AS exec_count"),
|
|
172
|
+
Arel.sql("COALESCE(SUM(total_cost), 0) AS sum_cost"),
|
|
173
|
+
Arel.sql("COALESCE(SUM(total_tokens), 0) AS sum_tokens")
|
|
174
|
+
)
|
|
175
|
+
.group(:model_id)
|
|
176
|
+
.order(Arel.sql("sum_cost DESC"))
|
|
177
|
+
|
|
178
|
+
model_data = models.map do |row|
|
|
179
|
+
count = row["exec_count"].to_i
|
|
180
|
+
cost = row["sum_cost"].to_f
|
|
181
|
+
tokens = row["sum_tokens"].to_i
|
|
182
|
+
{
|
|
183
|
+
model_id: row.model_id,
|
|
184
|
+
runs: count,
|
|
185
|
+
cost: cost,
|
|
186
|
+
tokens: tokens,
|
|
187
|
+
cost_per_run: (count > 0) ? (cost / count) : 0,
|
|
188
|
+
cost_per_1k_tokens: (tokens > 0) ? (cost / tokens * 1000) : 0
|
|
189
|
+
}
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
@savings = nil
|
|
193
|
+
return if model_data.size < 2
|
|
194
|
+
|
|
195
|
+
expensive = model_data.first
|
|
196
|
+
cheapest = model_data.min_by { |m| m[:cost_per_run] }
|
|
197
|
+
|
|
198
|
+
return if expensive[:model_id] == cheapest[:model_id]
|
|
199
|
+
return if expensive[:cost_per_run] <= cheapest[:cost_per_run]
|
|
200
|
+
|
|
201
|
+
potential_savings = (expensive[:cost_per_run] - cheapest[:cost_per_run]) * expensive[:runs]
|
|
202
|
+
return if potential_savings < 0.001
|
|
203
|
+
|
|
204
|
+
@savings = {
|
|
205
|
+
expensive_model: expensive[:model_id],
|
|
206
|
+
expensive_runs: expensive[:runs],
|
|
207
|
+
expensive_cost_per_run: expensive[:cost_per_run],
|
|
208
|
+
cheap_model: cheapest[:model_id],
|
|
209
|
+
cheap_cost_per_run: cheapest[:cost_per_run],
|
|
210
|
+
potential_savings: potential_savings
|
|
211
|
+
}
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
def load_efficiency(base)
|
|
215
|
+
@efficiency = Execution.model_stats(scope: base)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def load_error_cost(base)
|
|
219
|
+
error_scope = base.where(status: "error")
|
|
220
|
+
@error_total_cost = error_scope.sum(:total_cost) || 0
|
|
221
|
+
@error_total_count = error_scope.count
|
|
222
|
+
|
|
223
|
+
@error_breakdown = error_scope
|
|
224
|
+
.select(
|
|
225
|
+
:error_class,
|
|
226
|
+
:agent_type,
|
|
227
|
+
Arel.sql("COUNT(*) AS err_count"),
|
|
228
|
+
Arel.sql("COALESCE(SUM(total_cost), 0) AS err_cost"),
|
|
229
|
+
Arel.sql("MAX(created_at) AS last_seen")
|
|
230
|
+
)
|
|
231
|
+
.group(:error_class, :agent_type)
|
|
232
|
+
.order(Arel.sql("err_cost DESC"))
|
|
233
|
+
.limit(10)
|
|
234
|
+
.map do |row|
|
|
235
|
+
{
|
|
236
|
+
error_class: row.error_class || "Unknown",
|
|
237
|
+
agent_type: row.agent_type,
|
|
238
|
+
count: row["err_count"].to_i,
|
|
239
|
+
cost: row["err_cost"].to_f,
|
|
240
|
+
last_seen: row["last_seen"]
|
|
241
|
+
}
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# ── Helpers ──────────────────────────
|
|
246
|
+
|
|
247
|
+
def aggregate(scope)
|
|
248
|
+
result = scope.pick(
|
|
249
|
+
Arel.sql("COUNT(*)"),
|
|
250
|
+
Arel.sql("COALESCE(SUM(total_cost), 0)"),
|
|
251
|
+
Arel.sql("COALESCE(SUM(total_tokens), 0)")
|
|
252
|
+
)
|
|
253
|
+
{count: result[0].to_i, cost: result[1].to_f, tokens: result[2].to_i}
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def pct_change(old_val, new_val)
|
|
257
|
+
return 0.0 if old_val.nil? || old_val.to_f.zero?
|
|
258
|
+
((new_val.to_f - old_val.to_f) / old_val.to_f * 100).round(1)
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
def date_bucket_sql
|
|
262
|
+
if ::ActiveRecord::Base.connection.adapter_name.downcase.include?("sqlite")
|
|
263
|
+
"strftime('%Y-%m-%d', created_at)"
|
|
264
|
+
else
|
|
265
|
+
"date_trunc('day', created_at)::date"
|
|
266
|
+
end
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
# Builds current + prior period overlay chart
|
|
270
|
+
def build_overlay_chart_json(current_scope, prior_scope)
|
|
271
|
+
current_daily = daily_cost(current_scope)
|
|
272
|
+
prior_daily = daily_cost(prior_scope)
|
|
273
|
+
|
|
274
|
+
current_series = current_daily.sort.map { |date, cost| [date_to_ms(date), cost.round(6)] }
|
|
275
|
+
# Shift prior dates forward so they overlay on the same x-axis
|
|
276
|
+
prior_series = if prior_daily.any? && current_daily.any?
|
|
277
|
+
offset = (Date.parse(current_daily.keys.min) - Date.parse(prior_daily.keys.min)).to_i
|
|
278
|
+
prior_daily.sort.map { |date, cost| [date_to_ms(date, offset_days: offset), cost.round(6)] }
|
|
279
|
+
else
|
|
280
|
+
[]
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
{
|
|
284
|
+
series: [
|
|
285
|
+
{name: "Current period", data: current_series, type: "areaspline"},
|
|
286
|
+
{name: "Prior period", data: prior_series, type: "spline", dashStyle: "Dash"}
|
|
287
|
+
]
|
|
288
|
+
}
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
def daily_cost(scope)
|
|
292
|
+
scope.select(
|
|
293
|
+
Arel.sql("#{date_bucket_sql} AS bucket"),
|
|
294
|
+
Arel.sql("COALESCE(SUM(total_cost), 0) AS sum_cost")
|
|
295
|
+
).group(Arel.sql("bucket"))
|
|
296
|
+
.each_with_object({}) { |row, h| h[row["bucket"].to_s] = row["sum_cost"].to_f }
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
def date_to_ms(date_str, offset_days: 0)
|
|
300
|
+
(Date.parse(date_str) + offset_days.days).strftime("%s").to_i * 1000
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
end
|
|
304
|
+
end
|
|
@@ -23,13 +23,15 @@ module RubyLLM
|
|
|
23
23
|
|
|
24
24
|
if params[:q].present?
|
|
25
25
|
@search_query = params[:q].to_s.strip
|
|
26
|
+
escaped = TenantBudget.sanitize_sql_like(@search_query)
|
|
26
27
|
scope = scope.where(
|
|
27
|
-
"tenant_id LIKE :q OR name LIKE :q",
|
|
28
|
-
q: "%#{
|
|
28
|
+
"tenant_id LIKE :q OR name LIKE :q OR tenant_record_type LIKE :q OR tenant_record_id LIKE :q",
|
|
29
|
+
q: "%#{escaped}%"
|
|
29
30
|
)
|
|
30
31
|
end
|
|
31
32
|
|
|
32
33
|
@tenants = scope.order(@sort_params[:column] => @sort_params[:direction].to_sym)
|
|
34
|
+
preload_tenant_index_data
|
|
33
35
|
end
|
|
34
36
|
|
|
35
37
|
# Shows a single tenant's budget details
|
|
@@ -39,6 +41,9 @@ module RubyLLM
|
|
|
39
41
|
@tenant = TenantBudget.find(params[:id])
|
|
40
42
|
@executions = tenant_executions(@tenant.tenant_id).recent.limit(10)
|
|
41
43
|
@usage_stats = calculate_usage_stats(@tenant)
|
|
44
|
+
@usage_by_agent = @tenant.usage_by_agent(period: nil)
|
|
45
|
+
@usage_by_model = @tenant.usage_by_model(period: nil)
|
|
46
|
+
load_tenant_analytics
|
|
42
47
|
end
|
|
43
48
|
|
|
44
49
|
# Renders the edit form for a tenant budget
|
|
@@ -48,6 +53,15 @@ module RubyLLM
|
|
|
48
53
|
@tenant = TenantBudget.find(params[:id])
|
|
49
54
|
end
|
|
50
55
|
|
|
56
|
+
# Recalculates budget counters from the executions table
|
|
57
|
+
#
|
|
58
|
+
# @return [void]
|
|
59
|
+
def refresh_counters
|
|
60
|
+
@tenant = TenantBudget.find(params[:id])
|
|
61
|
+
@tenant.refresh_counters!
|
|
62
|
+
redirect_to tenant_path(@tenant), notice: "Counters refreshed"
|
|
63
|
+
end
|
|
64
|
+
|
|
51
65
|
# Updates a tenant budget
|
|
52
66
|
#
|
|
53
67
|
# @return [void]
|
|
@@ -110,6 +124,64 @@ module RubyLLM
|
|
|
110
124
|
}
|
|
111
125
|
end
|
|
112
126
|
|
|
127
|
+
# Loads trend data and period comparison for the tenant analytics section.
|
|
128
|
+
#
|
|
129
|
+
# @return [void]
|
|
130
|
+
def load_tenant_analytics
|
|
131
|
+
# 30-day daily cost/tokens trend
|
|
132
|
+
@daily_trend = @tenant.usage_by_day(period: 30.days.ago..Time.current)
|
|
133
|
+
|
|
134
|
+
# Period comparison: this month vs last month
|
|
135
|
+
this_month = @tenant.usage_summary(period: :this_month)
|
|
136
|
+
last_month = @tenant.usage_summary(period: :last_month)
|
|
137
|
+
@period_comparison = {
|
|
138
|
+
this_month: this_month,
|
|
139
|
+
last_month: last_month,
|
|
140
|
+
cost_change: percent_change(last_month[:cost], this_month[:cost]),
|
|
141
|
+
tokens_change: percent_change(last_month[:tokens], this_month[:tokens]),
|
|
142
|
+
executions_change: percent_change(last_month[:executions], this_month[:executions]),
|
|
143
|
+
avg_cost_this: (this_month[:executions] > 0) ? (this_month[:cost].to_f / this_month[:executions]) : 0,
|
|
144
|
+
avg_cost_last: (last_month[:executions] > 0) ? (last_month[:cost].to_f / last_month[:executions]) : 0
|
|
145
|
+
}
|
|
146
|
+
@period_comparison[:avg_cost_change] = percent_change(
|
|
147
|
+
@period_comparison[:avg_cost_last], @period_comparison[:avg_cost_this]
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
# Error cost: money spent on failed executions
|
|
151
|
+
error_scope = @tenant.executions.where(status: "error")
|
|
152
|
+
@error_cost = error_scope.sum(:total_cost) || 0
|
|
153
|
+
@error_count = error_scope.count
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# Calculates percentage change between two values
|
|
157
|
+
#
|
|
158
|
+
# @return [Float]
|
|
159
|
+
def percent_change(old_val, new_val)
|
|
160
|
+
return 0.0 if old_val.nil? || old_val.to_f.zero?
|
|
161
|
+
((new_val.to_f - old_val.to_f) / old_val.to_f * 100).round(1)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Preloads cost and last-execution data for all tenants in one query each,
|
|
165
|
+
# avoiding N+1 queries in the index view.
|
|
166
|
+
#
|
|
167
|
+
# @return [void]
|
|
168
|
+
def preload_tenant_index_data
|
|
169
|
+
@tenant_costs = {}
|
|
170
|
+
@tenant_last_executions = {}
|
|
171
|
+
tenant_ids = @tenants.map(&:tenant_id)
|
|
172
|
+
return if tenant_ids.empty?
|
|
173
|
+
|
|
174
|
+
# Batch-load total cost per tenant
|
|
175
|
+
@tenant_costs = Execution.where(tenant_id: tenant_ids)
|
|
176
|
+
.group(:tenant_id)
|
|
177
|
+
.sum(:total_cost)
|
|
178
|
+
|
|
179
|
+
# Batch-load last execution time per tenant
|
|
180
|
+
@tenant_last_executions = Execution.where(tenant_id: tenant_ids)
|
|
181
|
+
.group(:tenant_id)
|
|
182
|
+
.maximum(:created_at)
|
|
183
|
+
end
|
|
184
|
+
|
|
113
185
|
# Parses and validates sort parameters for tenants list
|
|
114
186
|
#
|
|
115
187
|
# @return [Hash] Contains :column and :direction keys
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyLLM
|
|
4
|
+
module Agents
|
|
5
|
+
# Stores dashboard-managed overrides for agent settings.
|
|
6
|
+
#
|
|
7
|
+
# When an agent declares a field as `overridable: true` in its DSL,
|
|
8
|
+
# the dashboard can persist an override value in this table. The DSL
|
|
9
|
+
# getter merges the override on top of the code-defined default.
|
|
10
|
+
#
|
|
11
|
+
# Each row maps one agent class name to a JSON hash of overridden fields.
|
|
12
|
+
#
|
|
13
|
+
# @example
|
|
14
|
+
# AgentOverride.create!(
|
|
15
|
+
# agent_type: "SupportAgent",
|
|
16
|
+
# settings: { "model" => "claude-sonnet-4-5", "temperature" => 0.3 }
|
|
17
|
+
# )
|
|
18
|
+
#
|
|
19
|
+
# @see DSL::Base#resolve_override
|
|
20
|
+
# @api private
|
|
21
|
+
class AgentOverride < ::ActiveRecord::Base
|
|
22
|
+
self.table_name = "ruby_llm_agents_overrides"
|
|
23
|
+
|
|
24
|
+
validates :agent_type, presence: true, uniqueness: true
|
|
25
|
+
|
|
26
|
+
after_save :bust_agent_cache
|
|
27
|
+
after_destroy :bust_agent_cache
|
|
28
|
+
|
|
29
|
+
# Returns the override value for a single field, or nil
|
|
30
|
+
#
|
|
31
|
+
# @param field [Symbol, String] The field name
|
|
32
|
+
# @return [Object, nil] The override value
|
|
33
|
+
def [](field)
|
|
34
|
+
(settings || {})[field.to_s]
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
# Clears the in-memory override cache on the agent class so the
|
|
40
|
+
# next call picks up the new values.
|
|
41
|
+
def bust_agent_cache
|
|
42
|
+
klass = agent_type.safe_constantize
|
|
43
|
+
klass&.clear_override_cache! if klass.respond_to?(:clear_override_cache!)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -30,16 +30,31 @@ module RubyLLM
|
|
|
30
30
|
def daily_report
|
|
31
31
|
scope = today
|
|
32
32
|
|
|
33
|
+
# Single aggregated query for core metrics
|
|
34
|
+
total, successful, failed, cost, tokens, avg_dur, avg_tok = scope.pick(
|
|
35
|
+
Arel.sql("COUNT(*)"),
|
|
36
|
+
Arel.sql("SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END)"),
|
|
37
|
+
Arel.sql("SUM(CASE WHEN status IN ('error','timeout') THEN 1 ELSE 0 END)"),
|
|
38
|
+
Arel.sql("COALESCE(SUM(total_cost), 0)"),
|
|
39
|
+
Arel.sql("COALESCE(SUM(total_tokens), 0)"),
|
|
40
|
+
Arel.sql("AVG(duration_ms)"),
|
|
41
|
+
Arel.sql("AVG(total_tokens)")
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
total = total.to_i
|
|
45
|
+
successful = successful.to_i
|
|
46
|
+
failed = failed.to_i
|
|
47
|
+
|
|
33
48
|
{
|
|
34
49
|
date: Date.current,
|
|
35
|
-
total_executions:
|
|
36
|
-
successful:
|
|
37
|
-
failed:
|
|
38
|
-
total_cost:
|
|
39
|
-
total_tokens:
|
|
40
|
-
avg_duration_ms:
|
|
41
|
-
avg_tokens:
|
|
42
|
-
error_rate:
|
|
50
|
+
total_executions: total,
|
|
51
|
+
successful: successful,
|
|
52
|
+
failed: failed,
|
|
53
|
+
total_cost: cost.to_f,
|
|
54
|
+
total_tokens: tokens.to_i,
|
|
55
|
+
avg_duration_ms: avg_dur.to_i,
|
|
56
|
+
avg_tokens: avg_tok.to_i,
|
|
57
|
+
error_rate: (total > 0) ? (failed.to_f / total * 100).round(2) : 0.0,
|
|
43
58
|
by_agent: scope.group(:agent_type).count,
|
|
44
59
|
top_errors: scope.errors.group(:error_class).count.sort_by { |_, v| -v }.first(5).to_h
|
|
45
60
|
}
|
|
@@ -89,19 +104,25 @@ module RubyLLM
|
|
|
89
104
|
# @return [Array<Hash>] Daily metrics sorted oldest to newest
|
|
90
105
|
def trend_analysis(agent_type: nil, days: 7)
|
|
91
106
|
scope = agent_type ? by_agent(agent_type) : all
|
|
107
|
+
end_date = Date.current
|
|
108
|
+
start_date = end_date - (days - 1).days
|
|
109
|
+
|
|
110
|
+
time_scope = scope.where(created_at: start_date.beginning_of_day..end_date.end_of_day)
|
|
111
|
+
results = aggregated_chart_query(time_scope, granularity: :day)
|
|
92
112
|
|
|
93
|
-
(0...days).map do |
|
|
94
|
-
date =
|
|
95
|
-
|
|
113
|
+
(0...days).map do |i|
|
|
114
|
+
date = start_date + i.days
|
|
115
|
+
key = date.to_s
|
|
116
|
+
row = results[key] || {success: 0, failed: 0, cost: 0.0, duration: 0, tokens: 0}
|
|
96
117
|
|
|
97
118
|
{
|
|
98
119
|
date: date,
|
|
99
|
-
count:
|
|
100
|
-
total_cost:
|
|
101
|
-
avg_duration_ms:
|
|
102
|
-
error_count:
|
|
120
|
+
count: row[:success] + row[:failed],
|
|
121
|
+
total_cost: row[:cost],
|
|
122
|
+
avg_duration_ms: row[:duration],
|
|
123
|
+
error_count: row[:failed]
|
|
103
124
|
}
|
|
104
|
-
end
|
|
125
|
+
end
|
|
105
126
|
end
|
|
106
127
|
|
|
107
128
|
# Builds hourly activity chart data for today
|
|
@@ -70,7 +70,14 @@ module RubyLLM
|
|
|
70
70
|
}
|
|
71
71
|
|
|
72
72
|
type = detect_agent_type(agent_class)
|
|
73
|
-
base.merge(type_config_for(agent_class, type))
|
|
73
|
+
config = base.merge(type_config_for(agent_class, type))
|
|
74
|
+
|
|
75
|
+
# Include dashboard override metadata
|
|
76
|
+
config[:overridable_fields] = safe_call(agent_class, :overridable_fields) || []
|
|
77
|
+
config[:active_overrides] = safe_call(agent_class, :active_overrides) || {}
|
|
78
|
+
config[:has_overrides] = config[:active_overrides].any?
|
|
79
|
+
|
|
80
|
+
config
|
|
74
81
|
end
|
|
75
82
|
|
|
76
83
|
private
|
|
@@ -296,7 +296,8 @@
|
|
|
296
296
|
[ruby_llm_agents.root_path, "dashboard"],
|
|
297
297
|
[ruby_llm_agents.agents_path, "agents"],
|
|
298
298
|
[ruby_llm_agents.executions_path, "executions"],
|
|
299
|
-
[ruby_llm_agents.requests_path, "requests"]
|
|
299
|
+
[ruby_llm_agents.requests_path, "requests"],
|
|
300
|
+
[ruby_llm_agents.analytics_path, "analytics"]
|
|
300
301
|
]
|
|
301
302
|
nav_items << [ruby_llm_agents.tenants_path, "tenants"] if tenant_filter_enabled?
|
|
302
303
|
nav_items.each do |path, label| %>
|
|
@@ -347,7 +348,8 @@
|
|
|
347
348
|
[ruby_llm_agents.root_path, "dashboard"],
|
|
348
349
|
[ruby_llm_agents.agents_path, "agents"],
|
|
349
350
|
[ruby_llm_agents.executions_path, "executions"],
|
|
350
|
-
[ruby_llm_agents.requests_path, "requests"]
|
|
351
|
+
[ruby_llm_agents.requests_path, "requests"],
|
|
352
|
+
[ruby_llm_agents.analytics_path, "analytics"]
|
|
351
353
|
]
|
|
352
354
|
mobile_nav_items << [ruby_llm_agents.tenants_path, "tenants"] if tenant_filter_enabled?
|
|
353
355
|
mobile_nav_items.each do |path, label| %>
|