ruby_llm-agents 3.5.5 → 3.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +21 -0
- data/app/controllers/ruby_llm/agents/dashboard_controller.rb +155 -10
- data/app/controllers/ruby_llm/agents/executions_controller.rb +1 -3
- data/app/helpers/ruby_llm/agents/application_helper.rb +15 -28
- data/app/models/ruby_llm/agents/execution/replayable.rb +124 -0
- data/app/models/ruby_llm/agents/execution/scopes.rb +42 -1
- data/app/models/ruby_llm/agents/execution.rb +50 -1
- data/app/models/ruby_llm/agents/tenant/budgetable.rb +28 -4
- data/app/views/layouts/ruby_llm/agents/application.html.erb +41 -28
- data/app/views/ruby_llm/agents/agents/show.html.erb +16 -1
- data/app/views/ruby_llm/agents/dashboard/_top_tenants.html.erb +47 -0
- data/app/views/ruby_llm/agents/dashboard/index.html.erb +404 -107
- data/app/views/ruby_llm/agents/system_config/show.html.erb +0 -13
- data/lib/generators/ruby_llm_agents/rename_agent_generator.rb +53 -0
- data/lib/generators/ruby_llm_agents/templates/initializer.rb.tt +0 -15
- data/lib/generators/ruby_llm_agents/templates/rename_agent_migration.rb.tt +19 -0
- data/lib/ruby_llm/agents/agent_tool.rb +125 -0
- data/lib/ruby_llm/agents/audio/speaker.rb +5 -3
- data/lib/ruby_llm/agents/audio/speech_pricing.rb +63 -187
- data/lib/ruby_llm/agents/audio/transcriber.rb +5 -3
- data/lib/ruby_llm/agents/audio/transcription_pricing.rb +5 -7
- data/lib/ruby_llm/agents/base_agent.rb +144 -5
- data/lib/ruby_llm/agents/core/configuration.rb +178 -53
- data/lib/ruby_llm/agents/core/errors.rb +3 -77
- data/lib/ruby_llm/agents/core/instrumentation.rb +0 -17
- data/lib/ruby_llm/agents/core/version.rb +1 -1
- data/lib/ruby_llm/agents/dsl/base.rb +0 -8
- data/lib/ruby_llm/agents/dsl/queryable.rb +124 -0
- data/lib/ruby_llm/agents/dsl.rb +1 -0
- data/lib/ruby_llm/agents/eval/eval_result.rb +73 -0
- data/lib/ruby_llm/agents/eval/eval_run.rb +124 -0
- data/lib/ruby_llm/agents/eval/eval_suite.rb +264 -0
- data/lib/ruby_llm/agents/eval.rb +5 -0
- data/lib/ruby_llm/agents/image/concerns/image_operation_execution.rb +2 -1
- data/lib/ruby_llm/agents/image/generator/pricing.rb +75 -217
- data/lib/ruby_llm/agents/image/generator.rb +5 -3
- data/lib/ruby_llm/agents/infrastructure/attempt_tracker.rb +8 -0
- data/lib/ruby_llm/agents/infrastructure/circuit_breaker.rb +4 -2
- data/lib/ruby_llm/agents/pipeline/builder.rb +43 -0
- data/lib/ruby_llm/agents/pipeline/context.rb +11 -1
- data/lib/ruby_llm/agents/pipeline/executor.rb +1 -25
- data/lib/ruby_llm/agents/pipeline/middleware/budget.rb +26 -1
- data/lib/ruby_llm/agents/pipeline/middleware/cache.rb +18 -0
- data/lib/ruby_llm/agents/pipeline/middleware/instrumentation.rb +90 -0
- data/lib/ruby_llm/agents/pipeline/middleware/reliability.rb +29 -0
- data/lib/ruby_llm/agents/pipeline/middleware/tenant.rb +11 -4
- data/lib/ruby_llm/agents/pipeline.rb +0 -92
- data/lib/ruby_llm/agents/results/background_removal_result.rb +11 -1
- data/lib/ruby_llm/agents/results/base.rb +23 -1
- data/lib/ruby_llm/agents/results/embedding_result.rb +14 -1
- data/lib/ruby_llm/agents/results/image_analysis_result.rb +11 -1
- data/lib/ruby_llm/agents/results/image_edit_result.rb +11 -1
- data/lib/ruby_llm/agents/results/image_generation_result.rb +12 -3
- data/lib/ruby_llm/agents/results/image_pipeline_result.rb +11 -1
- data/lib/ruby_llm/agents/results/image_transform_result.rb +11 -1
- data/lib/ruby_llm/agents/results/image_upscale_result.rb +11 -1
- data/lib/ruby_llm/agents/results/image_variation_result.rb +11 -1
- data/lib/ruby_llm/agents/results/speech_result.rb +20 -1
- data/lib/ruby_llm/agents/results/transcription_result.rb +20 -1
- data/lib/ruby_llm/agents/text/embedder.rb +23 -18
- data/lib/ruby_llm/agents.rb +73 -5
- data/lib/tasks/ruby_llm_agents.rake +21 -0
- metadata +11 -6
- data/lib/ruby_llm/agents/infrastructure/reliability/breaker_manager.rb +0 -80
- data/lib/ruby_llm/agents/infrastructure/reliability/execution_constraints.rb +0 -69
- data/lib/ruby_llm/agents/infrastructure/reliability/executor.rb +0 -125
- data/lib/ruby_llm/agents/infrastructure/reliability/fallback_routing.rb +0 -72
- data/lib/ruby_llm/agents/infrastructure/reliability/retry_strategy.rb +0 -82
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ce728e318b0681df1f65dc93e4c264f35573e863b86841394ab218994dd3dd29
|
|
4
|
+
data.tar.gz: f036ab7df822277740a0f840afd52d3d68c36bbd37843ae16032da7f9406864e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6e63acc86ac7413983957abc46bbf5ba997230ae06480d6aa2eb77d6f25524e02cc58131112cda0b5eb413452078766d8afec001d98e56e6fb3ae2e8a0602dff
|
|
7
|
+
data.tar.gz: d01dba7531c3e503f7b00156a254c8a71ab582d20b194ad22d1f6bc1734cdd38b814e6581855274e08b49bb2c8d33e7655a02c2ed834771d9a990085ce743c84
|
data/README.md
CHANGED
|
@@ -162,6 +162,21 @@ result.url # => "https://..."
|
|
|
162
162
|
result.save("logo.png")
|
|
163
163
|
```
|
|
164
164
|
|
|
165
|
+
```ruby
|
|
166
|
+
# Evaluate agent quality with built-in scoring
|
|
167
|
+
class SupportRouter::Eval < RubyLLM::Agents::Eval::EvalSuite
|
|
168
|
+
agent SupportRouter
|
|
169
|
+
|
|
170
|
+
test_case "billing", input: { message: "charged twice" }, expected: "billing"
|
|
171
|
+
test_case "technical", input: { message: "500 error" }, expected: "technical"
|
|
172
|
+
test_case "greeting", input: { message: "hello" }, expected: "general"
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
run = SupportRouter::Eval.run!
|
|
176
|
+
puts run.summary
|
|
177
|
+
# SupportRouter eval: 3/3 passed (score: 1.0)
|
|
178
|
+
```
|
|
179
|
+
|
|
165
180
|
## Features
|
|
166
181
|
|
|
167
182
|
| Feature | Description | Docs |
|
|
@@ -182,7 +197,12 @@ result.save("logo.png")
|
|
|
182
197
|
| **Image Operations** | Generation, analysis, editing, pipelines with cost tracking | [Images](https://github.com/adham90/ruby_llm-agents/wiki/Image-Generation) |
|
|
183
198
|
| **Routing** | Message classification and routing with auto-generated prompts, inline classify | [Routing](https://github.com/adham90/ruby_llm-agents/wiki/Routing) |
|
|
184
199
|
| **Audio** | Text-to-speech (OpenAI, ElevenLabs), speech-to-text, dynamic pricing, 28+ output formats, dashboard audio playback | [Audio](https://github.com/adham90/ruby_llm-agents/wiki/Audio) |
|
|
200
|
+
| **Agent Composition** | Use agents as tools in other agents with automatic hierarchy tracking | [Tools](https://github.com/adham90/ruby_llm-agents/wiki/Tools) |
|
|
201
|
+
| **Queryable Agents** | Query execution history from agent classes with stats, replay, and cost breakdown | [Querying](https://github.com/adham90/ruby_llm-agents/wiki/Querying-Executions) |
|
|
202
|
+
| **Evaluation** | Test agent quality with exact match, contains, LLM judge, and custom scorers | [Evaluation](https://github.com/adham90/ruby_llm-agents/wiki/Evaluation) |
|
|
185
203
|
| **Alerts** | Slack, webhook, and custom notifications | [Alerts](https://github.com/adham90/ruby_llm-agents/wiki/Alerts) |
|
|
204
|
+
| **AS::Notifications** | 11 instrumentation events across execution, cache, budget, and reliability | [Events](https://github.com/adham90/ruby_llm-agents/wiki/ActiveSupport-Notifications) |
|
|
205
|
+
| **Custom Middleware** | Inject custom middleware globally or per-agent with positioning control | [Middleware](https://github.com/adham90/ruby_llm-agents/wiki/Custom-Middleware) |
|
|
186
206
|
|
|
187
207
|
## Quick Start
|
|
188
208
|
|
|
@@ -263,6 +283,7 @@ mount RubyLLM::Agents::Engine => "/agents"
|
|
|
263
283
|
| [Multi-Tenancy](https://github.com/adham90/ruby_llm-agents/wiki/Multi-Tenancy) | Per-tenant budgets, isolation, configuration |
|
|
264
284
|
| [Async/Fiber](https://github.com/adham90/ruby_llm-agents/wiki/Async-Fiber) | Concurrent execution with Ruby fibers |
|
|
265
285
|
| [Testing Agents](https://github.com/adham90/ruby_llm-agents/wiki/Testing-Agents) | RSpec patterns, mocking, dry_run mode |
|
|
286
|
+
| [Evaluation](https://github.com/adham90/ruby_llm-agents/wiki/Evaluation) | Score agent quality with built-in and custom scorers |
|
|
266
287
|
| [Error Handling](https://github.com/adham90/ruby_llm-agents/wiki/Error-Handling) | Error types, recovery patterns |
|
|
267
288
|
| [Routing](https://github.com/adham90/ruby_llm-agents/wiki/Routing) | Message classification, routing DSL, inline classify |
|
|
268
289
|
| [Embeddings](https://github.com/adham90/ruby_llm-agents/wiki/Embeddings) | Vector embeddings, batching, caching, preprocessing |
|
|
@@ -18,43 +18,131 @@ module RubyLLM
|
|
|
18
18
|
#
|
|
19
19
|
# @return [void]
|
|
20
20
|
def index
|
|
21
|
-
@selected_range = params[:range]
|
|
21
|
+
@selected_range = sanitize_range(params[:range])
|
|
22
22
|
@days = range_to_days(@selected_range)
|
|
23
|
+
parse_custom_dates if @selected_range == "custom"
|
|
23
24
|
base_scope = tenant_scoped_executions
|
|
24
|
-
@now_strip = base_scope
|
|
25
|
+
@now_strip = build_now_strip(base_scope)
|
|
25
26
|
@critical_alerts = load_critical_alerts(base_scope)
|
|
26
27
|
@recent_executions = base_scope.recent(10)
|
|
27
28
|
@agent_stats = build_agent_comparison(base_scope)
|
|
28
29
|
@top_errors = build_top_errors(base_scope)
|
|
29
30
|
@tenant_budget = load_tenant_budget(base_scope)
|
|
30
31
|
@model_stats = build_model_stats(base_scope)
|
|
32
|
+
@cache_savings = build_cache_savings(base_scope)
|
|
33
|
+
@top_tenants = build_top_tenants
|
|
31
34
|
end
|
|
32
35
|
|
|
33
36
|
# Returns chart data as JSON for live updates
|
|
34
37
|
#
|
|
35
|
-
# @param range [String] Time range: "today", "7d", or "
|
|
38
|
+
# @param range [String] Time range: "today", "7d", "30d", "90d", or "custom"
|
|
36
39
|
# @return [JSON] Chart data with series
|
|
37
40
|
def chart_data
|
|
38
|
-
range = params[:range]
|
|
39
|
-
|
|
41
|
+
range = sanitize_range(params[:range])
|
|
42
|
+
scope = tenant_scoped_executions
|
|
43
|
+
|
|
44
|
+
data = if range == "custom"
|
|
45
|
+
from = parse_date(params[:from])
|
|
46
|
+
to = parse_date(params[:to])
|
|
47
|
+
if from && to
|
|
48
|
+
from, to = [from, to].sort
|
|
49
|
+
to = [to, Date.current].min
|
|
50
|
+
scope.activity_chart_json_for_dates(from: from, to: to)
|
|
51
|
+
else
|
|
52
|
+
scope.activity_chart_json(range: "today")
|
|
53
|
+
end
|
|
54
|
+
else
|
|
55
|
+
scope.activity_chart_json(range: range)
|
|
56
|
+
end
|
|
57
|
+
|
|
40
58
|
render json: data
|
|
41
59
|
end
|
|
42
60
|
|
|
43
61
|
private
|
|
44
62
|
|
|
63
|
+
# Whitelists valid range values, defaulting to "today"
|
|
64
|
+
#
|
|
65
|
+
# @param range [String, nil] Raw range parameter
|
|
66
|
+
# @return [String] Sanitized range value
|
|
67
|
+
def sanitize_range(range)
|
|
68
|
+
%w[today 7d 30d 90d custom].include?(range) ? range : "today"
|
|
69
|
+
end
|
|
70
|
+
|
|
45
71
|
# Converts range parameter to number of days
|
|
46
72
|
#
|
|
47
|
-
# @param range [String] Range parameter (today, 7d, 30d)
|
|
48
|
-
# @return [Integer] Number of days
|
|
73
|
+
# @param range [String] Range parameter (today, 7d, 30d, 90d, custom)
|
|
74
|
+
# @return [Integer, nil] Number of days, or nil for custom
|
|
49
75
|
def range_to_days(range)
|
|
50
76
|
case range
|
|
51
77
|
when "today" then 1
|
|
52
78
|
when "7d" then 7
|
|
53
79
|
when "30d" then 30
|
|
80
|
+
when "90d" then 90
|
|
81
|
+
when "custom" then nil
|
|
54
82
|
else 1
|
|
55
83
|
end
|
|
56
84
|
end
|
|
57
85
|
|
|
86
|
+
# Safely parses a date string with validation
|
|
87
|
+
#
|
|
88
|
+
# Rejects future dates and dates more than 1 year ago.
|
|
89
|
+
#
|
|
90
|
+
# @param value [String, nil] Date string (YYYY-MM-DD)
|
|
91
|
+
# @return [Date, nil] Parsed date or nil if invalid
|
|
92
|
+
def parse_date(value)
|
|
93
|
+
return nil if value.blank?
|
|
94
|
+
date = Date.parse(value)
|
|
95
|
+
return nil if date > Date.current
|
|
96
|
+
return nil if date < 1.year.ago.to_date
|
|
97
|
+
date
|
|
98
|
+
rescue ArgumentError
|
|
99
|
+
nil
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Parses custom date range params and sets instance variables
|
|
103
|
+
#
|
|
104
|
+
# Falls back to "today" if dates are missing or invalid.
|
|
105
|
+
#
|
|
106
|
+
# @return [void]
|
|
107
|
+
def parse_custom_dates
|
|
108
|
+
from = parse_date(params[:from])
|
|
109
|
+
to = parse_date(params[:to])
|
|
110
|
+
|
|
111
|
+
if from && to
|
|
112
|
+
from, to = [from, to].sort
|
|
113
|
+
@custom_from = from
|
|
114
|
+
@custom_to = [to, Date.current].min
|
|
115
|
+
@days = (@custom_to - @custom_from).to_i + 1
|
|
116
|
+
else
|
|
117
|
+
@selected_range = "today"
|
|
118
|
+
@days = 1
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Returns the correct time scope for the current range
|
|
123
|
+
#
|
|
124
|
+
# @param base_scope [ActiveRecord::Relation] Base scope to filter
|
|
125
|
+
# @return [ActiveRecord::Relation] Time-scoped relation
|
|
126
|
+
def time_scoped(base_scope)
|
|
127
|
+
if @selected_range == "custom" && @custom_from && @custom_to
|
|
128
|
+
base_scope.where(created_at: @custom_from.beginning_of_day..@custom_to.end_of_day)
|
|
129
|
+
else
|
|
130
|
+
base_scope.last_n_days(@days)
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Routes to the correct now_strip_data method based on range
|
|
135
|
+
#
|
|
136
|
+
# @param base_scope [ActiveRecord::Relation] Base scope
|
|
137
|
+
# @return [Hash] Now strip metrics
|
|
138
|
+
def build_now_strip(base_scope)
|
|
139
|
+
if @selected_range == "custom" && @custom_from && @custom_to
|
|
140
|
+
base_scope.now_strip_data_for_dates(from: @custom_from, to: @custom_to)
|
|
141
|
+
else
|
|
142
|
+
base_scope.now_strip_data(range: @selected_range)
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
58
146
|
# Builds per-agent comparison statistics for all agent types
|
|
59
147
|
#
|
|
60
148
|
# Creates separate instance variables for each agent type:
|
|
@@ -67,7 +155,7 @@ module RubyLLM
|
|
|
67
155
|
# @param base_scope [ActiveRecord::Relation] Base scope to filter from
|
|
68
156
|
# @return [Array<Hash>] Array of base agent stats (for backward compatibility)
|
|
69
157
|
def build_agent_comparison(base_scope = Execution)
|
|
70
|
-
scope = base_scope
|
|
158
|
+
scope = time_scoped(base_scope)
|
|
71
159
|
|
|
72
160
|
# Get ALL agents from registry (file system + execution history)
|
|
73
161
|
all_agent_types = AgentRegistry.all
|
|
@@ -111,7 +199,7 @@ module RubyLLM
|
|
|
111
199
|
# @param base_scope [ActiveRecord::Relation] Base scope to filter from
|
|
112
200
|
# @return [Array<Hash>] Array of model stats sorted by total cost descending
|
|
113
201
|
def build_model_stats(base_scope = Execution)
|
|
114
|
-
scope = base_scope
|
|
202
|
+
scope = time_scoped(base_scope).where.not(model_id: nil)
|
|
115
203
|
|
|
116
204
|
# Batch fetch stats grouped by model
|
|
117
205
|
counts = scope.group(:model_id).count
|
|
@@ -147,7 +235,7 @@ module RubyLLM
|
|
|
147
235
|
# @param base_scope [ActiveRecord::Relation] Base scope to filter from
|
|
148
236
|
# @return [Array<Hash>] Top 5 error classes with counts
|
|
149
237
|
def build_top_errors(base_scope = Execution)
|
|
150
|
-
scope = base_scope
|
|
238
|
+
scope = time_scoped(base_scope).where(status: "error")
|
|
151
239
|
total_errors = scope.count
|
|
152
240
|
|
|
153
241
|
scope.group(:error_class)
|
|
@@ -347,6 +435,63 @@ module RubyLLM
|
|
|
347
435
|
alerts.take(3)
|
|
348
436
|
end
|
|
349
437
|
|
|
438
|
+
# Builds cache savings statistics for the dashboard
|
|
439
|
+
#
|
|
440
|
+
# @param base_scope [ActiveRecord::Relation] Base scope to filter from
|
|
441
|
+
# @return [Hash] Cache savings data with count, estimated savings, and hit rate
|
|
442
|
+
def build_cache_savings(base_scope)
|
|
443
|
+
scope = time_scoped(base_scope)
|
|
444
|
+
total_count = scope.count
|
|
445
|
+
return {count: 0, estimated_savings: 0, hit_rate: 0, total_executions: 0} if total_count.zero?
|
|
446
|
+
|
|
447
|
+
cached_scope = scope.cached
|
|
448
|
+
cache_count = cached_scope.count
|
|
449
|
+
estimated_savings = cached_scope.sum(:total_cost)
|
|
450
|
+
|
|
451
|
+
{
|
|
452
|
+
count: cache_count,
|
|
453
|
+
estimated_savings: estimated_savings,
|
|
454
|
+
hit_rate: (cache_count.to_f / total_count * 100).round(1),
|
|
455
|
+
total_executions: total_count
|
|
456
|
+
}
|
|
457
|
+
end
|
|
458
|
+
|
|
459
|
+
# Builds top tenants list for the dashboard overview
|
|
460
|
+
#
|
|
461
|
+
# @return [Array<Hash>, nil] Top 5 tenants by monthly spend, or nil if none
|
|
462
|
+
def build_top_tenants
|
|
463
|
+
return nil unless Tenant.table_exists?
|
|
464
|
+
|
|
465
|
+
tenants = Tenant.active
|
|
466
|
+
.where("monthly_cost_spent > 0 OR monthly_executions_count > 0")
|
|
467
|
+
.order(monthly_cost_spent: :desc)
|
|
468
|
+
.limit(5)
|
|
469
|
+
|
|
470
|
+
return nil if tenants.empty?
|
|
471
|
+
|
|
472
|
+
tenants.map do |tenant|
|
|
473
|
+
tenant.ensure_daily_reset!
|
|
474
|
+
tenant.ensure_monthly_reset!
|
|
475
|
+
|
|
476
|
+
monthly_limit = tenant.effective_monthly_limit
|
|
477
|
+
daily_limit = tenant.effective_daily_limit
|
|
478
|
+
|
|
479
|
+
{
|
|
480
|
+
id: tenant.id,
|
|
481
|
+
tenant_id: tenant.tenant_id,
|
|
482
|
+
name: tenant.display_name,
|
|
483
|
+
enforcement: tenant.effective_enforcement,
|
|
484
|
+
monthly_spend: tenant.monthly_cost_spent,
|
|
485
|
+
monthly_limit: monthly_limit,
|
|
486
|
+
monthly_percentage: (monthly_limit.to_f > 0) ? (tenant.monthly_cost_spent / monthly_limit * 100).round(1) : 0,
|
|
487
|
+
daily_spend: tenant.daily_cost_spent,
|
|
488
|
+
daily_limit: daily_limit,
|
|
489
|
+
daily_percentage: (daily_limit.to_f > 0) ? (tenant.daily_cost_spent / daily_limit * 100).round(1) : 0,
|
|
490
|
+
monthly_executions: tenant.monthly_executions_count
|
|
491
|
+
}
|
|
492
|
+
end
|
|
493
|
+
end
|
|
494
|
+
|
|
350
495
|
# Batch fetches execution stats for all agents in a time period
|
|
351
496
|
#
|
|
352
497
|
# @param scope [ActiveRecord::Relation] Base scope with time filter
|
|
@@ -96,9 +96,7 @@ module RubyLLM
|
|
|
96
96
|
# @param execution [Execution] The execution record
|
|
97
97
|
# @return [String] CSV row string
|
|
98
98
|
def generate_csv_row(execution)
|
|
99
|
-
redacted_error_message =
|
|
100
|
-
Redactor.redact_string(execution.error_message)
|
|
101
|
-
end
|
|
99
|
+
redacted_error_message = execution.error_message
|
|
102
100
|
|
|
103
101
|
CSV.generate_line([
|
|
104
102
|
execution.id,
|
|
@@ -120,33 +120,6 @@ module RubyLLM
|
|
|
120
120
|
end
|
|
121
121
|
end
|
|
122
122
|
|
|
123
|
-
# Redacts sensitive data from an object for display
|
|
124
|
-
#
|
|
125
|
-
# Uses the configured redaction rules to mask sensitive fields
|
|
126
|
-
# and patterns in the data.
|
|
127
|
-
#
|
|
128
|
-
# @param obj [Object] The object to redact (Hash, Array, or primitive)
|
|
129
|
-
# @return [Object] The redacted object
|
|
130
|
-
# @example
|
|
131
|
-
# redact_for_display({ password: "secret", name: "John" })
|
|
132
|
-
# #=> { password: "[REDACTED]", name: "John" }
|
|
133
|
-
def redact_for_display(obj)
|
|
134
|
-
Redactor.redact(obj)
|
|
135
|
-
end
|
|
136
|
-
|
|
137
|
-
# Syntax-highlights a redacted Ruby object as pretty-printed JSON
|
|
138
|
-
#
|
|
139
|
-
# Combines redaction and highlighting in one call.
|
|
140
|
-
#
|
|
141
|
-
# @param obj [Object] Any JSON-serializable Ruby object
|
|
142
|
-
# @return [ActiveSupport::SafeBuffer] HTML-safe highlighted redacted JSON
|
|
143
|
-
def highlight_json_redacted(obj)
|
|
144
|
-
return "" if obj.nil?
|
|
145
|
-
|
|
146
|
-
redacted = redact_for_display(obj)
|
|
147
|
-
highlight_json(redacted)
|
|
148
|
-
end
|
|
149
|
-
|
|
150
123
|
# Syntax-highlights a Ruby object as pretty-printed JSON
|
|
151
124
|
#
|
|
152
125
|
# Converts the object to JSON and applies color highlighting
|
|
@@ -286,7 +259,7 @@ module RubyLLM
|
|
|
286
259
|
|
|
287
260
|
# Returns human-readable display name for time range
|
|
288
261
|
#
|
|
289
|
-
# @param range [String] Range parameter (today, 7d, 30d)
|
|
262
|
+
# @param range [String] Range parameter (today, 7d, 30d, 90d, custom)
|
|
290
263
|
# @return [String] Human-readable range name
|
|
291
264
|
# @example
|
|
292
265
|
# range_display_name("7d") #=> "7 Days"
|
|
@@ -295,10 +268,24 @@ module RubyLLM
|
|
|
295
268
|
when "today" then "Today"
|
|
296
269
|
when "7d" then "7 Days"
|
|
297
270
|
when "30d" then "30 Days"
|
|
271
|
+
when "90d" then "90 Days"
|
|
272
|
+
when "custom" then "Custom"
|
|
298
273
|
else "Today"
|
|
299
274
|
end
|
|
300
275
|
end
|
|
301
276
|
|
|
277
|
+
# Returns preset range options for the time range dropdown
|
|
278
|
+
#
|
|
279
|
+
# @return [Array<Hash>] Array of {value:, label:} pairs
|
|
280
|
+
def range_presets
|
|
281
|
+
[
|
|
282
|
+
{value: "today", label: "Today"},
|
|
283
|
+
{value: "7d", label: "7 Days"},
|
|
284
|
+
{value: "30d", label: "30 Days"},
|
|
285
|
+
{value: "90d", label: "90 Days"}
|
|
286
|
+
]
|
|
287
|
+
end
|
|
288
|
+
|
|
302
289
|
# Formats milliseconds to human-readable duration
|
|
303
290
|
#
|
|
304
291
|
# @param ms [Numeric, nil] Duration in milliseconds
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyLLM
|
|
4
|
+
module Agents
|
|
5
|
+
class Execution
|
|
6
|
+
# Adds replay capability to execution records.
|
|
7
|
+
#
|
|
8
|
+
# Allows re-executing a previous run with the same inputs,
|
|
9
|
+
# or with tweaked parameters for A/B testing and debugging.
|
|
10
|
+
#
|
|
11
|
+
# @example Replay with same settings
|
|
12
|
+
# run = SupportAgent.last_run
|
|
13
|
+
# new_run = run.replay
|
|
14
|
+
#
|
|
15
|
+
# @example Replay with different model
|
|
16
|
+
# run.replay(model: "claude-sonnet-4-6")
|
|
17
|
+
#
|
|
18
|
+
# @example Compare two models
|
|
19
|
+
# run1 = SupportAgent.last_run
|
|
20
|
+
# run2 = run1.replay(model: "gpt-4o-mini")
|
|
21
|
+
# puts "Original: #{run1.total_cost} | Replay: #{run2.total_cost}"
|
|
22
|
+
#
|
|
23
|
+
module Replayable
|
|
24
|
+
extend ActiveSupport::Concern
|
|
25
|
+
|
|
26
|
+
# Re-executes this agent run with the same (or overridden) inputs.
|
|
27
|
+
#
|
|
28
|
+
# Loads the original agent class, reconstructs its parameters from
|
|
29
|
+
# the execution detail record, and executes through the full pipeline.
|
|
30
|
+
# The new execution is tracked separately and linked via metadata.
|
|
31
|
+
#
|
|
32
|
+
# @param model [String, nil] Override the model
|
|
33
|
+
# @param temperature [Float, nil] Override the temperature
|
|
34
|
+
# @param overrides [Hash] Additional parameter overrides
|
|
35
|
+
# @return [Object] The result from the new execution
|
|
36
|
+
#
|
|
37
|
+
# @raise [ReplayError] If the agent class cannot be resolved or detail is missing
|
|
38
|
+
#
|
|
39
|
+
def replay(model: nil, temperature: nil, **overrides)
|
|
40
|
+
validate_replayable!
|
|
41
|
+
agent_klass = resolve_agent_class
|
|
42
|
+
params = build_replay_params(overrides)
|
|
43
|
+
|
|
44
|
+
opts = params.merge(_replay_source_id: id)
|
|
45
|
+
opts[:model] = model if model
|
|
46
|
+
opts[:temperature] = temperature if temperature
|
|
47
|
+
|
|
48
|
+
agent_klass.call(**opts)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Returns whether this execution can be replayed.
|
|
52
|
+
#
|
|
53
|
+
# @return [Boolean]
|
|
54
|
+
#
|
|
55
|
+
def replayable?
|
|
56
|
+
return false if agent_type.blank?
|
|
57
|
+
return false if detail.nil?
|
|
58
|
+
|
|
59
|
+
resolve_agent_class
|
|
60
|
+
true
|
|
61
|
+
rescue
|
|
62
|
+
false
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Returns all executions that are replays of this one.
|
|
66
|
+
#
|
|
67
|
+
# @return [ActiveRecord::Relation]
|
|
68
|
+
#
|
|
69
|
+
def replays
|
|
70
|
+
self.class.replays_of(id)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Returns the original execution this was replayed from.
|
|
74
|
+
#
|
|
75
|
+
# @return [RubyLLM::Agents::Execution, nil]
|
|
76
|
+
#
|
|
77
|
+
def replay_source
|
|
78
|
+
source_id = metadata&.dig("replay_source_id")
|
|
79
|
+
return nil unless source_id
|
|
80
|
+
|
|
81
|
+
self.class.find_by(id: source_id)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Returns whether this execution is a replay of another.
|
|
85
|
+
#
|
|
86
|
+
# @return [Boolean]
|
|
87
|
+
#
|
|
88
|
+
def replay?
|
|
89
|
+
metadata&.dig("replay_source_id").present?
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
private
|
|
93
|
+
|
|
94
|
+
def validate_replayable!
|
|
95
|
+
if agent_type.blank?
|
|
96
|
+
raise ReplayError, "Cannot replay: execution has no agent_type"
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
if detail.nil?
|
|
100
|
+
raise ReplayError,
|
|
101
|
+
"Cannot replay execution ##{id}: no detail record " \
|
|
102
|
+
"(prompts and parameters are required for replay)"
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
resolve_agent_class
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def resolve_agent_class
|
|
109
|
+
agent_type.constantize
|
|
110
|
+
rescue NameError => e
|
|
111
|
+
raise ReplayError,
|
|
112
|
+
"Cannot replay execution ##{id}: agent class '#{agent_type}' " \
|
|
113
|
+
"not found (#{e.message})"
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def build_replay_params(overrides)
|
|
117
|
+
original_params = detail.parameters || {}
|
|
118
|
+
symbolized = original_params.transform_keys(&:to_sym)
|
|
119
|
+
symbolized.merge(overrides)
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
@@ -83,7 +83,10 @@ module RubyLLM
|
|
|
83
83
|
# Filters to a specific LLM model
|
|
84
84
|
# @param model_id [String] The model identifier
|
|
85
85
|
# @return [ActiveRecord::Relation]
|
|
86
|
-
scope :by_agent, ->(agent_type) {
|
|
86
|
+
scope :by_agent, ->(agent_type) {
|
|
87
|
+
names = resolve_agent_names(agent_type)
|
|
88
|
+
where(agent_type: names)
|
|
89
|
+
}
|
|
87
90
|
scope :by_model, ->(model_id) { where(model_id: model_id.to_s) }
|
|
88
91
|
|
|
89
92
|
# @!endgroup
|
|
@@ -265,6 +268,18 @@ module RubyLLM
|
|
|
265
268
|
scope :without_tool_calls, -> { where(tool_calls_count: 0) }
|
|
266
269
|
|
|
267
270
|
# @!endgroup
|
|
271
|
+
|
|
272
|
+
# @!group Replay Scopes
|
|
273
|
+
|
|
274
|
+
# @!method replays_of(execution_id)
|
|
275
|
+
# Returns executions that are replays of the given execution
|
|
276
|
+
# @param execution_id [Integer, String] The source execution ID
|
|
277
|
+
# @return [ActiveRecord::Relation]
|
|
278
|
+
scope :replays_of, ->(execution_id) {
|
|
279
|
+
metadata_value("replay_source_id", execution_id.to_s)
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
# @!endgroup
|
|
268
283
|
end
|
|
269
284
|
|
|
270
285
|
# @!group Aggregation Methods
|
|
@@ -273,6 +288,32 @@ module RubyLLM
|
|
|
273
288
|
# They can be called on scoped relations.
|
|
274
289
|
|
|
275
290
|
class_methods do
|
|
291
|
+
# Resolves all known names for an agent, including aliases
|
|
292
|
+
#
|
|
293
|
+
# Accepts a class (uses all_agent_names), or a string (looks up
|
|
294
|
+
# the class in ObjectSpace to check for aliases, falls back to
|
|
295
|
+
# the string itself).
|
|
296
|
+
#
|
|
297
|
+
# @param agent_type [Class, String] Agent class or class name
|
|
298
|
+
# @return [Array<String>] All names to query
|
|
299
|
+
def resolve_agent_names(agent_type)
|
|
300
|
+
if agent_type.is_a?(Class) && agent_type.respond_to?(:all_agent_names)
|
|
301
|
+
agent_type.all_agent_names
|
|
302
|
+
else
|
|
303
|
+
name = agent_type.to_s
|
|
304
|
+
klass = begin
|
|
305
|
+
name.constantize
|
|
306
|
+
rescue NameError
|
|
307
|
+
nil
|
|
308
|
+
end
|
|
309
|
+
if klass&.respond_to?(:all_agent_names)
|
|
310
|
+
klass.all_agent_names
|
|
311
|
+
else
|
|
312
|
+
[name]
|
|
313
|
+
end
|
|
314
|
+
end
|
|
315
|
+
end
|
|
316
|
+
|
|
276
317
|
# Database-agnostic JSON metadata queries
|
|
277
318
|
# These fields (fallback_reason, retryable, rate_limited, etc.) are stored
|
|
278
319
|
# in the metadata JSON column rather than as direct columns.
|
|
@@ -49,6 +49,7 @@ module RubyLLM
|
|
|
49
49
|
include Execution::Metrics
|
|
50
50
|
include Execution::Scopes
|
|
51
51
|
include Execution::Analytics
|
|
52
|
+
include Execution::Replayable
|
|
52
53
|
|
|
53
54
|
# Status enum
|
|
54
55
|
# - running: execution in progress
|
|
@@ -274,18 +275,20 @@ module RubyLLM
|
|
|
274
275
|
|
|
275
276
|
# Returns real-time dashboard data for the Now Strip
|
|
276
277
|
#
|
|
277
|
-
# @param range [String] Time range: "today", "7d", or "
|
|
278
|
+
# @param range [String] Time range: "today", "7d", "30d", or "90d"
|
|
278
279
|
# @return [Hash] Now strip metrics with period-over-period comparisons
|
|
279
280
|
def self.now_strip_data(range: "today")
|
|
280
281
|
current_scope = case range
|
|
281
282
|
when "7d" then last_n_days(7)
|
|
282
283
|
when "30d" then last_n_days(30)
|
|
284
|
+
when "90d" then last_n_days(90)
|
|
283
285
|
else today
|
|
284
286
|
end
|
|
285
287
|
|
|
286
288
|
previous_scope = case range
|
|
287
289
|
when "7d" then where(created_at: 14.days.ago.beginning_of_day..7.days.ago.beginning_of_day)
|
|
288
290
|
when "30d" then where(created_at: 60.days.ago.beginning_of_day..30.days.ago.beginning_of_day)
|
|
291
|
+
when "90d" then where(created_at: 180.days.ago.beginning_of_day..90.days.ago.beginning_of_day)
|
|
289
292
|
else yesterday
|
|
290
293
|
end
|
|
291
294
|
|
|
@@ -320,6 +323,52 @@ module RubyLLM
|
|
|
320
323
|
)
|
|
321
324
|
end
|
|
322
325
|
|
|
326
|
+
# Returns Now Strip data for a custom date range
|
|
327
|
+
#
|
|
328
|
+
# Compares the selected range against the same-length window
|
|
329
|
+
# immediately preceding it.
|
|
330
|
+
#
|
|
331
|
+
# @param from [Date] Start date (inclusive)
|
|
332
|
+
# @param to [Date] End date (inclusive)
|
|
333
|
+
# @return [Hash] Now strip metrics with period-over-period comparisons
|
|
334
|
+
def self.now_strip_data_for_dates(from:, to:)
|
|
335
|
+
span_days = (to - from).to_i + 1
|
|
336
|
+
current_scope = where(created_at: from.beginning_of_day..to.end_of_day)
|
|
337
|
+
previous_from = from - span_days.days
|
|
338
|
+
previous_to = from - 1.day
|
|
339
|
+
previous_scope = where(created_at: previous_from.beginning_of_day..previous_to.end_of_day)
|
|
340
|
+
|
|
341
|
+
current = {
|
|
342
|
+
running: running.count,
|
|
343
|
+
success_today: current_scope.status_success.count,
|
|
344
|
+
errors_today: current_scope.status_error.count,
|
|
345
|
+
timeouts_today: current_scope.status_timeout.count,
|
|
346
|
+
cost_today: current_scope.sum(:total_cost) || 0,
|
|
347
|
+
executions_today: current_scope.count,
|
|
348
|
+
success_rate: calculate_period_success_rate(current_scope),
|
|
349
|
+
avg_duration_ms: current_scope.avg_duration&.round || 0,
|
|
350
|
+
total_tokens: current_scope.total_tokens_sum || 0
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
previous = {
|
|
354
|
+
success: previous_scope.status_success.count,
|
|
355
|
+
errors: previous_scope.status_error.count,
|
|
356
|
+
cost: previous_scope.sum(:total_cost) || 0,
|
|
357
|
+
avg_duration_ms: previous_scope.avg_duration&.round || 0,
|
|
358
|
+
total_tokens: previous_scope.total_tokens_sum || 0
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
current.merge(
|
|
362
|
+
comparisons: {
|
|
363
|
+
success_change: pct_change(previous[:success], current[:success_today]),
|
|
364
|
+
errors_change: pct_change(previous[:errors], current[:errors_today]),
|
|
365
|
+
cost_change: pct_change(previous[:cost], current[:cost_today]),
|
|
366
|
+
duration_change: pct_change(previous[:avg_duration_ms], current[:avg_duration_ms]),
|
|
367
|
+
tokens_change: pct_change(previous[:total_tokens], current[:total_tokens])
|
|
368
|
+
}
|
|
369
|
+
)
|
|
370
|
+
end
|
|
371
|
+
|
|
323
372
|
# Calculates percentage change between old and new values
|
|
324
373
|
#
|
|
325
374
|
# @param old_val [Numeric, nil] Previous period value
|