ruby_llm-agents 0.3.3 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/controllers/ruby_llm/agents/dashboard_controller.rb +68 -4
- data/app/models/ruby_llm/agents/execution/analytics.rb +114 -13
- data/app/models/ruby_llm/agents/execution.rb +19 -58
- data/app/views/layouts/rubyllm/agents/application.html.erb +92 -350
- data/app/views/rubyllm/agents/agents/show.html.erb +331 -385
- data/app/views/rubyllm/agents/dashboard/_agent_comparison.html.erb +46 -0
- data/app/views/rubyllm/agents/dashboard/_budgets_bar.html.erb +0 -90
- data/app/views/rubyllm/agents/dashboard/_now_strip.html.erb +79 -5
- data/app/views/rubyllm/agents/dashboard/_top_errors.html.erb +49 -0
- data/app/views/rubyllm/agents/dashboard/index.html.erb +76 -121
- data/app/views/rubyllm/agents/executions/show.html.erb +134 -85
- data/app/views/rubyllm/agents/settings/show.html.erb +1 -1
- data/app/views/rubyllm/agents/shared/_breadcrumbs.html.erb +48 -0
- data/app/views/rubyllm/agents/shared/_nav_link.html.erb +27 -0
- data/config/routes.rb +2 -0
- data/lib/ruby_llm/agents/base/caching.rb +43 -0
- data/lib/ruby_llm/agents/base/cost_calculation.rb +103 -0
- data/lib/ruby_llm/agents/base/dsl.rb +261 -0
- data/lib/ruby_llm/agents/base/execution.rb +206 -0
- data/lib/ruby_llm/agents/base/reliability_execution.rb +131 -0
- data/lib/ruby_llm/agents/base/response_building.rb +86 -0
- data/lib/ruby_llm/agents/base/tool_tracking.rb +57 -0
- data/lib/ruby_llm/agents/base.rb +15 -805
- data/lib/ruby_llm/agents/version.rb +1 -1
- metadata +12 -20
- data/app/channels/ruby_llm/agents/executions_channel.rb +0 -46
- data/app/javascript/ruby_llm/agents/controllers/filter_controller.js +0 -56
- data/app/javascript/ruby_llm/agents/controllers/index.js +0 -12
- data/app/javascript/ruby_llm/agents/controllers/refresh_controller.js +0 -83
- data/app/views/rubyllm/agents/dashboard/_now_strip_values.html.erb +0 -71
data/lib/ruby_llm/agents/base.rb
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "base/dsl"
|
|
4
|
+
require_relative "base/caching"
|
|
5
|
+
require_relative "base/cost_calculation"
|
|
6
|
+
require_relative "base/tool_tracking"
|
|
7
|
+
require_relative "base/response_building"
|
|
8
|
+
require_relative "base/execution"
|
|
9
|
+
require_relative "base/reliability_execution"
|
|
10
|
+
|
|
3
11
|
module RubyLLM
|
|
4
12
|
module Agents
|
|
5
13
|
# Base class for LLM-powered agents
|
|
@@ -37,11 +45,14 @@ module RubyLLM
|
|
|
37
45
|
# @api public
|
|
38
46
|
class Base
|
|
39
47
|
include Instrumentation
|
|
48
|
+
include Caching
|
|
49
|
+
include CostCalculation
|
|
50
|
+
include ToolTracking
|
|
51
|
+
include ResponseBuilding
|
|
52
|
+
include Execution
|
|
53
|
+
include ReliabilityExecution
|
|
40
54
|
|
|
41
|
-
|
|
42
|
-
VERSION = "1.0".freeze
|
|
43
|
-
# @!visibility private
|
|
44
|
-
CACHE_TTL = 1.hour
|
|
55
|
+
extend DSL
|
|
45
56
|
|
|
46
57
|
class << self
|
|
47
58
|
# Factory method to instantiate and execute an agent
|
|
@@ -72,249 +83,6 @@ module RubyLLM
|
|
|
72
83
|
def call(*args, **kwargs, &block)
|
|
73
84
|
new(*args, **kwargs).call(&block)
|
|
74
85
|
end
|
|
75
|
-
|
|
76
|
-
# @!group Configuration DSL
|
|
77
|
-
|
|
78
|
-
# Sets or returns the LLM model for this agent class
|
|
79
|
-
#
|
|
80
|
-
# @param value [String, nil] The model identifier to set
|
|
81
|
-
# @return [String] The current model setting
|
|
82
|
-
# @example
|
|
83
|
-
# model "gpt-4o"
|
|
84
|
-
def model(value = nil)
|
|
85
|
-
@model = value if value
|
|
86
|
-
@model || inherited_or_default(:model, RubyLLM::Agents.configuration.default_model)
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
# Sets or returns the temperature for LLM responses
|
|
90
|
-
#
|
|
91
|
-
# @param value [Float, nil] Temperature value (0.0-2.0)
|
|
92
|
-
# @return [Float] The current temperature setting
|
|
93
|
-
# @example
|
|
94
|
-
# temperature 0.7
|
|
95
|
-
def temperature(value = nil)
|
|
96
|
-
@temperature = value if value
|
|
97
|
-
@temperature || inherited_or_default(:temperature, RubyLLM::Agents.configuration.default_temperature)
|
|
98
|
-
end
|
|
99
|
-
|
|
100
|
-
# Sets or returns the version string for cache invalidation
|
|
101
|
-
#
|
|
102
|
-
# @param value [String, nil] Version string
|
|
103
|
-
# @return [String] The current version
|
|
104
|
-
# @example
|
|
105
|
-
# version "2.0"
|
|
106
|
-
def version(value = nil)
|
|
107
|
-
@version = value if value
|
|
108
|
-
@version || inherited_or_default(:version, VERSION)
|
|
109
|
-
end
|
|
110
|
-
|
|
111
|
-
# Sets or returns the timeout in seconds for LLM requests
|
|
112
|
-
#
|
|
113
|
-
# @param value [Integer, nil] Timeout in seconds
|
|
114
|
-
# @return [Integer] The current timeout setting
|
|
115
|
-
# @example
|
|
116
|
-
# timeout 30
|
|
117
|
-
def timeout(value = nil)
|
|
118
|
-
@timeout = value if value
|
|
119
|
-
@timeout || inherited_or_default(:timeout, RubyLLM::Agents.configuration.default_timeout)
|
|
120
|
-
end
|
|
121
|
-
|
|
122
|
-
# @!endgroup
|
|
123
|
-
|
|
124
|
-
# @!group Reliability DSL
|
|
125
|
-
|
|
126
|
-
# Configures retry behavior for this agent
|
|
127
|
-
#
|
|
128
|
-
# @param max [Integer] Maximum number of retry attempts (default: 0)
|
|
129
|
-
# @param backoff [Symbol] Backoff strategy (:constant or :exponential)
|
|
130
|
-
# @param base [Float] Base delay in seconds
|
|
131
|
-
# @param max_delay [Float] Maximum delay between retries
|
|
132
|
-
# @param on [Array<Class>] Error classes to retry on (extends defaults)
|
|
133
|
-
# @return [Hash] The current retry configuration
|
|
134
|
-
# @example
|
|
135
|
-
# retries max: 2, backoff: :exponential, base: 0.4, max_delay: 3.0, on: [Timeout::Error]
|
|
136
|
-
def retries(max: nil, backoff: nil, base: nil, max_delay: nil, on: nil)
|
|
137
|
-
if max || backoff || base || max_delay || on
|
|
138
|
-
@retries_config ||= RubyLLM::Agents.configuration.default_retries.dup
|
|
139
|
-
@retries_config[:max] = max if max
|
|
140
|
-
@retries_config[:backoff] = backoff if backoff
|
|
141
|
-
@retries_config[:base] = base if base
|
|
142
|
-
@retries_config[:max_delay] = max_delay if max_delay
|
|
143
|
-
@retries_config[:on] = on if on
|
|
144
|
-
end
|
|
145
|
-
@retries_config || inherited_or_default(:retries_config, RubyLLM::Agents.configuration.default_retries)
|
|
146
|
-
end
|
|
147
|
-
|
|
148
|
-
# Returns the retry configuration for this agent
|
|
149
|
-
#
|
|
150
|
-
# @return [Hash, nil] The retry configuration
|
|
151
|
-
def retries_config
|
|
152
|
-
@retries_config || (superclass.respond_to?(:retries_config) ? superclass.retries_config : nil)
|
|
153
|
-
end
|
|
154
|
-
|
|
155
|
-
# Sets or returns fallback models to try when primary model fails
|
|
156
|
-
#
|
|
157
|
-
# @param models [Array<String>, nil] Model identifiers to use as fallbacks
|
|
158
|
-
# @return [Array<String>] The current fallback models
|
|
159
|
-
# @example
|
|
160
|
-
# fallback_models ["gpt-4o-mini", "gpt-4o"]
|
|
161
|
-
def fallback_models(models = nil)
|
|
162
|
-
@fallback_models = models if models
|
|
163
|
-
@fallback_models || inherited_or_default(:fallback_models, RubyLLM::Agents.configuration.default_fallback_models)
|
|
164
|
-
end
|
|
165
|
-
|
|
166
|
-
# Sets or returns the total timeout for all retry/fallback attempts
|
|
167
|
-
#
|
|
168
|
-
# @param seconds [Integer, nil] Total timeout in seconds
|
|
169
|
-
# @return [Integer, nil] The current total timeout
|
|
170
|
-
# @example
|
|
171
|
-
# total_timeout 20
|
|
172
|
-
def total_timeout(seconds = nil)
|
|
173
|
-
@total_timeout = seconds if seconds
|
|
174
|
-
@total_timeout || inherited_or_default(:total_timeout, RubyLLM::Agents.configuration.default_total_timeout)
|
|
175
|
-
end
|
|
176
|
-
|
|
177
|
-
# Configures circuit breaker for this agent
|
|
178
|
-
#
|
|
179
|
-
# @param errors [Integer] Number of errors to trigger open state
|
|
180
|
-
# @param within [Integer] Rolling window in seconds
|
|
181
|
-
# @param cooldown [Integer] Cooldown period in seconds when open
|
|
182
|
-
# @return [Hash, nil] The current circuit breaker configuration
|
|
183
|
-
# @example
|
|
184
|
-
# circuit_breaker errors: 10, within: 60, cooldown: 300
|
|
185
|
-
def circuit_breaker(errors: nil, within: nil, cooldown: nil)
|
|
186
|
-
if errors || within || cooldown
|
|
187
|
-
@circuit_breaker_config ||= { errors: 10, within: 60, cooldown: 300 }
|
|
188
|
-
@circuit_breaker_config[:errors] = errors if errors
|
|
189
|
-
@circuit_breaker_config[:within] = within if within
|
|
190
|
-
@circuit_breaker_config[:cooldown] = cooldown if cooldown
|
|
191
|
-
end
|
|
192
|
-
@circuit_breaker_config || (superclass.respond_to?(:circuit_breaker_config) ? superclass.circuit_breaker_config : nil)
|
|
193
|
-
end
|
|
194
|
-
|
|
195
|
-
# Returns the circuit breaker configuration for this agent
|
|
196
|
-
#
|
|
197
|
-
# @return [Hash, nil] The circuit breaker configuration
|
|
198
|
-
def circuit_breaker_config
|
|
199
|
-
@circuit_breaker_config || (superclass.respond_to?(:circuit_breaker_config) ? superclass.circuit_breaker_config : nil)
|
|
200
|
-
end
|
|
201
|
-
|
|
202
|
-
# @!endgroup
|
|
203
|
-
|
|
204
|
-
# @!group Parameter DSL
|
|
205
|
-
|
|
206
|
-
# Defines a parameter for the agent
|
|
207
|
-
#
|
|
208
|
-
# Creates an accessor method for the parameter that retrieves values
|
|
209
|
-
# from the options hash, falling back to the default value.
|
|
210
|
-
#
|
|
211
|
-
# @param name [Symbol] The parameter name
|
|
212
|
-
# @param required [Boolean] Whether the parameter is required
|
|
213
|
-
# @param default [Object, nil] Default value if not provided
|
|
214
|
-
# @return [void]
|
|
215
|
-
# @example
|
|
216
|
-
# param :query, required: true
|
|
217
|
-
# param :limit, default: 10
|
|
218
|
-
def param(name, required: false, default: nil)
|
|
219
|
-
@params ||= {}
|
|
220
|
-
@params[name] = { required: required, default: default }
|
|
221
|
-
define_method(name) do
|
|
222
|
-
@options[name] || @options[name.to_s] || self.class.params.dig(name, :default)
|
|
223
|
-
end
|
|
224
|
-
end
|
|
225
|
-
|
|
226
|
-
# Returns all defined parameters including inherited ones
|
|
227
|
-
#
|
|
228
|
-
# @return [Hash{Symbol => Hash}] Parameter definitions
|
|
229
|
-
def params
|
|
230
|
-
parent = superclass.respond_to?(:params) ? superclass.params : {}
|
|
231
|
-
parent.merge(@params || {})
|
|
232
|
-
end
|
|
233
|
-
|
|
234
|
-
# @!endgroup
|
|
235
|
-
|
|
236
|
-
# @!group Caching DSL
|
|
237
|
-
|
|
238
|
-
# Enables caching for this agent with optional TTL
|
|
239
|
-
#
|
|
240
|
-
# @param ttl [ActiveSupport::Duration] Time-to-live for cached responses
|
|
241
|
-
# @return [void]
|
|
242
|
-
# @example
|
|
243
|
-
# cache 1.hour
|
|
244
|
-
def cache(ttl = CACHE_TTL)
|
|
245
|
-
@cache_enabled = true
|
|
246
|
-
@cache_ttl = ttl
|
|
247
|
-
end
|
|
248
|
-
|
|
249
|
-
# Returns whether caching is enabled for this agent
|
|
250
|
-
#
|
|
251
|
-
# @return [Boolean] true if caching is enabled
|
|
252
|
-
def cache_enabled?
|
|
253
|
-
@cache_enabled || false
|
|
254
|
-
end
|
|
255
|
-
|
|
256
|
-
# Returns the cache TTL for this agent
|
|
257
|
-
#
|
|
258
|
-
# @return [ActiveSupport::Duration] The cache TTL
|
|
259
|
-
def cache_ttl
|
|
260
|
-
@cache_ttl || CACHE_TTL
|
|
261
|
-
end
|
|
262
|
-
|
|
263
|
-
# @!endgroup
|
|
264
|
-
|
|
265
|
-
# @!group Streaming DSL
|
|
266
|
-
|
|
267
|
-
# Enables or returns streaming mode for this agent
|
|
268
|
-
#
|
|
269
|
-
# When streaming is enabled and a block is passed to call,
|
|
270
|
-
# chunks will be yielded to the block as they arrive.
|
|
271
|
-
#
|
|
272
|
-
# @param value [Boolean, nil] Whether to enable streaming
|
|
273
|
-
# @return [Boolean] The current streaming setting
|
|
274
|
-
# @example
|
|
275
|
-
# streaming true
|
|
276
|
-
def streaming(value = nil)
|
|
277
|
-
@streaming = value unless value.nil?
|
|
278
|
-
return @streaming unless @streaming.nil?
|
|
279
|
-
|
|
280
|
-
inherited_or_default(:streaming, RubyLLM::Agents.configuration.default_streaming)
|
|
281
|
-
end
|
|
282
|
-
|
|
283
|
-
# @!endgroup
|
|
284
|
-
|
|
285
|
-
# @!group Tools DSL
|
|
286
|
-
|
|
287
|
-
# Sets or returns the tools available to this agent
|
|
288
|
-
#
|
|
289
|
-
# Tools are RubyLLM::Tool classes that the model can invoke.
|
|
290
|
-
# The agent will automatically execute tool calls and continue
|
|
291
|
-
# until the model produces a final response.
|
|
292
|
-
#
|
|
293
|
-
# @param tool_classes [Array<Class>] Tool classes to make available
|
|
294
|
-
# @return [Array<Class>] The current tools
|
|
295
|
-
# @example Single tool
|
|
296
|
-
# tools WeatherTool
|
|
297
|
-
# @example Multiple tools
|
|
298
|
-
# tools WeatherTool, SearchTool, CalculatorTool
|
|
299
|
-
def tools(*tool_classes)
|
|
300
|
-
if tool_classes.any?
|
|
301
|
-
@tools = tool_classes.flatten
|
|
302
|
-
end
|
|
303
|
-
@tools || inherited_or_default(:tools, RubyLLM::Agents.configuration.default_tools)
|
|
304
|
-
end
|
|
305
|
-
|
|
306
|
-
# @!endgroup
|
|
307
|
-
|
|
308
|
-
private
|
|
309
|
-
|
|
310
|
-
# Looks up setting from superclass or uses default
|
|
311
|
-
#
|
|
312
|
-
# @param method [Symbol] The method to call on superclass
|
|
313
|
-
# @param default [Object] Default value if not found
|
|
314
|
-
# @return [Object] The resolved value
|
|
315
|
-
def inherited_or_default(method, default)
|
|
316
|
-
superclass.respond_to?(method) ? superclass.send(method) : default
|
|
317
|
-
end
|
|
318
86
|
end
|
|
319
87
|
|
|
320
88
|
# @!attribute [r] model
|
|
@@ -344,24 +112,6 @@ module RubyLLM
|
|
|
344
112
|
@client = build_client
|
|
345
113
|
end
|
|
346
114
|
|
|
347
|
-
# Executes the agent and returns the processed response
|
|
348
|
-
#
|
|
349
|
-
# Handles caching, dry-run mode, and delegates to uncached_call
|
|
350
|
-
# for actual LLM execution.
|
|
351
|
-
#
|
|
352
|
-
# @yield [chunk] Yields chunks when streaming is enabled
|
|
353
|
-
# @yieldparam chunk [RubyLLM::Chunk] A streaming chunk with content
|
|
354
|
-
# @return [Object] The processed LLM response
|
|
355
|
-
def call(&block)
|
|
356
|
-
return dry_run_response if @options[:dry_run]
|
|
357
|
-
return uncached_call(&block) if @options[:skip_cache] || !self.class.cache_enabled?
|
|
358
|
-
|
|
359
|
-
# Note: Cached responses don't stream (already complete)
|
|
360
|
-
cache_store.fetch(cache_key, expires_in: self.class.cache_ttl) do
|
|
361
|
-
uncached_call(&block)
|
|
362
|
-
end
|
|
363
|
-
end
|
|
364
|
-
|
|
365
115
|
# @!group Template Methods (override in subclasses)
|
|
366
116
|
|
|
367
117
|
# User prompt to send to the LLM
|
|
@@ -401,546 +151,6 @@ module RubyLLM
|
|
|
401
151
|
end
|
|
402
152
|
|
|
403
153
|
# @!endgroup
|
|
404
|
-
|
|
405
|
-
# Returns the consolidated reliability configuration for this agent instance
|
|
406
|
-
#
|
|
407
|
-
# @return [Hash] Reliability config with :retries, :fallback_models, :total_timeout, :circuit_breaker
|
|
408
|
-
def reliability_config
|
|
409
|
-
default_retries = RubyLLM::Agents.configuration.default_retries
|
|
410
|
-
{
|
|
411
|
-
retries: self.class.retries || default_retries,
|
|
412
|
-
fallback_models: self.class.fallback_models,
|
|
413
|
-
total_timeout: self.class.total_timeout,
|
|
414
|
-
circuit_breaker: self.class.circuit_breaker_config
|
|
415
|
-
}
|
|
416
|
-
end
|
|
417
|
-
|
|
418
|
-
# Returns whether any reliability features are enabled for this agent
|
|
419
|
-
#
|
|
420
|
-
# @return [Boolean] true if retries, fallbacks, or circuit breaker is configured
|
|
421
|
-
def reliability_enabled?
|
|
422
|
-
config = reliability_config
|
|
423
|
-
(config[:retries]&.dig(:max) || 0) > 0 ||
|
|
424
|
-
config[:fallback_models]&.any? ||
|
|
425
|
-
config[:circuit_breaker].present?
|
|
426
|
-
end
|
|
427
|
-
|
|
428
|
-
# Returns prompt info without making an API call (debug mode)
|
|
429
|
-
#
|
|
430
|
-
# @return [Result] A Result with dry run configuration info
|
|
431
|
-
def dry_run_response
|
|
432
|
-
Result.new(
|
|
433
|
-
content: {
|
|
434
|
-
dry_run: true,
|
|
435
|
-
agent: self.class.name,
|
|
436
|
-
model: model,
|
|
437
|
-
temperature: temperature,
|
|
438
|
-
timeout: self.class.timeout,
|
|
439
|
-
system_prompt: system_prompt,
|
|
440
|
-
user_prompt: user_prompt,
|
|
441
|
-
attachments: @options[:with],
|
|
442
|
-
schema: schema&.class&.name,
|
|
443
|
-
streaming: self.class.streaming,
|
|
444
|
-
tools: self.class.tools.map { |t| t.respond_to?(:name) ? t.name : t.to_s }
|
|
445
|
-
},
|
|
446
|
-
model_id: model,
|
|
447
|
-
temperature: temperature,
|
|
448
|
-
streaming: self.class.streaming
|
|
449
|
-
)
|
|
450
|
-
end
|
|
451
|
-
|
|
452
|
-
private
|
|
453
|
-
|
|
454
|
-
# Executes the agent without caching
|
|
455
|
-
#
|
|
456
|
-
# Routes to reliability-enabled execution if configured, otherwise
|
|
457
|
-
# uses simple single-attempt execution.
|
|
458
|
-
#
|
|
459
|
-
# @yield [chunk] Yields chunks when streaming is enabled
|
|
460
|
-
# @return [Object] The processed response
|
|
461
|
-
def uncached_call(&block)
|
|
462
|
-
if reliability_enabled?
|
|
463
|
-
execute_with_reliability(&block)
|
|
464
|
-
else
|
|
465
|
-
instrument_execution { execute_single_attempt(&block) }
|
|
466
|
-
end
|
|
467
|
-
end
|
|
468
|
-
|
|
469
|
-
# Executes a single LLM attempt with timeout
|
|
470
|
-
#
|
|
471
|
-
# @param model_override [String, nil] Optional model to use instead of default
|
|
472
|
-
# @yield [chunk] Yields chunks when streaming is enabled
|
|
473
|
-
# @return [Result] A Result object with processed content and metadata
|
|
474
|
-
def execute_single_attempt(model_override: nil, &block)
|
|
475
|
-
current_client = model_override ? build_client_with_model(model_override) : client
|
|
476
|
-
@execution_started_at ||= Time.current
|
|
477
|
-
reset_accumulated_tool_calls!
|
|
478
|
-
|
|
479
|
-
Timeout.timeout(self.class.timeout) do
|
|
480
|
-
if self.class.streaming && block_given?
|
|
481
|
-
execute_with_streaming(current_client, &block)
|
|
482
|
-
else
|
|
483
|
-
response = current_client.ask(user_prompt, **ask_options)
|
|
484
|
-
extract_tool_calls_from_client(current_client)
|
|
485
|
-
capture_response(response)
|
|
486
|
-
build_result(process_response(response), response)
|
|
487
|
-
end
|
|
488
|
-
end
|
|
489
|
-
end
|
|
490
|
-
|
|
491
|
-
# Executes an LLM request with streaming enabled
|
|
492
|
-
#
|
|
493
|
-
# Yields chunks to the provided block as they arrive and tracks
|
|
494
|
-
# time to first token for latency analysis.
|
|
495
|
-
#
|
|
496
|
-
# @param current_client [RubyLLM::Chat] The configured client
|
|
497
|
-
# @yield [chunk] Yields each chunk as it arrives
|
|
498
|
-
# @yieldparam chunk [RubyLLM::Chunk] A streaming chunk
|
|
499
|
-
# @return [Result] A Result object with processed content and metadata
|
|
500
|
-
def execute_with_streaming(current_client, &block)
|
|
501
|
-
first_chunk_at = nil
|
|
502
|
-
|
|
503
|
-
response = current_client.ask(user_prompt, **ask_options) do |chunk|
|
|
504
|
-
first_chunk_at ||= Time.current
|
|
505
|
-
yield chunk if block_given?
|
|
506
|
-
end
|
|
507
|
-
|
|
508
|
-
if first_chunk_at && @execution_started_at
|
|
509
|
-
@time_to_first_token_ms = ((first_chunk_at - @execution_started_at) * 1000).to_i
|
|
510
|
-
end
|
|
511
|
-
|
|
512
|
-
extract_tool_calls_from_client(current_client)
|
|
513
|
-
capture_response(response)
|
|
514
|
-
build_result(process_response(response), response)
|
|
515
|
-
end
|
|
516
|
-
|
|
517
|
-
# Executes the agent with retry/fallback/circuit breaker support
|
|
518
|
-
#
|
|
519
|
-
# @yield [chunk] Yields chunks when streaming is enabled
|
|
520
|
-
# @return [Object] The processed response
|
|
521
|
-
# @raise [Reliability::AllModelsExhaustedError] If all models fail
|
|
522
|
-
# @raise [Reliability::BudgetExceededError] If budget limits exceeded
|
|
523
|
-
# @raise [Reliability::TotalTimeoutError] If total timeout exceeded
|
|
524
|
-
def execute_with_reliability(&block)
|
|
525
|
-
config = reliability_config
|
|
526
|
-
models_to_try = [model, *config[:fallback_models]].uniq
|
|
527
|
-
total_deadline = config[:total_timeout] ? Time.current + config[:total_timeout] : nil
|
|
528
|
-
started_at = Time.current
|
|
529
|
-
|
|
530
|
-
# Pre-check budget
|
|
531
|
-
BudgetTracker.check_budget!(self.class.name) if RubyLLM::Agents.configuration.budgets_enabled?
|
|
532
|
-
|
|
533
|
-
instrument_execution_with_attempts(models_to_try: models_to_try) do |attempt_tracker|
|
|
534
|
-
last_error = nil
|
|
535
|
-
|
|
536
|
-
models_to_try.each do |current_model|
|
|
537
|
-
# Check circuit breaker
|
|
538
|
-
breaker = get_circuit_breaker(current_model)
|
|
539
|
-
if breaker&.open?
|
|
540
|
-
attempt_tracker.record_short_circuit(current_model)
|
|
541
|
-
next
|
|
542
|
-
end
|
|
543
|
-
|
|
544
|
-
retries_remaining = config[:retries]&.dig(:max) || 0
|
|
545
|
-
attempt_index = 0
|
|
546
|
-
|
|
547
|
-
loop do
|
|
548
|
-
# Check total timeout
|
|
549
|
-
if total_deadline && Time.current > total_deadline
|
|
550
|
-
elapsed = Time.current - started_at
|
|
551
|
-
raise Reliability::TotalTimeoutError.new(config[:total_timeout], elapsed)
|
|
552
|
-
end
|
|
553
|
-
|
|
554
|
-
attempt = attempt_tracker.start_attempt(current_model)
|
|
555
|
-
|
|
556
|
-
begin
|
|
557
|
-
result = execute_single_attempt(model_override: current_model, &block)
|
|
558
|
-
attempt_tracker.complete_attempt(attempt, success: true, response: @last_response)
|
|
559
|
-
|
|
560
|
-
# Record success in circuit breaker
|
|
561
|
-
breaker&.record_success!
|
|
562
|
-
|
|
563
|
-
# Record budget spend
|
|
564
|
-
if @last_response && RubyLLM::Agents.configuration.budgets_enabled?
|
|
565
|
-
record_attempt_cost(attempt_tracker)
|
|
566
|
-
end
|
|
567
|
-
|
|
568
|
-
# Use throw instead of return to allow instrument_execution_with_attempts
|
|
569
|
-
# to properly complete the execution record before returning
|
|
570
|
-
throw :execution_success, result
|
|
571
|
-
|
|
572
|
-
rescue *retryable_errors(config) => e
|
|
573
|
-
last_error = e
|
|
574
|
-
attempt_tracker.complete_attempt(attempt, success: false, error: e)
|
|
575
|
-
breaker&.record_failure!
|
|
576
|
-
|
|
577
|
-
if retries_remaining > 0 && !past_deadline?(total_deadline)
|
|
578
|
-
retries_remaining -= 1
|
|
579
|
-
attempt_index += 1
|
|
580
|
-
retries_config = config[:retries] || {}
|
|
581
|
-
delay = Reliability.calculate_backoff(
|
|
582
|
-
strategy: retries_config[:backoff] || :exponential,
|
|
583
|
-
base: retries_config[:base] || 0.4,
|
|
584
|
-
max_delay: retries_config[:max_delay] || 3.0,
|
|
585
|
-
attempt: attempt_index
|
|
586
|
-
)
|
|
587
|
-
sleep(delay)
|
|
588
|
-
else
|
|
589
|
-
break # Move to next model
|
|
590
|
-
end
|
|
591
|
-
|
|
592
|
-
rescue StandardError => e
|
|
593
|
-
# Non-retryable error - record and move to next model
|
|
594
|
-
last_error = e
|
|
595
|
-
attempt_tracker.complete_attempt(attempt, success: false, error: e)
|
|
596
|
-
breaker&.record_failure!
|
|
597
|
-
break
|
|
598
|
-
end
|
|
599
|
-
end
|
|
600
|
-
end
|
|
601
|
-
|
|
602
|
-
# All models exhausted
|
|
603
|
-
raise Reliability::AllModelsExhaustedError.new(models_to_try, last_error)
|
|
604
|
-
end
|
|
605
|
-
end
|
|
606
|
-
|
|
607
|
-
# Returns the list of retryable error classes
|
|
608
|
-
#
|
|
609
|
-
# @param config [Hash] Reliability configuration
|
|
610
|
-
# @return [Array<Class>] Error classes to retry on
|
|
611
|
-
def retryable_errors(config)
|
|
612
|
-
custom_errors = config[:retries]&.dig(:on) || []
|
|
613
|
-
Reliability.default_retryable_errors + custom_errors
|
|
614
|
-
end
|
|
615
|
-
|
|
616
|
-
# Checks if the total deadline has passed
|
|
617
|
-
#
|
|
618
|
-
# @param deadline [Time, nil] The deadline
|
|
619
|
-
# @return [Boolean] true if past deadline
|
|
620
|
-
def past_deadline?(deadline)
|
|
621
|
-
deadline && Time.current > deadline
|
|
622
|
-
end
|
|
623
|
-
|
|
624
|
-
# Gets or creates a circuit breaker for a model
|
|
625
|
-
#
|
|
626
|
-
# @param model_id [String] The model identifier
|
|
627
|
-
# @return [CircuitBreaker, nil] The circuit breaker or nil if not configured
|
|
628
|
-
def get_circuit_breaker(model_id)
|
|
629
|
-
config = reliability_config[:circuit_breaker]
|
|
630
|
-
return nil unless config
|
|
631
|
-
|
|
632
|
-
CircuitBreaker.from_config(self.class.name, model_id, config)
|
|
633
|
-
end
|
|
634
|
-
|
|
635
|
-
# Records cost from an attempt to the budget tracker
|
|
636
|
-
#
|
|
637
|
-
# @param attempt_tracker [AttemptTracker] The attempt tracker
|
|
638
|
-
# @return [void]
|
|
639
|
-
def record_attempt_cost(attempt_tracker)
|
|
640
|
-
successful = attempt_tracker.successful_attempt
|
|
641
|
-
return unless successful
|
|
642
|
-
|
|
643
|
-
# Calculate cost for this execution
|
|
644
|
-
# Note: Full cost calculation happens in instrumentation, but we
|
|
645
|
-
# record the spend here for budget tracking
|
|
646
|
-
model_info = resolve_model_info(successful[:model_id])
|
|
647
|
-
return unless model_info&.pricing
|
|
648
|
-
|
|
649
|
-
input_tokens = successful[:input_tokens] || 0
|
|
650
|
-
output_tokens = successful[:output_tokens] || 0
|
|
651
|
-
|
|
652
|
-
input_price = model_info.pricing.text_tokens&.input || 0
|
|
653
|
-
output_price = model_info.pricing.text_tokens&.output || 0
|
|
654
|
-
|
|
655
|
-
total_cost = (input_tokens / 1_000_000.0 * input_price) +
|
|
656
|
-
(output_tokens / 1_000_000.0 * output_price)
|
|
657
|
-
|
|
658
|
-
BudgetTracker.record_spend!(self.class.name, total_cost)
|
|
659
|
-
rescue StandardError => e
|
|
660
|
-
Rails.logger.warn("[RubyLLM::Agents] Failed to record budget spend: #{e.message}")
|
|
661
|
-
end
|
|
662
|
-
|
|
663
|
-
# Resolves model info for cost calculation
|
|
664
|
-
#
|
|
665
|
-
# @param model_id [String] The model identifier
|
|
666
|
-
# @return [Object, nil] Model info or nil
|
|
667
|
-
def resolve_model_info(model_id)
|
|
668
|
-
RubyLLM::Models.resolve(model_id)
|
|
669
|
-
rescue StandardError
|
|
670
|
-
nil
|
|
671
|
-
end
|
|
672
|
-
|
|
673
|
-
# Builds a client with a specific model
|
|
674
|
-
#
|
|
675
|
-
# @param model_id [String] The model identifier
|
|
676
|
-
# @return [RubyLLM::Chat] Configured chat client
|
|
677
|
-
def build_client_with_model(model_id)
|
|
678
|
-
client = RubyLLM.chat
|
|
679
|
-
.with_model(model_id)
|
|
680
|
-
.with_temperature(temperature)
|
|
681
|
-
client = client.with_instructions(system_prompt) if system_prompt
|
|
682
|
-
client = client.with_schema(schema) if schema
|
|
683
|
-
client = client.with_tools(*self.class.tools) if self.class.tools.any?
|
|
684
|
-
client
|
|
685
|
-
end
|
|
686
|
-
|
|
687
|
-
# Returns the configured cache store
|
|
688
|
-
#
|
|
689
|
-
# @return [ActiveSupport::Cache::Store] The cache store
|
|
690
|
-
def cache_store
|
|
691
|
-
RubyLLM::Agents.configuration.cache_store
|
|
692
|
-
end
|
|
693
|
-
|
|
694
|
-
# Generates the full cache key for this agent invocation
|
|
695
|
-
#
|
|
696
|
-
# @return [String] Cache key in format "ruby_llm_agent/ClassName/version/hash"
|
|
697
|
-
def cache_key
|
|
698
|
-
["ruby_llm_agent", self.class.name, self.class.version, cache_key_hash].join("/")
|
|
699
|
-
end
|
|
700
|
-
|
|
701
|
-
# Generates a hash of the cache key data
|
|
702
|
-
#
|
|
703
|
-
# @return [String] SHA256 hex digest of the cache key data
|
|
704
|
-
def cache_key_hash
|
|
705
|
-
Digest::SHA256.hexdigest(cache_key_data.to_json)
|
|
706
|
-
end
|
|
707
|
-
|
|
708
|
-
# Returns data to include in cache key generation
|
|
709
|
-
#
|
|
710
|
-
# Override to customize what parameters affect cache invalidation.
|
|
711
|
-
#
|
|
712
|
-
# @return [Hash] Data to hash for cache key
|
|
713
|
-
def cache_key_data
|
|
714
|
-
@options.except(:skip_cache, :dry_run, :with)
|
|
715
|
-
end
|
|
716
|
-
|
|
717
|
-
# Returns options to pass to the ask method
|
|
718
|
-
#
|
|
719
|
-
# Currently supports :with for attachments (images, PDFs, etc.)
|
|
720
|
-
#
|
|
721
|
-
# @return [Hash] Options for the ask call
|
|
722
|
-
def ask_options
|
|
723
|
-
opts = {}
|
|
724
|
-
opts[:with] = @options[:with] if @options[:with]
|
|
725
|
-
opts
|
|
726
|
-
end
|
|
727
|
-
|
|
728
|
-
# Validates that all required parameters are present
|
|
729
|
-
#
|
|
730
|
-
# @raise [ArgumentError] If required parameters are missing
|
|
731
|
-
# @return [void]
|
|
732
|
-
def validate_required_params!
|
|
733
|
-
required = self.class.params.select { |_, v| v[:required] }.keys
|
|
734
|
-
missing = required.reject { |p| @options.key?(p) || @options.key?(p.to_s) }
|
|
735
|
-
raise ArgumentError, "#{self.class} missing required params: #{missing.join(', ')}" if missing.any?
|
|
736
|
-
end
|
|
737
|
-
|
|
738
|
-
# Builds and configures the RubyLLM client
|
|
739
|
-
#
|
|
740
|
-
# @return [RubyLLM::Chat] Configured chat client
|
|
741
|
-
def build_client
|
|
742
|
-
client = RubyLLM.chat
|
|
743
|
-
.with_model(model)
|
|
744
|
-
.with_temperature(temperature)
|
|
745
|
-
client = client.with_instructions(system_prompt) if system_prompt
|
|
746
|
-
client = client.with_schema(schema) if schema
|
|
747
|
-
client = client.with_tools(*self.class.tools) if self.class.tools.any?
|
|
748
|
-
client
|
|
749
|
-
end
|
|
750
|
-
|
|
751
|
-
# Builds a client with pre-populated conversation history
|
|
752
|
-
#
|
|
753
|
-
# Useful for multi-turn conversations or providing context.
|
|
754
|
-
#
|
|
755
|
-
# @param messages [Array<Hash>] Messages with :role and :content keys
|
|
756
|
-
# @return [RubyLLM::Chat] Client with messages added
|
|
757
|
-
# @example
|
|
758
|
-
# build_client_with_messages([
|
|
759
|
-
# { role: "user", content: "Hello" },
|
|
760
|
-
# { role: "assistant", content: "Hi there!" }
|
|
761
|
-
# ])
|
|
762
|
-
def build_client_with_messages(messages)
|
|
763
|
-
messages.reduce(build_client) do |client, message|
|
|
764
|
-
client.with_message(message[:role], message[:content])
|
|
765
|
-
end
|
|
766
|
-
end
|
|
767
|
-
|
|
768
|
-
# @!group Result Building
|
|
769
|
-
|
|
770
|
-
# Builds a Result object from processed content and response metadata
|
|
771
|
-
#
|
|
772
|
-
# @param content [Hash, String] The processed response content
|
|
773
|
-
# @param response [RubyLLM::Message] The raw LLM response
|
|
774
|
-
# @return [Result] A Result object with full execution metadata
|
|
775
|
-
def build_result(content, response)
|
|
776
|
-
completed_at = Time.current
|
|
777
|
-
input_tokens = result_response_value(response, :input_tokens)
|
|
778
|
-
output_tokens = result_response_value(response, :output_tokens)
|
|
779
|
-
response_model_id = result_response_value(response, :model_id)
|
|
780
|
-
|
|
781
|
-
Result.new(
|
|
782
|
-
content: content,
|
|
783
|
-
input_tokens: input_tokens,
|
|
784
|
-
output_tokens: output_tokens,
|
|
785
|
-
cached_tokens: result_response_value(response, :cached_tokens, 0),
|
|
786
|
-
cache_creation_tokens: result_response_value(response, :cache_creation_tokens, 0),
|
|
787
|
-
model_id: model,
|
|
788
|
-
chosen_model_id: response_model_id || model,
|
|
789
|
-
temperature: temperature,
|
|
790
|
-
started_at: @execution_started_at,
|
|
791
|
-
completed_at: completed_at,
|
|
792
|
-
duration_ms: result_duration_ms(completed_at),
|
|
793
|
-
time_to_first_token_ms: @time_to_first_token_ms,
|
|
794
|
-
finish_reason: result_finish_reason(response),
|
|
795
|
-
streaming: self.class.streaming,
|
|
796
|
-
input_cost: result_input_cost(input_tokens, response_model_id),
|
|
797
|
-
output_cost: result_output_cost(output_tokens, response_model_id),
|
|
798
|
-
total_cost: result_total_cost(input_tokens, output_tokens, response_model_id),
|
|
799
|
-
tool_calls: @accumulated_tool_calls,
|
|
800
|
-
tool_calls_count: @accumulated_tool_calls.size
|
|
801
|
-
)
|
|
802
|
-
end
|
|
803
|
-
|
|
804
|
-
# Safely extracts a value from the response object
|
|
805
|
-
#
|
|
806
|
-
# @param response [Object] The response object
|
|
807
|
-
# @param method [Symbol] The method to call
|
|
808
|
-
# @param default [Object] Default value if method doesn't exist
|
|
809
|
-
# @return [Object] The extracted value or default
|
|
810
|
-
def result_response_value(response, method, default = nil)
|
|
811
|
-
return default unless response.respond_to?(method)
|
|
812
|
-
response.send(method) || default
|
|
813
|
-
end
|
|
814
|
-
|
|
815
|
-
# Calculates execution duration in milliseconds
|
|
816
|
-
#
|
|
817
|
-
# @param completed_at [Time] When execution completed
|
|
818
|
-
# @return [Integer, nil] Duration in ms or nil
|
|
819
|
-
def result_duration_ms(completed_at)
|
|
820
|
-
return nil unless @execution_started_at
|
|
821
|
-
((completed_at - @execution_started_at) * 1000).to_i
|
|
822
|
-
end
|
|
823
|
-
|
|
824
|
-
# Extracts finish reason from response
|
|
825
|
-
#
|
|
826
|
-
# @param response [Object] The response object
|
|
827
|
-
# @return [String, nil] Normalized finish reason
|
|
828
|
-
def result_finish_reason(response)
|
|
829
|
-
reason = result_response_value(response, :finish_reason) ||
|
|
830
|
-
result_response_value(response, :stop_reason)
|
|
831
|
-
return nil unless reason
|
|
832
|
-
|
|
833
|
-
# Normalize to standard values
|
|
834
|
-
case reason.to_s.downcase
|
|
835
|
-
when "stop", "end_turn" then "stop"
|
|
836
|
-
when "length", "max_tokens" then "length"
|
|
837
|
-
when "content_filter", "safety" then "content_filter"
|
|
838
|
-
when "tool_calls", "tool_use" then "tool_calls"
|
|
839
|
-
else "other"
|
|
840
|
-
end
|
|
841
|
-
end
|
|
842
|
-
|
|
843
|
-
# Calculates input cost from tokens
|
|
844
|
-
#
|
|
845
|
-
# @param input_tokens [Integer, nil] Number of input tokens
|
|
846
|
-
# @param response_model_id [String, nil] Model that responded
|
|
847
|
-
# @return [Float, nil] Input cost in USD
|
|
848
|
-
def result_input_cost(input_tokens, response_model_id)
|
|
849
|
-
return nil unless input_tokens
|
|
850
|
-
model_info = result_model_info(response_model_id)
|
|
851
|
-
return nil unless model_info&.pricing
|
|
852
|
-
price = model_info.pricing.text_tokens&.input || 0
|
|
853
|
-
(input_tokens / 1_000_000.0 * price).round(6)
|
|
854
|
-
end
|
|
855
|
-
|
|
856
|
-
# Calculates output cost from tokens
|
|
857
|
-
#
|
|
858
|
-
# @param output_tokens [Integer, nil] Number of output tokens
|
|
859
|
-
# @param response_model_id [String, nil] Model that responded
|
|
860
|
-
# @return [Float, nil] Output cost in USD
|
|
861
|
-
def result_output_cost(output_tokens, response_model_id)
|
|
862
|
-
return nil unless output_tokens
|
|
863
|
-
model_info = result_model_info(response_model_id)
|
|
864
|
-
return nil unless model_info&.pricing
|
|
865
|
-
price = model_info.pricing.text_tokens&.output || 0
|
|
866
|
-
(output_tokens / 1_000_000.0 * price).round(6)
|
|
867
|
-
end
|
|
868
|
-
|
|
869
|
-
# Calculates total cost from tokens
|
|
870
|
-
#
|
|
871
|
-
# @param input_tokens [Integer, nil] Number of input tokens
|
|
872
|
-
# @param output_tokens [Integer, nil] Number of output tokens
|
|
873
|
-
# @param response_model_id [String, nil] Model that responded
|
|
874
|
-
# @return [Float, nil] Total cost in USD
|
|
875
|
-
def result_total_cost(input_tokens, output_tokens, response_model_id)
|
|
876
|
-
input_cost = result_input_cost(input_tokens, response_model_id)
|
|
877
|
-
output_cost = result_output_cost(output_tokens, response_model_id)
|
|
878
|
-
return nil unless input_cost || output_cost
|
|
879
|
-
((input_cost || 0) + (output_cost || 0)).round(6)
|
|
880
|
-
end
|
|
881
|
-
|
|
882
|
-
# Resolves model info for cost calculation
|
|
883
|
-
#
|
|
884
|
-
# @param response_model_id [String, nil] Model ID from response
|
|
885
|
-
# @return [Object, nil] Model info or nil
|
|
886
|
-
def result_model_info(response_model_id)
|
|
887
|
-
lookup_id = response_model_id || model
|
|
888
|
-
return nil unless lookup_id
|
|
889
|
-
model_obj, _provider = RubyLLM::Models.resolve(lookup_id)
|
|
890
|
-
model_obj
|
|
891
|
-
rescue StandardError
|
|
892
|
-
nil
|
|
893
|
-
end
|
|
894
|
-
|
|
895
|
-
# @!endgroup
|
|
896
|
-
|
|
897
|
-
# @!group Tool Call Tracking
|
|
898
|
-
|
|
899
|
-
# Resets accumulated tool calls for a new execution
|
|
900
|
-
#
|
|
901
|
-
# @return [void]
|
|
902
|
-
def reset_accumulated_tool_calls!
|
|
903
|
-
@accumulated_tool_calls = []
|
|
904
|
-
end
|
|
905
|
-
|
|
906
|
-
# Extracts tool calls from all assistant messages in the conversation
|
|
907
|
-
#
|
|
908
|
-
# RubyLLM handles tool call loops internally. After ask() completes,
|
|
909
|
-
# the conversation history contains all intermediate assistant messages
|
|
910
|
-
# that had tool_calls. This method extracts those tool calls.
|
|
911
|
-
#
|
|
912
|
-
# @param client [RubyLLM::Chat] The chat client with conversation history
|
|
913
|
-
# @return [void]
|
|
914
|
-
def extract_tool_calls_from_client(client)
|
|
915
|
-
return unless client.respond_to?(:messages)
|
|
916
|
-
|
|
917
|
-
client.messages.each do |message|
|
|
918
|
-
next unless message.role == :assistant
|
|
919
|
-
next unless message.respond_to?(:tool_calls) && message.tool_calls.present?
|
|
920
|
-
|
|
921
|
-
message.tool_calls.each_value do |tool_call|
|
|
922
|
-
@accumulated_tool_calls << serialize_tool_call(tool_call)
|
|
923
|
-
end
|
|
924
|
-
end
|
|
925
|
-
end
|
|
926
|
-
|
|
927
|
-
# Serializes a single tool call to a hash
|
|
928
|
-
#
|
|
929
|
-
# @param tool_call [Object] The tool call object
|
|
930
|
-
# @return [Hash] Serialized tool call
|
|
931
|
-
def serialize_tool_call(tool_call)
|
|
932
|
-
if tool_call.respond_to?(:to_h)
|
|
933
|
-
tool_call.to_h.transform_keys(&:to_s)
|
|
934
|
-
else
|
|
935
|
-
{
|
|
936
|
-
"id" => tool_call.id,
|
|
937
|
-
"name" => tool_call.name,
|
|
938
|
-
"arguments" => tool_call.arguments
|
|
939
|
-
}
|
|
940
|
-
end
|
|
941
|
-
end
|
|
942
|
-
|
|
943
|
-
# @!endgroup
|
|
944
154
|
end
|
|
945
155
|
end
|
|
946
156
|
end
|