ruby_llm-agents 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 03e22b362e20b0322d49726f9bb22f202d4fa642691f3a3b885d5cc4e7c661cf
-  data.tar.gz: 327a34b0ef344b4a65edf23fda0360b1e2bcc5f33e43881521418686f2113565
+  metadata.gz: d0cf7b20ca960eab7d26da1c568c5aa5078e12e887a4d93c9b472d6897cf1e5e
+  data.tar.gz: 59372d2c80bd8fdb23d49ffdd672911948bf1b78da8243aaac7df926a378bd59
 SHA512:
-  metadata.gz: 83bdf11edc73064e92f98b220849e5d29ad62aa79ee853880eda80763c70e4a7aec13129f299df70db6dc3a9d08c102a12ca857ca6557b9b6a56db81430ea38a
-  data.tar.gz: 8c8f29bb00b446e44d2b15578b823812f72e533976244407549bcf4414db51a8e33676fdbb36ebab41a7f22d2d85a1a63de1d2c0c7e40d5b58ebfad128a9a634
+  metadata.gz: 1902ad245d20d405fe69633e122a83302bbd5b95e5b65fe0d24b1da14b0effaf27e9308e4a5a42bd9072545b2d128ed173214812667727fb2ea0de43948f1ee1
+  data.tar.gz: d3937f03bc62481c6257c55c9a7f1a8c6eccc9fc08a759c8f40334b21000a12cc67e5277ca0d018b551c3abb6b077d4ffc1caa8bfaa3f2d8e784aac98c1e9bf8

data/README.md

@@ -12,6 +12,10 @@ A powerful Rails engine for building, managing, and monitoring LLM-powered agent
 - **🛠️ Generators** - Quickly scaffold new agents with customizable templates
 - **🔍 Anomaly Detection** - Automatic warnings for unusual cost or duration patterns
 - **🎯 Type Safety** - Structured output with RubyLLM::Schema integration
+- **🔄 Reliability** - Automatic retries, model fallbacks, and circuit breakers for resilient agents
+- **💵 Budget Controls** - Daily/monthly spending limits with hard and soft enforcement
+- **🔔 Alerts** - Slack, webhook, and custom notifications for budget and circuit breaker events
+- **🔒 PII Redaction** - Automatic sanitization of sensitive data in execution logs
 
 ## Requirements
 
@@ -394,6 +398,269 @@ class RecommendationAgent < ApplicationAgent
 end
 ```
 
+## Reliability Features
+
+RubyLLM::Agents provides built-in reliability features to make your agents resilient against API failures, rate limits, and transient errors.
+
+### Automatic Retries
+
+Configure retry behavior for transient failures:
+
+```ruby
+class ReliableAgent < ApplicationAgent
+  model "gpt-4o"
+
+  # Retry up to 3 times with exponential backoff
+  retries max: 3, backoff: :exponential, base: 0.5, max_delay: 10.0
+
+  # Only retry on specific errors (defaults include timeout, network errors)
+  retries max: 3, on: [Timeout::Error, Net::ReadTimeout, Faraday::TimeoutError]
+
+  param :query, required: true
+
+  def user_prompt
+    query
+  end
+end
+```
+
+Backoff strategies:
+- `:exponential` - Delay doubles each retry (0.5s, 1s, 2s, 4s...)
+- `:constant` - Same delay each retry
+- Jitter is automatically added to prevent thundering herd
+
+### Model Fallbacks
+
+Automatically try alternative models if the primary fails:
+
+```ruby
+class FallbackAgent < ApplicationAgent
+  model "gpt-4o"
+
+  # Try these models in order if primary fails
+  fallback_models "gpt-4o-mini", "claude-3-5-sonnet", "gemini-2.0-flash"
+
+  # Combine with retries
+  retries max: 2
+  fallback_models "gpt-4o-mini", "claude-3-sonnet"
+
+  param :query, required: true
+
+  def user_prompt
+    query
+  end
+end
+```
+
+The agent will try `gpt-4o` (with 2 retries), then `gpt-4o-mini` (with 2 retries), and so on.
+
+### Circuit Breaker
+
+Prevent cascading failures by temporarily blocking requests to failing models:
+
+```ruby
+class ProtectedAgent < ApplicationAgent
+  model "gpt-4o"
+  fallback_models "claude-3-sonnet"
+
+  # Open circuit after 10 errors within 60 seconds
+  # Keep circuit open for 5 minutes before retrying
+  circuit_breaker errors: 10, within: 60, cooldown: 300
+
+  param :query, required: true
+
+  def user_prompt
+    query
+  end
+end
+```
+
+Circuit breaker states:
+- **Closed** - Normal operation, requests pass through
+- **Open** - Model is blocked, requests skip to fallback or fail fast
+- **Half-Open** - After cooldown, one request is allowed to test recovery
+
+### Total Timeout
+
+Set a maximum time for the entire operation including all retries:
+
+```ruby
+class TimeBoundAgent < ApplicationAgent
+  model "gpt-4o"
+  retries max: 5
+  fallback_models "gpt-4o-mini"
+
+  # Abort everything after 30 seconds total
+  total_timeout 30
+
+  param :query, required: true
+
+  def user_prompt
+    query
+  end
+end
+```
+
+### Viewing Attempt Details
+
+When reliability features are enabled, the dashboard shows all attempts:
+
+```ruby
+execution = RubyLLM::Agents::Execution.last
+
+# Check if retries/fallbacks were used
+execution.has_retries?      # => true
+execution.used_fallback?    # => true
+execution.attempts_count    # => 3
+
+# Get attempt details
+execution.attempts.each do |attempt|
+  puts "Model: #{attempt['model_id']}"
+  puts "Duration: #{attempt['duration_ms']}ms"
+  puts "Error: #{attempt['error_class']}" if attempt['error_class']
+  puts "Short-circuited: #{attempt['short_circuited']}"
+end
+
+# Find the successful attempt
+execution.successful_attempt  # => Hash with attempt data
+execution.chosen_model_id     # => "claude-3-sonnet" (the model that succeeded)
+```
+
+## Governance & Cost Controls
+
+### Budget Limits
+
+Set spending limits at global and per-agent levels:
+
+```ruby
+# config/initializers/ruby_llm_agents.rb
+RubyLLM::Agents.configure do |config|
+  config.budgets = {
+    # Global limits apply to all agents combined
+    global_daily: 100.0,      # $100/day across all agents
+    global_monthly: 2000.0,   # $2000/month across all agents
+
+    # Per-agent limits
+    per_agent_daily: {
+      "ExpensiveAgent" => 50.0,  # $50/day for this agent
+      "CheapAgent" => 5.0        # $5/day for this agent
+    },
+    per_agent_monthly: {
+      "ExpensiveAgent" => 500.0
+    },
+
+    # Enforcement mode
+    # :hard - Block requests when budget exceeded
+    # :soft - Allow requests but log warnings
+    enforcement: :hard
+  }
+end
+```
+
+Querying budget status:
+
+```ruby
+# Get current budget status
+status = RubyLLM::Agents::BudgetTracker.status(agent_type: "MyAgent")
+# => {
+#   global_daily: { limit: 100.0, current: 45.50, remaining: 54.50, percentage_used: 45.5 },
+#   global_monthly: { limit: 2000.0, current: 890.0, remaining: 1110.0, percentage_used: 44.5 }
+# }
+
+# Check remaining budget
+RubyLLM::Agents::BudgetTracker.remaining_budget(:global, :daily)
+# => 54.50
+```
+
+### Alerts
+
+Get notified when important events occur:
+
+```ruby
+# config/initializers/ruby_llm_agents.rb
+RubyLLM::Agents.configure do |config|
+  config.alerts = {
+    # Events to alert on
+    on_events: [
+      :budget_soft_cap,   # Budget threshold reached (configurable %)
+      :budget_hard_cap,   # Budget exceeded (with hard enforcement)
+      :breaker_open       # Circuit breaker opened
+    ],
+
+    # Slack webhook
+    slack_webhook_url: ENV['SLACK_WEBHOOK_URL'],
+
+    # Generic webhook (receives JSON payload)
+    webhook_url: "https://your-app.com/webhooks/llm-alerts",
+
+    # Custom handler
+    custom: ->(event, payload) {
+      # event: :budget_hard_cap
+      # payload: { scope: :global_daily, limit: 100.0, current: 105.0 }
+
+      MyNotificationService.notify(
+        title: "LLM Budget Alert",
+        message: "#{event}: #{payload}"
+      )
+    }
+  }
+end
+```
+
+Alert payload examples:
+
+```ruby
+# Budget alert
+{
+  event: :budget_hard_cap,
+  scope: :global_daily,
+  limit: 100.0,
+  current: 105.50,
+  agent_type: "ExpensiveAgent"
+}
+
+# Circuit breaker alert
+{
+  event: :breaker_open,
+  agent_type: "MyAgent",
+  model_id: "gpt-4o",
+  failure_count: 10,
+  window_seconds: 60
+}
+```
+
+### PII Redaction
+
+Automatically redact sensitive data from execution logs:
+
+```ruby
+# config/initializers/ruby_llm_agents.rb
+RubyLLM::Agents.configure do |config|
+  config.redaction = {
+    # Fields to redact (applied to parameters)
+    # Default: password, token, api_key, secret, credential, auth, key, access_token
+    fields: %w[ssn credit_card phone_number],
+
+    # Regex patterns to redact from prompts/responses
+    patterns: [
+      /\b\d{3}-\d{2}-\d{4}\b/,  # SSN
+      /\b\d{16}\b/,              # Credit card
+      /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i  # Email
+    ],
+
+    # Replacement text
+    placeholder: "[REDACTED]",
+
+    # Truncate long values
+    max_value_length: 1000
+  }
+
+  # Control what gets persisted
+  config.persist_prompts = true    # Store system/user prompts
+  config.persist_responses = true  # Store LLM responses
+end
+```
+
 ## Configuration
 
 Edit `config/initializers/ruby_llm_agents.rb`:
@@ -689,8 +956,14 @@ rails generate ruby_llm_agents:install
 ```bash
 # Upgrade to latest schema (when gem is updated)
 rails generate ruby_llm_agents:upgrade
+rails db:migrate
 ```
 
+This creates migrations for new features like:
+- `system_prompt` and `user_prompt` columns for prompt persistence
+- `attempts` JSONB column for reliability tracking
+- `chosen_model_id` for fallback model tracking
+
 ## Background Jobs
 
 For production environments, enable async logging:

data/app/channels/ruby_llm/agents/executions_channel.rb

@@ -7,14 +7,37 @@ module RubyLLM
     # Broadcasts execution create/update events to subscribed clients.
     # Used by the dashboard to show live execution status changes.
     #
-    # Inherits from the host app's ApplicationCable::Channel (note the :: prefix)
+    # Inherits from the host app's ApplicationCable::Channel (note the :: prefix
+    # to reference the root namespace, not the engine's namespace).
     #
+    # @example JavaScript subscription
+    #   import { createConsumer } from "@rails/actioncable"
+    #   const consumer = createConsumer()
+    #   consumer.subscriptions.create("RubyLLM::Agents::ExecutionsChannel", {
+    #     received(data) {
+    #       console.log("Execution update:", data)
+    #     }
+    #   })
+    #
+    # @see Execution#broadcast_execution Broadcast trigger
+    # @api private
     class ExecutionsChannel < ::ApplicationCable::Channel
+      # Subscribes the client to the executions broadcast stream
+      #
+      # Called automatically when a client connects to this channel.
+      # Streams from the "ruby_llm_agents:executions" channel name.
+      #
+      # @return [void]
       def subscribed
         stream_from "ruby_llm_agents:executions"
         logger.info "[RubyLLM::Agents] Client subscribed to executions channel"
       end
 
+      # Cleans up when a client disconnects
+      #
+      # Called automatically when the WebSocket connection is closed.
+      #
+      # @return [void]
       def unsubscribed
         logger.info "[RubyLLM::Agents] Client unsubscribed from executions channel"
       end

data/app/controllers/concerns/ruby_llm/agents/filterable.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    # Controller concern for parsing and applying filters
+    #
+    # Provides methods for parsing filter parameters from requests
+    # and applying them to ActiveRecord scopes.
+    #
+    # @example Including in a controller
+    #   class ExecutionsController < ApplicationController
+    #     include Filterable
+    #
+    #     def index
+    #       statuses = parse_array_param(:statuses)
+    #       @scope = apply_status_filter(Execution.all, statuses)
+    #     end
+    #   end
+    #
+    # @api private
+    module Filterable
+      extend ActiveSupport::Concern
+
+      # Valid status values for filtering
+      VALID_STATUSES = %w[running success error timeout].freeze
+
+      private
+
+      # Parses an array parameter from the request
+      #
+      # Handles both array format (?key[]=a&key[]=b) and
+      # comma-separated format (?key=a,b).
+      #
+      # @param key [Symbol] The parameter key
+      # @return [Array<String>] Parsed values (empty if blank)
+      def parse_array_param(key)
+        value = params[key]
+        return [] if value.blank?
+
+        (value.is_a?(Array) ? value : value.to_s.split(",")).select(&:present?)
+      end
+
+      # Parses the days parameter for time filtering
+      #
+      # @return [Integer, nil] Number of days or nil if invalid/missing
+      def parse_days_param
+        return nil unless params[:days].present?
+
+        days = params[:days].to_i
+        days.positive? ? days : nil
+      end
+
+      # Filters status values to only valid ones
+      #
+      # @param statuses [Array<String>] Status values to validate
+      # @return [Array<String>] Valid status values only
+      def validate_statuses(statuses)
+        statuses.select { |s| VALID_STATUSES.include?(s) }
+      end
+
+      # Applies status filter to a scope
+      #
+      # @param scope [ActiveRecord::Relation] The base scope
+      # @param statuses [Array<String>] Status values to filter by
+      # @return [ActiveRecord::Relation] Filtered scope
+      def apply_status_filter(scope, statuses)
+        valid_statuses = validate_statuses(statuses)
+        valid_statuses.any? ? scope.where(status: valid_statuses) : scope
+      end
+
+      # Applies time filter to a scope
+      #
+      # @param scope [ActiveRecord::Relation] The base scope
+      # @param days [Integer, nil] Number of days to filter by
+      # @return [ActiveRecord::Relation] Filtered scope
+      def apply_time_filter(scope, days)
+        days.present? && days.positive? ? scope.where("created_at >= ?", days.days.ago) : scope
+      end
+    end
+  end
+end

data/app/controllers/concerns/ruby_llm/agents/paginatable.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    # Controller concern for pagination
+    #
+    # Provides simple offset-based pagination with consistent return format.
+    #
+    # @example Using in a controller
+    #   result = paginate(Execution.all)
+    #   @executions = result[:records]
+    #   @pagination = result[:pagination]
+    #
+    # @api private
+    module Paginatable
+      extend ActiveSupport::Concern
+
+      private
+
+      # Paginates a scope with optional ordering
+      #
+      # @param scope [ActiveRecord::Relation] The scope to paginate
+      # @param ordered [Boolean] Whether to apply default descending order (default: true)
+      # @return [Hash] Contains :records and :pagination keys
+      # @option return [ActiveRecord::Relation] :records Paginated records
+      # @option return [Hash] :pagination Pagination metadata
+      #   - :current_page [Integer] Current page number
+      #   - :per_page [Integer] Records per page
+      #   - :total_count [Integer] Total record count
+      #   - :total_pages [Integer] Total page count
+      def paginate(scope, ordered: true)
+        page = [(params[:page] || 1).to_i, 1].max
+        per_page = RubyLLM::Agents.configuration.per_page
+        offset = (page - 1) * per_page
+
+        scope = scope.order(created_at: :desc) if ordered
+        total_count = scope.count
+
+        {
+          records: scope.offset(offset).limit(per_page),
+          pagination: {
+            current_page: page,
+            per_page: per_page,
+            total_count: total_count,
+            total_pages: (total_count.to_f / per_page).ceil
+          }
+        }
+      end
+    end
+  end
+end