ruby_llm-agents 0.1.0

data/lib/ruby_llm/agents/configuration.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    class Configuration
+      attr_accessor :default_model,
+                    :default_temperature,
+                    :default_timeout,
+                    :async_logging,
+                    :retention_period,
+                    :anomaly_cost_threshold,
+                    :anomaly_duration_threshold,
+                    :dashboard_auth,
+                    :dashboard_parent_controller
+
+      attr_writer :cache_store
+
+      def initialize
+        @default_model = "gemini-2.0-flash"
+        @default_temperature = 0.0
+        @default_timeout = 60
+        @cache_store = nil
+        @async_logging = true
+        @retention_period = 30.days
+        @anomaly_cost_threshold = 5.00
+        @anomaly_duration_threshold = 10_000
+        @dashboard_auth = ->(_controller) { true }
+        @dashboard_parent_controller = "ActionController::Base"
+      end
+
+      def cache_store
+        @cache_store || Rails.cache
+      end
+    end
+  end
+end

data/lib/ruby_llm/agents/engine.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    class Engine < ::Rails::Engine
+      isolate_namespace RubyLLM::Agents
+
+      # Use to_prepare to load classes after autoloading is set up
+      # This ensures app/models are available when referenced
+      config.to_prepare do
+        require_relative "execution_logger_job"
+        require_relative "instrumentation"
+        require_relative "base"
+      end
+
+      # Configure generators
+      config.generators do |g|
+        g.test_framework :rspec
+        g.fixture_replacement :factory_bot
+        g.factory_bot dir: "spec/factories"
+      end
+
+      # Add app/agents to autoload paths for host app (must be done before initialization)
+      initializer "ruby_llm_agents.autoload_agents", before: :set_autoload_paths do |app|
+        agents_path = app.root.join("app/agents")
+        if agents_path.exist?
+          Rails.autoloaders.main.push_dir(agents_path.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/agents/execution_logger_job.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    # Background job for logging agent executions to the database
+    #
+    # This job is called automatically after each agent execution to:
+    # - Create an Execution record with all execution data
+    # - Calculate costs based on token usage
+    # - Log anomalies (expensive, slow, or failed executions)
+    #
+    # Configuration:
+    #   RubyLLM::Agents.configure do |config|
+    #     config.anomaly_cost_threshold = 5.00       # Log if cost > $5
+    #     config.anomaly_duration_threshold = 10_000 # Log if duration > 10s
+    #   end
+    #
+    class ExecutionLoggerJob < ActiveJob::Base
+      queue_as :default
+
+      # Retry with polynomial backoff
+      retry_on StandardError, wait: :polynomially_longer, attempts: 3
+
+      def perform(execution_data)
+        execution = Execution.create!(execution_data)
+
+        # Calculate costs if token data is available
+        if execution.input_tokens && execution.output_tokens
+          execution.calculate_costs!
+          execution.save!
+        end
+
+        # Log if execution was anomalous
+        log_anomaly(execution) if anomaly?(execution)
+      end
+
+      private
+
+      def anomaly?(execution)
+        config = RubyLLM::Agents.configuration
+
+        (execution.total_cost && execution.total_cost > config.anomaly_cost_threshold) ||
+          (execution.duration_ms && execution.duration_ms > config.anomaly_duration_threshold) ||
+          execution.status_error?
+      end
+
+      def log_anomaly(execution)
+        Rails.logger.warn(
+          "[RubyLLM::Agents] Execution anomaly detected: " \
+          "agent=#{execution.agent_type} " \
+          "id=#{execution.id} " \
+          "cost=$#{execution.total_cost} " \
+          "duration_ms=#{execution.duration_ms} " \
+          "status=#{execution.status}"
+        )
+      end
+    end
+  end
+end

data/lib/ruby_llm/agents/inflections.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# Configure acronym for Rails inflector (used in routes)
+ActiveSupport::Inflector.inflections(:en) do |inflect|
+  inflect.acronym "LLM"
+end
+
+# Configure Zeitwerk inflection when autoloaders are available
+ActiveSupport.on_load(:before_configuration) do
+  Rails.autoloaders.each do |autoloader|
+    autoloader.inflector.inflect("ruby_llm" => "RubyLLM")
+  end
+end

data/lib/ruby_llm/agents/instrumentation.rb

@@ -0,0 +1,245 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    # Instrumentation concern for tracking agent executions
+    #
+    # Provides execution timing, token tracking, cost calculation, and error handling.
+    # Logs all executions to the database via a background job.
+    #
+    # == Usage
+    #
+    # Included automatically in RubyLLM::Agents::Base
+    #
+    # == Customization
+    #
+    # Override `execution_metadata` in your agent to add custom data:
+    #
+    #   def execution_metadata
+    #     { query: query, user_id: Current.user&.id }
+    #   end
+    #
+    module Instrumentation
+      extend ActiveSupport::Concern
+
+      included do
+        attr_accessor :execution_id
+      end
+
+      # Wrap agent execution with metrics tracking
+      # Creates execution record at start with 'running' status, updates on completion
+      def instrument_execution(&block)
+        started_at = Time.current
+        @last_response = nil
+
+        # Create execution record immediately with running status
+        execution = create_running_execution(started_at)
+        self.execution_id = execution&.id
+
+        begin
+          result = yield
+
+          # Update to success
+          complete_execution(
+            execution,
+            completed_at: Time.current,
+            status: "success",
+            response: @last_response
+          )
+
+          result
+        rescue Timeout::Error => e
+          complete_execution(
+            execution,
+            completed_at: Time.current,
+            status: "timeout",
+            error: e
+          )
+          raise
+        rescue => e
+          complete_execution(
+            execution,
+            completed_at: Time.current,
+            status: "error",
+            error: e
+          )
+          raise
+        end
+      end
+
+      # Store response for metrics extraction
+      def capture_response(response)
+        @last_response = response
+        response
+      end
+
+      private
+
+      # Create execution record with running status at start
+      def create_running_execution(started_at)
+        execution_data = {
+          agent_type: self.class.name,
+          agent_version: self.class.version,
+          model_id: model,
+          temperature: temperature,
+          started_at: started_at,
+          status: "running",
+          parameters: sanitized_parameters,
+          metadata: execution_metadata,
+          system_prompt: safe_system_prompt,
+          user_prompt: safe_user_prompt
+        }
+
+        RubyLLM::Agents::Execution.create!(execution_data)
+      rescue StandardError => e
+        # Log error but don't fail the execution
+        Rails.logger.error("[RubyLLM::Agents] Failed to create execution record: #{e.message}")
+        nil
+      end
+
+      # Update execution record on completion
+      def complete_execution(execution, completed_at:, status:, response: nil, error: nil)
+        return legacy_log_execution(completed_at: completed_at, status: status, response: response, error: error) unless execution
+
+        started_at = execution.started_at
+        duration_ms = ((completed_at - started_at) * 1000).round
+
+        update_data = {
+          completed_at: completed_at,
+          duration_ms: duration_ms,
+          status: status
+        }
+
+        # Add response data if available
+        if response.is_a?(RubyLLM::Message)
+          update_data.merge!(
+            input_tokens: response.input_tokens,
+            output_tokens: response.output_tokens,
+            cached_tokens: response.cached_tokens || 0,
+            cache_creation_tokens: response.cache_creation_tokens || 0,
+            model_id: response.model_id || model,
+            response: serialize_response(response)
+          )
+        end
+
+        # Add error data if failed
+        if error
+          update_data.merge!(
+            error_message: error.message,
+            error_class: error.class.name
+          )
+        end
+
+        execution.update!(update_data)
+
+        # Calculate costs if token data is available
+        if execution.input_tokens && execution.output_tokens
+          execution.calculate_costs!
+          execution.save!
+        end
+      rescue StandardError => e
+        Rails.logger.error("[RubyLLM::Agents] Failed to update execution record: #{e.message}")
+      end
+
+      # Fallback for when initial execution creation failed
+      def legacy_log_execution(completed_at:, status:, response: nil, error: nil)
+        execution_data = {
+          agent_type: self.class.name,
+          agent_version: self.class.version,
+          model_id: model,
+          temperature: temperature,
+          started_at: Time.current,
+          completed_at: completed_at,
+          duration_ms: 0,
+          status: status,
+          parameters: sanitized_parameters,
+          metadata: execution_metadata,
+          system_prompt: safe_system_prompt,
+          user_prompt: safe_user_prompt
+        }
+
+        if response.is_a?(RubyLLM::Message)
+          execution_data.merge!(
+            input_tokens: response.input_tokens,
+            output_tokens: response.output_tokens,
+            cached_tokens: response.cached_tokens || 0,
+            cache_creation_tokens: response.cache_creation_tokens || 0,
+            model_id: response.model_id || model,
+            response: serialize_response(response)
+          )
+        end
+
+        if error
+          execution_data.merge!(
+            error_message: error.message,
+            error_class: error.class.name
+          )
+        end
+
+        if RubyLLM::Agents.configuration.async_logging
+          RubyLLM::Agents::ExecutionLoggerJob.perform_later(execution_data)
+        else
+          RubyLLM::Agents::ExecutionLoggerJob.new.perform(execution_data)
+        end
+      end
+
+      # Sanitize parameters to remove sensitive data
+      def sanitized_parameters
+        params = @options.dup
+
+        # Remove sensitive keys
+        sensitive_keys = %i[password token api_key secret credential auth key]
+        sensitive_keys.each { |key| params.delete(key) }
+
+        # Convert ActiveRecord objects to IDs
+        params.transform_values do |value|
+          case value
+          when defined?(ActiveRecord::Base) && ActiveRecord::Base
+            { id: value.id, type: value.class.name }
+          when Array
+            if value.first.is_a?(ActiveRecord::Base)
+              { ids: value.first(10).map(&:id), type: value.first.class.name, count: value.size }
+            else
+              value.first(10)
+            end
+          else
+            value
+          end
+        end
+      end
+
+      # Serialize full RubyLLM::Message response to JSON
+      def serialize_response(response)
+        {
+          content: response.content,
+          model_id: response.model_id,
+          input_tokens: response.input_tokens,
+          output_tokens: response.output_tokens,
+          cached_tokens: response.cached_tokens,
+          cache_creation_tokens: response.cache_creation_tokens
+        }.compact
+      end
+
+      # Hook for subclasses to add custom metadata
+      def execution_metadata
+        {}
+      end
+
+      # Safely capture system prompt (may raise or return nil)
+      def safe_system_prompt
+        respond_to?(:system_prompt) ? system_prompt.to_s : nil
+      rescue StandardError => e
+        Rails.logger.warn("[RubyLLM::Agents] Could not capture system_prompt: #{e.message}")
+        nil
+      end
+
+      # Safely capture user prompt (may raise or return nil)
+      def safe_user_prompt
+        respond_to?(:user_prompt) ? user_prompt.to_s : nil
+      rescue StandardError => e
+        Rails.logger.warn("[RubyLLM::Agents] Could not capture user_prompt: #{e.message}")
+        nil
+      end
+    end
+  end
+end

data/lib/ruby_llm/agents/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    VERSION = "0.1.0"
+  end
+end

data/lib/ruby_llm/agents.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative "agents/version"
+require_relative "agents/configuration"
+require_relative "agents/inflections" if defined?(Rails)
+require_relative "agents/engine" if defined?(Rails::Engine)
+
+module RubyLLM
+  module Agents
+    class Error < StandardError; end
+
+    class << self
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      def configure
+        yield(configuration)
+      end
+
+      def reset_configuration!
+        @configuration = Configuration.new
+      end
+    end
+  end
+end

data/lib/ruby_llm-agents.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "ruby_llm/agents"

metadata

@@ -0,0 +1,164 @@
+--- !ruby/object:Gem::Specification
+name: ruby_llm-agents
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- adham90
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: ruby_llm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: turbo-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: stimulus-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: chartkick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: A Rails engine for creating, managing, and monitoring LLM-powered agents.
+  Includes a DSL for agent configuration, execution tracking, cost analytics, and
+  a mountable dashboard UI.
+email:
+- hi@adham.dev
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- app/channels/ruby_llm/agents/executions_channel.rb
+- app/controllers/ruby_llm/agents/agents_controller.rb
+- app/controllers/ruby_llm/agents/application_controller.rb
+- app/controllers/ruby_llm/agents/dashboard_controller.rb
+- app/controllers/ruby_llm/agents/executions_controller.rb
+- app/helpers/ruby_llm/agents/application_helper.rb
+- app/javascript/ruby_llm/agents/controllers/filter_controller.js
+- app/javascript/ruby_llm/agents/controllers/index.js
+- app/javascript/ruby_llm/agents/controllers/refresh_controller.js
+- app/models/ruby_llm/agents/execution.rb
+- app/models/ruby_llm/agents/execution/analytics.rb
+- app/models/ruby_llm/agents/execution/metrics.rb
+- app/models/ruby_llm/agents/execution/scopes.rb
+- app/services/ruby_llm/agents/agent_registry.rb
+- app/views/layouts/rubyllm/agents/application.html.erb
+- app/views/rubyllm/agents/agents/index.html.erb
+- app/views/rubyllm/agents/agents/show.html.erb
+- app/views/rubyllm/agents/dashboard/_execution_item.html.erb
+- app/views/rubyllm/agents/dashboard/index.html.erb
+- app/views/rubyllm/agents/executions/_execution.html.erb
+- app/views/rubyllm/agents/executions/_filters.html.erb
+- app/views/rubyllm/agents/executions/_list.html.erb
+- app/views/rubyllm/agents/executions/index.html.erb
+- app/views/rubyllm/agents/executions/index.turbo_stream.erb
+- app/views/rubyllm/agents/executions/show.html.erb
+- app/views/rubyllm/agents/shared/_executions_table.html.erb
+- app/views/rubyllm/agents/shared/_stat_card.html.erb
+- app/views/rubyllm/agents/shared/_status_badge.html.erb
+- app/views/rubyllm/agents/shared/_status_dot.html.erb
+- config/routes.rb
+- lib/generators/ruby_llm_agents/agent_generator.rb
+- lib/generators/ruby_llm_agents/install_generator.rb
+- lib/generators/ruby_llm_agents/templates/add_prompts_migration.rb.tt
+- lib/generators/ruby_llm_agents/templates/agent.rb.tt
+- lib/generators/ruby_llm_agents/templates/application_agent.rb.tt
+- lib/generators/ruby_llm_agents/templates/initializer.rb.tt
+- lib/generators/ruby_llm_agents/templates/migration.rb.tt
+- lib/generators/ruby_llm_agents/upgrade_generator.rb
+- lib/ruby_llm-agents.rb
+- lib/ruby_llm/agents.rb
+- lib/ruby_llm/agents/base.rb
+- lib/ruby_llm/agents/configuration.rb
+- lib/ruby_llm/agents/engine.rb
+- lib/ruby_llm/agents/execution_logger_job.rb
+- lib/ruby_llm/agents/inflections.rb
+- lib/ruby_llm/agents/instrumentation.rb
+- lib/ruby_llm/agents/version.rb
+homepage: https://github.com/adham90/ruby_llm-agents
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/adham90/ruby_llm-agents
+  source_code_uri: https://github.com/adham90/ruby_llm-agents
+  changelog_uri: https://github.com/adham90/ruby_llm-agents/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Agent framework for building LLM-powered agents with RubyLLM
+test_files: []