ruby_llm-agents 3.5.5 → 3.7.0

data/lib/ruby_llm/agents/dsl/queryable.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    module DSL
+      # Adds execution querying capabilities to agent classes.
+      #
+      # Mixed into BaseAgent via `extend DSL::Queryable`, making all methods
+      # available as class methods on agent classes.
+      #
+      # @example Basic queries
+      #   SupportAgent.executions.successful.recent
+      #   SupportAgent.executions.today.expensive(0.50)
+      #
+      # @example Convenience methods
+      #   SupportAgent.last_run
+      #   SupportAgent.stats
+      #   SupportAgent.total_spent(since: 1.week)
+      #
+      module Queryable
+        # Returns an ActiveRecord::Relation scoped to this agent's executions.
+        #
+        # @return [ActiveRecord::Relation]
+        #
+        # @example
+        #   SupportAgent.executions.successful.last(5)
+        #   SupportAgent.executions.where("total_cost > ?", 0.01)
+        #
+        def executions
+          RubyLLM::Agents::Execution.by_agent(name)
+        end
+
+        # Returns the most recent execution for this agent.
+        #
+        # @return [RubyLLM::Agents::Execution, nil]
+        #
+        def last_run
+          executions.order(created_at: :desc).first
+        end
+
+        # Returns recent failed executions.
+        #
+        # @param since [ActiveSupport::Duration] Time window (default: 24.hours)
+        # @return [ActiveRecord::Relation]
+        #
+        def failures(since: 24.hours)
+          executions.failed.where("created_at > ?", since.ago)
+        end
+
+        # Returns total cost spent by this agent.
+        #
+        # @param since [ActiveSupport::Duration, nil] Optional time window
+        # @return [BigDecimal] Total cost in USD
+        #
+        def total_spent(since: nil)
+          scope = executions
+          scope = scope.where("created_at > ?", since.ago) if since
+          scope.sum(:total_cost)
+        end
+
+        # Returns a stats summary hash for this agent.
+        #
+        # @param since [ActiveSupport::Duration, nil] Time window
+        # @return [Hash] Stats summary
+        #
+        def stats(since: nil)
+          scope = executions
+          scope = scope.where("created_at > ?", since.ago) if since
+
+          total = scope.count
+          successful = scope.successful.count
+
+          {
+            total: total,
+            successful: successful,
+            failed: scope.failed.count,
+            success_rate: total.zero? ? 0.0 : (successful.to_f / total * 100).round(1),
+            avg_duration_ms: scope.average(:duration_ms)&.round,
+            avg_cost: total.zero? ? 0 : (scope.sum(:total_cost).to_f / total).round(6),
+            total_cost: scope.sum(:total_cost),
+            total_tokens: scope.sum(:total_tokens),
+            avg_tokens: scope.average(:total_tokens)&.round
+          }
+        end
+
+        # Returns cost breakdown by model for this agent.
+        #
+        # @param since [ActiveSupport::Duration, nil] Time window
+        # @return [Hash{String => Hash}] Costs per model
+        #
+        def cost_by_model(since: nil)
+          scope = executions
+          scope = scope.where("created_at > ?", since.ago) if since
+
+          scope.group(:model_id).pluck(
+            :model_id,
+            Arel.sql("COUNT(*)"),
+            Arel.sql("SUM(total_cost)"),
+            Arel.sql("AVG(total_cost)")
+          ).each_with_object({}) do |(model, count, total, avg), hash|
+            hash[model] = {
+              count: count,
+              total_cost: total&.to_f&.round(6) || 0,
+              avg_cost: avg&.to_f&.round(6) || 0
+            }
+          end
+        end
+
+        # Returns executions matching specific parameter values.
+        #
+        # @param params [Hash] Parameter key-value pairs to match
+        # @return [ActiveRecord::Relation]
+        #
+        def with_params(**params)
+          scope = executions
+          params.each do |key, value|
+            scope = scope.with_parameter(key, value)
+          end
+          scope
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/agents/dsl.rb

@@ -3,6 +3,7 @@
 require_relative "dsl/base"
 require_relative "dsl/reliability"
 require_relative "dsl/caching"
+require_relative "dsl/queryable"
 
 module RubyLLM
   module Agents

data/lib/ruby_llm/agents/eval/eval_result.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    module Eval
+      # Holds the result of evaluating a single test case.
+      #
+      # Contains the test case definition, the agent's result, the score,
+      # and any error that occurred during execution.
+      class EvalResult
+        attr_reader :test_case, :agent_result, :score, :execution_id, :error
+
+        def initialize(test_case:, agent_result:, score:, execution_id: nil, error: nil)
+          @test_case = test_case
+          @agent_result = agent_result
+          @score = score
+          @execution_id = execution_id
+          @error = error
+        end
+
+        def test_case_name
+          test_case.name
+        end
+
+        def input
+          test_case.input
+        end
+
+        def expected
+          test_case.expected
+        end
+
+        def passed?(threshold = 0.5)
+          score.passed?(threshold)
+        end
+
+        def failed?(threshold = 0.5)
+          score.failed?(threshold)
+        end
+
+        def errored?
+          !error.nil?
+        end
+
+        def actual
+          return nil unless agent_result
+
+          if agent_result.respond_to?(:route)
+            {route: agent_result.route}
+          elsif agent_result.respond_to?(:content)
+            agent_result.content
+          else
+            agent_result
+          end
+        end
+
+        def to_h
+          {
+            name: test_case_name,
+            score: score.value,
+            reason: score.reason,
+            passed: passed?,
+            input: input,
+            expected: expected,
+            actual: actual,
+            execution_id: execution_id,
+            error: error&.message
+          }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/agents/eval/eval_run.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    module Eval
+      # Aggregate results from running an eval suite.
+      #
+      # Provides score calculation, pass/fail counts, failure details,
+      # and a formatted summary string.
+      class EvalRun
+        attr_reader :suite, :results, :model, :pass_threshold,
+          :started_at, :completed_at
+
+        def initialize(suite:, results:, model:, pass_threshold:, started_at:, completed_at:)
+          @suite = suite
+          @results = results
+          @model = model
+          @pass_threshold = pass_threshold
+          @started_at = started_at
+          @completed_at = completed_at
+        end
+
+        def agent_class
+          suite.respond_to?(:agent_class) ? suite.agent_class : suite
+        end
+
+        # Average score across all test cases (0.0 to 1.0)
+        def score
+          return 0.0 if results.empty?
+
+          results.sum { |r| r.score.value } / results.size.to_f
+        end
+
+        def score_pct
+          (score * 100).round(1)
+        end
+
+        def total_cases
+          results.size
+        end
+
+        def passed
+          results.count { |r| r.passed?(pass_threshold) }
+        end
+
+        def failed
+          results.count { |r| r.failed?(pass_threshold) }
+        end
+
+        def failures
+          results.select { |r| r.failed?(pass_threshold) }
+        end
+
+        def errors
+          results.select(&:errored?)
+        end
+
+        def total_cost
+          results.sum do |r|
+            next 0 unless r.execution_id
+
+            if defined?(Execution)
+              Execution.find_by(id: r.execution_id)&.total_cost || 0
+            else
+              0
+            end
+          end
+        rescue
+          0
+        end
+
+        def duration_ms
+          return 0 unless started_at && completed_at
+
+          ((completed_at - started_at) * 1000).to_i
+        end
+
+        def summary
+          agent_name = agent_class.respond_to?(:name) ? agent_class.name : agent_class.to_s
+          lines = ["#{agent_name} Eval — #{started_at.strftime("%Y-%m-%d %H:%M")}"]
+          lines << "Model: #{model} | Score: #{score_pct}% | #{passed}/#{total_cases} passed"
+          lines << "Cost: $#{"%.4f" % total_cost} | Duration: #{(duration_ms / 1000.0).round(1)}s"
+
+          if failures.any?
+            lines << ""
+            lines << "Failures:"
+            failures.each do |r|
+              lines << "  - #{r.test_case_name}: expected #{r.expected.inspect}, got #{r.actual.inspect} (#{r.score.reason})"
+            end
+          end
+
+          if errors.any?
+            lines << ""
+            lines << "Errors:"
+            errors.each do |r|
+              lines << "  - #{r.test_case_name}: #{r.error.message}"
+            end
+          end
+
+          lines.join("\n")
+        end
+
+        def to_h
+          {
+            agent: agent_class.respond_to?(:name) ? agent_class.name : agent_class.to_s,
+            model: model,
+            score: score,
+            score_pct: score_pct,
+            total_cases: total_cases,
+            passed: passed,
+            failed: failed,
+            total_cost: total_cost,
+            duration_ms: duration_ms,
+            results: results.map(&:to_h)
+          }
+        end
+
+        def to_json(*args)
+          to_h.to_json(*args)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/agents/eval/eval_suite.rb

@@ -0,0 +1,264 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Agents
+    module Eval
+      # Score value object — returned by every scorer
+      Score = Struct.new(:value, :reason, keyword_init: true) do
+        def initialize(value:, reason: nil)
+          super(value: value.to_f.clamp(0.0, 1.0), reason: reason)
+        end
+
+        def passed?(threshold = 0.5)
+          value >= threshold
+        end
+
+        def failed?(threshold = 0.5)
+          !passed?(threshold)
+        end
+      end
+
+      # A single test case definition
+      TestCase = Struct.new(:name, :input, :expected, :scorer, :options, keyword_init: true) do
+        def resolve_input
+          input.is_a?(Proc) ? input.call : input
+        end
+      end
+
+      # Defines test cases for an agent, runs them, scores results.
+      #
+      # @example
+      #   class SupportRouter::Eval < RubyLLM::Agents::EvalSuite
+      #     agent SupportRouter
+      #     test_case "billing", input: { message: "charged twice" }, expected: { route: :billing }
+      #   end
+      #
+      #   run = SupportRouter::Eval.run!
+      #   puts run.summary
+      class EvalSuite
+        class << self
+          attr_reader :agent_class, :test_cases, :eval_options
+
+          def inherited(subclass)
+            super
+            subclass.instance_variable_set(:@test_cases, [])
+            subclass.instance_variable_set(:@eval_options, {})
+          end
+
+          # --- DSL ---
+
+          def agent(klass)
+            @agent_class = klass
+          end
+
+          def test_case(name, input:, expected: nil, score: nil, **options)
+            @test_cases << TestCase.new(
+              name: name,
+              input: input,
+              expected: expected,
+              scorer: score,
+              options: options
+            )
+          end
+
+          def dataset(path)
+            full_path = path.start_with?("/") ? path : Rails.root.join(path).to_s
+            cases = YAML.safe_load_file(full_path, permitted_classes: [Symbol], symbolize_names: true)
+            cases.each do |tc|
+              test_case(
+                tc[:name],
+                input: tc[:input],
+                expected: tc[:expected],
+                score: tc[:score]&.to_sym,
+                **tc.except(:name, :input, :expected, :score)
+              )
+            end
+          end
+
+          def eval_model(value)
+            @eval_options[:model] = value
+          end
+
+          def eval_temperature(value)
+            @eval_options[:temperature] = value
+          end
+
+          # --- Running ---
+
+          def run!(model: nil, only: nil, pass_threshold: 0.5, overrides: {}, **options)
+            validate!
+            cases = only ? @test_cases.select { |tc| Array(only).include?(tc.name) } : @test_cases
+            resolved_model = model || @eval_options[:model]
+            temperature = @eval_options[:temperature]
+            started_at = Time.current
+
+            results = cases.map do |tc|
+              run_single(tc, model: resolved_model, temperature: temperature, overrides: overrides)
+            end
+
+            EvalRun.new(
+              suite: self,
+              results: results,
+              model: resolved_model || (agent_class.respond_to?(:model) ? agent_class.model : nil),
+              pass_threshold: pass_threshold,
+              started_at: started_at,
+              completed_at: Time.current
+            )
+          end
+
+          def validate!
+            raise ConfigurationError, "No agent class set" unless @agent_class
+            raise ConfigurationError, "No test cases defined" if @test_cases.empty?
+
+            @test_cases.each do |tc|
+              next if tc.input.is_a?(Proc)
+              next unless @agent_class.respond_to?(:params)
+
+              agent_params = @agent_class.params
+              required = agent_params.select { |_, v| v[:required] }.keys
+              missing = required - tc.input.keys
+              if missing.any?
+                raise ConfigurationError,
+                  "Test case '#{tc.name}' missing required params: #{missing.join(", ")}"
+              end
+            end
+            true
+          end
+
+          def for(agent_klass, &block)
+            suite = Class.new(self)
+            suite.agent(agent_klass)
+            suite.instance_eval(&block) if block
+            suite
+          end
+
+          private
+
+          def run_single(tc, model:, temperature:, overrides:)
+            input = tc.resolve_input
+            call_options = input.dup
+            call_options.merge!(overrides) if overrides.any?
+            call_options[:model] = model if model
+            call_options[:temperature] = temperature if temperature
+
+            agent_result = agent_class.call(**call_options)
+            score = evaluate(tc, agent_result)
+
+            EvalResult.new(
+              test_case: tc,
+              agent_result: agent_result,
+              score: score,
+              execution_id: agent_result.respond_to?(:execution_id) ? agent_result.execution_id : nil
+            )
+          rescue ArgumentError
+            raise
+          rescue => e
+            EvalResult.new(
+              test_case: tc,
+              agent_result: nil,
+              score: Score.new(value: 0.0, reason: "Error: #{e.class}: #{e.message}"),
+              error: e
+            )
+          end
+
+          def evaluate(tc, agent_result)
+            case tc.scorer
+            when Proc
+              coerce_score(tc.scorer.call(agent_result, tc.expected))
+            when :contains
+              score_contains(agent_result, tc.expected)
+            when :llm_judge
+              score_llm_judge(agent_result, tc)
+            when :exact_match, nil
+              score_exact_match(agent_result, tc.expected)
+            else
+              raise ArgumentError, "Unknown scorer: #{tc.scorer}"
+            end
+          end
+
+          def coerce_score(value)
+            case value
+            when Score then value
+            when Numeric then Score.new(value: value)
+            when true then Score.new(value: 1.0)
+            when false then Score.new(value: 0.0)
+            else Score.new(value: 0.0, reason: "Scorer returned #{value.class}")
+            end
+          end
+
+          # --- Built-in scorers ---
+
+          def score_exact_match(result, expected)
+            actual = extract_comparable(result)
+            expected_val = normalize_expected(expected)
+
+            if actual == expected_val
+              Score.new(value: 1.0)
+            else
+              Score.new(value: 0.0, reason: "Expected #{expected_val.inspect}, got #{actual.inspect}")
+            end
+          end
+
+          def score_contains(result, expected)
+            content = result.respond_to?(:content) ? result.content.to_s : result.to_s
+            targets = Array(expected)
+            missing = targets.reject { |e| content.downcase.include?(e.to_s.downcase) }
+
+            if missing.empty?
+              Score.new(value: 1.0)
+            else
+              Score.new(value: 0.0, reason: "Missing: #{missing.join(", ")}")
+            end
+          end
+
+          def score_llm_judge(result, tc)
+            content = result.respond_to?(:content) ? result.content.to_s : result.to_s
+            criteria = tc.options[:criteria]
+            judge_model = tc.options[:judge_model] || "gpt-4o-mini"
+
+            prompt = <<~PROMPT
+              You are evaluating an AI agent's response. Score it from 0 to 10.
+
+              ## Input
+              #{tc.input.inspect}
+
+              ## Agent Response
+              #{content}
+
+              ## Criteria
+              #{criteria}
+
+              Respond with ONLY a JSON object:
+              {"score": <0-10>, "reason": "<brief explanation>"}
+            PROMPT
+
+            chat = RubyLLM.chat(model: judge_model)
+            parsed = JSON.parse(chat.ask(prompt).content)
+            Score.new(value: parsed["score"].to_f / 10.0, reason: parsed["reason"])
+          rescue => e
+            Score.new(value: 0.0, reason: "Judge error: #{e.message}")
+          end
+
+          def extract_comparable(result)
+            if result.respond_to?(:route)
+              {route: result.route}
+            elsif result.respond_to?(:content)
+              content = result.content
+              content.is_a?(Hash) ? content.transform_keys(&:to_sym) : content.to_s.strip
+            else
+              result
+            end
+          end
+
+          def normalize_expected(expected)
+            case expected
+            when Hash then expected.transform_keys(&:to_sym)
+            when String then expected.strip
+            else expected
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/agents/eval.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative "eval/eval_suite"
+require_relative "eval/eval_result"
+require_relative "eval/eval_run"

data/lib/ruby_llm/agents/image/concerns/image_operation_execution.rb

@@ -85,7 +85,8 @@ module RubyLLM
           if config.async_logging && defined?(ExecutionLoggerJob)
             ExecutionLoggerJob.perform_later(execution_data)
           else
-            RubyLLM::Agents::Execution.create!(execution_data)
+            execution = RubyLLM::Agents::Execution.create!(execution_data)
+            result.execution_id = execution.id if result.respond_to?(:execution_id=)
           end
         rescue => e
           Rails.logger.error("[RubyLLM::Agents] Failed to record #{execution_type} execution: #{e.message}") if defined?(Rails)