activeagent 1.3.1 → 1.5.0

data/lib/active_agent/evals/scorer.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # Scores one Replay against an evaluation's criteria and the scenario's
+    # expectations. Returns `criterion key => 0.0..1.0`, with nil for a
+    # criterion that could not be scored (an llm_judge criterion with no judge).
+    #
+    # Criteria are `{ "key", "type", "config" }` hashes:
+    #
+    #   response_present  — the answer is non-empty
+    #   min_length        — `config.chars` characters (partial credit below)
+    #   max_latency_ms    — `config.ms` budget (partial credit above)
+    #   token_budget      — `config.output_tokens` budget (partial credit above)
+    #   contains          — `config.pattern` (a substring, or a regex) is present
+    #   not_contains      — `config.pattern` is absent
+    #   llm_judge         — the judge scores the answer against `config.prompt`
+    #
+    # The scenario's own expectations add `expected_tools`, `expected_content`,
+    # `forbidden_content` (when declared) and `tools_succeeded` (when any tool
+    # was called).
+    class Scorer
+      RULE_CRITERION_TYPES = %w[response_present min_length max_latency_ms token_budget contains not_contains].freeze
+      CRITERION_TYPES = (RULE_CRITERION_TYPES + %w[llm_judge]).freeze
+
+      attr_reader :criteria, :judge
+
+      def initialize(criteria: [], judge: nil)
+        @criteria = Array(criteria).map { |criterion| criterion.to_h.deep_stringify_keys }
+        @judge = judge
+      end
+
+      def score(scenario, replay)
+        scores = {}
+        answer = replay.answer.to_s
+
+        @criteria.each do |criterion|
+          scores[criterion["key"]] = answer.present? ? score_criterion(criterion, scenario, replay) : 0.0
+        end
+
+        if scenario.expected_tools.any?
+          scores["expected_tools"] = (scenario.expected_tools & replay.tool_names).any? ? 1.0 : 0.0
+        end
+        if scenario.expected_patterns.any?
+          hits = scenario.expected_patterns.count { |pattern| self.class.matches_pattern?(answer, pattern) }
+          scores["expected_content"] = (hits.to_f / scenario.expected_patterns.size).round(3)
+        end
+        if scenario.forbidden_patterns.any?
+          hit = scenario.forbidden_patterns.any? { |pattern| self.class.matches_pattern?(answer, pattern) }
+          scores["forbidden_content"] = hit ? 0.0 : 1.0
+        end
+        if replay.tool_calls.any?
+          scores["tools_succeeded"] = replay.failed_tool_calls.any? ? 0.0 : 1.0
+        end
+
+        scores
+      end
+
+      # The mean of the scored criteria, or nil when nothing could be scored.
+      def self.mean(scores)
+        scored = scores.values.compact
+        return nil if scored.empty?
+
+        (scored.sum / scored.size).round(3)
+      end
+
+      # How long one pattern may take to match one answer. Patterns are
+      # whatever the scenario's author typed, and a pathological one must not
+      # stall the evaluation.
+      PATTERN_TIMEOUT = 1.0
+
+      # Whether `pattern` occurs in `text`: as a plain substring, case
+      # insensitively, or else as a regex. A pattern that is not a valid
+      # regex, or that takes longer than PATTERN_TIMEOUT, only counts as a
+      # substring.
+      def self.matches_pattern?(text, pattern)
+        pattern = pattern.to_s
+        return false if pattern.blank?
+
+        text = text.to_s
+        return true if text.downcase.include?(pattern.downcase)
+
+        text.match?(Regexp.new(pattern, Regexp::IGNORECASE, timeout: PATTERN_TIMEOUT))
+      rescue RegexpError
+        false
+      end
+
+      private
+
+      def score_criterion(criterion, scenario, replay)
+        config = criterion["config"] || {}
+        answer = replay.answer.to_s
+
+        case criterion["type"]
+        when "response_present"
+          answer.present? ? 1.0 : 0.0
+        when "min_length"
+          min = config.fetch("chars", 40).to_i
+          [ answer.length.to_f / [ min, 1 ].max, 1.0 ].min
+        when "max_latency_ms"
+          budget = config.fetch("ms", 5_000).to_f
+          duration = replay.duration_ms.to_f
+          duration.zero? || duration <= budget ? 1.0 : [ budget / duration, 1.0 ].min
+        when "token_budget"
+          budget = config.fetch("output_tokens", 1_000).to_f
+          tokens = replay.output_tokens.to_f
+          tokens <= budget ? 1.0 : [ budget / tokens, 1.0 ].min
+        when "contains"
+          self.class.matches_pattern?(answer, config["pattern"]) ? 1.0 : 0.0
+        when "not_contains"
+          self.class.matches_pattern?(answer, config["pattern"]) ? 0.0 : 1.0
+        when "llm_judge"
+          @judge&.score_criterion(criterion: criterion, prompt: scenario.prompt, answer: answer)
+        end
+      end
+    end
+  end
+end

data/lib/active_agent/evals/suite.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # An evaluation suite read from one or more YAML documents, later documents
+    # layered over earlier ones: a scenario with a key already present replaces
+    # it, a new key is appended to its group, and a new group is appended to the
+    # suite. That is how an app keeps a shared suite and lets a deployment add or
+    # reword questions.
+    #
+    #   suite: assistant_dashboard
+    #   description: The V1 question catalog
+    #   groups:
+    #     - key: find_records
+    #       name: Find record(s)
+    #       scenarios:
+    #         - key: find_records_1
+    #           prompt: Which medical search terms are under client control?
+    #           expect:
+    #             tools: [find_records, count_records]
+    #           notes: Uniform locally.
+    #           production_only: false
+    class Suite
+      class NotFound < StandardError; end
+
+      attr_reader :name, :description, :groups
+
+      # Loads the documents at `paths` (missing files are skipped) and raises
+      # NotFound when none exist.
+      def self.load(*paths, name: nil)
+        existing = paths.flatten.map(&:to_s).select { |path| File.exist?(path) }
+        raise NotFound, "no evaluation suite at #{paths.flatten.join(', ')}" if existing.empty?
+
+        documents = existing.map { |path| YAML.safe_load_file(path, aliases: true) || {} }
+        new(documents, name: name || File.basename(existing.first, ".yml"))
+      end
+
+      # @param documents [Array<Hash>] parsed YAML documents, base first
+      def initialize(documents, name: nil)
+        documents = Array(documents).map(&:deep_stringify_keys)
+        @name = documents.filter_map { |doc| doc["suite"] }.last || name
+        @description = documents.filter_map { |doc| doc["description"] }.last
+        @groups = merge_groups(documents)
+      end
+
+      # Scenarios narrowed by group keys, scenario keys, or both; `production_only`
+      # scenarios are dropped unless `include_production_only` is true.
+      def scenarios(groups: nil, keys: nil, include_production_only: true)
+        selected = all_scenarios
+        selected = selected.select { |scenario| Array(groups).map(&:to_s).include?(scenario.group) } if groups.present?
+        selected = selected.select { |scenario| Array(keys).map(&:to_s).include?(scenario.key) } if keys.present?
+        selected = selected.reject(&:production_only?) unless include_production_only
+        selected
+      end
+
+      def all_scenarios
+        @all_scenarios ||= @groups.flat_map do |group|
+          group["scenarios"].each_with_index.map do |entry, index|
+            Scenario.from_hash(entry.merge("position" => index), group: group["key"], group_name: group["name"])
+          end
+        end
+      end
+
+      def group_keys
+        @groups.map { |group| group["key"] }
+      end
+
+      def find(key)
+        all_scenarios.find { |scenario| scenario.key == key.to_s } ||
+          raise(NotFound, "no scenario #{key.inspect} in suite #{name}")
+      end
+
+      private
+
+      def merge_groups(documents)
+        documents.each_with_object([]) do |document, groups|
+          Array(document["groups"]).each do |incoming|
+            existing = groups.find { |group| group["key"] == incoming["key"] }
+
+            if existing
+              existing["name"] = incoming["name"] if incoming["name"].present?
+              existing["description"] = incoming["description"] if incoming["description"].present?
+              merge_scenarios(existing, Array(incoming["scenarios"]))
+            else
+              groups << incoming.merge("scenarios" => Array(incoming["scenarios"]))
+            end
+          end
+        end
+      end
+
+      def merge_scenarios(group, incoming)
+        incoming.each do |scenario|
+          index = group["scenarios"].index { |existing| existing["key"] == scenario["key"] }
+          index ? group["scenarios"][index] = scenario : group["scenarios"] << scenario
+        end
+      end
+    end
+  end
+end

data/lib/active_agent/evals.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "json"
+require "cgi"
+require "yaml"
+require "active_support"
+require "active_support/core_ext/object/blank"
+require "active_support/core_ext/string/filters"
+require "active_support/core_ext/string/access"
+require "active_support/core_ext/string/inflections"
+require "active_support/core_ext/hash/keys"
+require "active_support/core_ext/hash/indifferent_access"
+require "active_support/core_ext/enumerable"
+
+require_relative "evals/scenario"
+require_relative "evals/scenario_parser"
+require_relative "evals/suite"
+require_relative "evals/model_spec"
+require_relative "evals/replay"
+require_relative "evals/scorer"
+require_relative "evals/diagnosis"
+require_relative "evals/judge"
+require_relative "evals/result"
+require_relative "evals/design_tokens"
+require_relative "evals/report_html"
+require_relative "evals/report"
+require_relative "evals/runner"
+require_relative "evals/publisher"
+
+# Scenario evaluations for agents that answer with tools.
+#
+# `require "active_agent/evals"` loads this module on its own, without the
+# rest of the framework, so an app that calls models some other way (RubyLLM,
+# a plain HTTP client) can still run the same evaluations.
+#
+# The module owns everything about an evaluation except talking to the agent:
+# parsing a pasted list of tasks (ScenarioParser) or a YAML suite (Suite),
+# resolving candidate models (ModelSpec), scoring an answer against rule
+# criteria and the scenario's expectations (Scorer), naming why a scenario
+# fell short and what would fix it (Diagnosis, refined by an optional Judge),
+# and rolling everything up per model (Report). Runner ties them together
+# around one callable you supply: given a scenario and a model, run the
+# agent and return a Replay.
+#
+#   scenarios = ActiveAgent::Evals::ScenarioParser.scenarios(pasted_text)
+#   models    = ActiveAgent::Evals::ModelSpec.parse_all(%w[gpt-5-mini qwen3:8b], default_provider: "openai")
+#
+#   report = ActiveAgent::Evals::Runner.new(
+#     scenarios: scenarios,
+#     models: models,
+#     available_tools: { "find_records" => "Look up records by filters" },
+#     replay: ->(scenario, spec) { my_agent.run(scenario.prompt, model: spec.model, provider: spec.provider) }
+#   ).call
+#
+#   puts report.to_markdown
+module ActiveAgent
+  module Evals
+    # The score at or above which a scenario passes, 0.0..1.0.
+    PASS_THRESHOLD = 0.7
+  end
+end

data/lib/active_agent/providers/_base_provider.rb

@@ -59,7 +59,8 @@ module ActiveAgent
                     :tools_function,                 # Callback (Tools)
                     :usage_stack,                    # Usage Tracking
                     :stream_usage_index,             # Usage Tracking (Streams)
-                    :max_tool_turns, :tool_turns     # Tool-loop safety
+                    :max_tool_turns, :tool_turns,    # Tool-loop safety
+                    :instrumentation_enabled        # Per-generation privacy
 
       # Upper bound on tool-calling round-trips within one generation. A
       # model that keeps emitting tool calls otherwise recurses until the
@@ -117,6 +118,7 @@ module ActiveAgent
         self.tools_function     = kwargs.delete(:tools_function)
         self.max_tool_turns     = kwargs.delete(:max_tool_turns) || DEFAULT_MAX_TOOL_TURNS
         self.tool_turns         = 0
+        self.instrumentation_enabled = kwargs.delete(:instrumentation) != false
         self.options            = options_klass.new(kwargs.extract!(*options_klass.keys))
         self.context            = kwargs
         self.message_stack      = []
@@ -175,6 +177,8 @@ module ActiveAgent
       # @yield block to instrument
       # @return [Object] block result
       def instrument(name, payload = {}, &block)
+        return block&.call(payload) unless instrumentation_enabled
+
         full_payload = { provider: service_name, provider_module: tag_name, trace_id: }.merge(payload)
         ActiveSupport::Notifications.instrument(name, full_payload, &block)
       end

data/lib/active_agent/providers/mock/messages/base.rb

@@ -21,8 +21,10 @@ module ActiveAgent
               if content_type == :text
                 self.content = value
               else
-                # For image/document, MockProvider doesn't support these, so ignore
-                # (or could raise an error)
+                # No vision here: an image/document stands in as a text
+                # marker, so a media-only turn still has content to validate
+                # and to concatenate with its neighbours when serialized.
+                self.content ||= "[#{content_type}]"
               end
             end
           end

data/lib/active_agent/providers/open_ai/responses/transforms.rb

@@ -295,25 +295,43 @@ module ActiveAgent
               if message.respond_to?(:serialize)
                 message.serialize
               elsif message.is_a?(Hash)
-                # If it has a role, it's a message - convert :text to :content
+                # If it has a role, it's a message. Its :text becomes :content,
+                # and an :image / :document alongside it becomes a content
+                # part — the same `{role:, text:, image:}` shorthand the Chat
+                # API and Anthropic transforms accept, so a caller sending
+                # history plus a multimodal turn gets the same request shape
+                # from every provider.
                 if message.key?(:role)
                   normalized = message.dup
-                  if normalized.key?(:text) && !normalized.key?(:content)
-                    normalized[:content] = normalized.delete(:text)
+                  # The shorthand keys always come off the message: left on,
+                  # they reach the request body as unknown parameters and the
+                  # API rejects the whole call. A blank one contributes no
+                  # part rather than an empty input_image the API would
+                  # refuse (or a nil document, which has no URL to send).
+                  text = normalized.delete(:text)
+                  image = normalized.delete(:image)
+                  document = normalized.delete(:document)
+
+                  unless normalized.key?(:content)
+                    parts = []
+                    parts << { type: "input_text", text: text } if text.present?
+                    parts << { type: "input_image", image_url: image } if image.present?
+                    parts << document_part(document) if document.present?
+
+                    if parts.size == 1 && parts.first[:type] == "input_text"
+                      normalized[:content] = parts.first[:text]
+                    elsif parts.any?
+                      normalized[:content] = parts
+                    end
                   end
                   return normalized
                 end
 
                 # Expand shorthand formats to full structures for content items
-                if message.key?(:image)
+                if message[:image].present?
                   { type: "input_image", image_url: message[:image] }
-                elsif message.key?(:document)
-                  document_value = message[:document]
-                  if document_value.start_with?("data:")
-                    { type: "input_file", filename: "document.pdf", file_data: document_value }
-                  else
-                    { type: "input_file", file_url: document_value }
-                  end
+                elsif message[:document].present?
+                  document_part(message[:document])
                 elsif message.key?(:text) && message.size == 1
                   # Single :text key without :role - treat as user message
                   { role: "user", content: message[:text] }
@@ -336,6 +354,19 @@ module ActiveAgent
               end
             end
 
+            # An input_file part for a document given as a URL or a data URI.
+            #
+            # @param document_value [String] URL or data URI
+            # @return [Hash] input_file content part
+            def document_part(document_value)
+              document_value = document_value.to_s
+              if document_value.start_with?("data:")
+                { type: "input_file", filename: "document.pdf", file_data: document_value }
+              else
+                { type: "input_file", file_url: document_value }
+              end
+            end
+
             # Cleans up serialized request for API submission
             #
             # Removes default values and simplifies input where possible.

data/lib/active_agent/providers/ruby_llm/options.rb

@@ -11,6 +11,10 @@ module ActiveAgent
       # provider-specific API key attributes are needed here.
       class Options < Common::BaseModel
         attribute :model, :string
+        # Pins which RubyLLM backend serves the model (RubyLLM's provider:,
+        # e.g. :vertexai, :gemini, :bedrock). A model ID served by several
+        # backends otherwise resolves by RubyLLM's registry preference.
+        attribute :platform, :string
         attribute :temperature, :float
         attribute :max_tokens, :integer
 

data/lib/active_agent/providers/ruby_llm_provider.rb

@@ -12,6 +12,10 @@ module ActiveAgent
     # Provider for RubyLLM's unified API, supporting 15+ LLM providers
     # (OpenAI, Anthropic, Gemini, Bedrock, Azure, Ollama, etc.).
     #
+    # RubyLLM resolves which backend serves a request from the model ID; the
+    # platform option pins it when a model ID is served by more than one
+    # (e.g. Gemini models on the Gemini API vs Vertex AI).
+    #
     # Uses RubyLLM's provider-level API (provider.complete()) rather than
     # the high-level Chat object to avoid conflicts with ActiveAgent's own
     # conversation management and tool execution loop.
@@ -254,13 +258,22 @@ module ActiveAgent
       # Reuses the cached provider if the model hasn't changed (e.g., during
       # multi-turn tool calling loops).
       #
+      # The platform option is forwarded as RubyLLM's provider: so a model ID
+      # served by several backends (e.g. gemini-2.5-flash on the Gemini API
+      # and Vertex AI) can be pinned instead of resolving by RubyLLM's
+      # registry preference.
+      #
       # @param model_id [String] model identifier
       # @return [void]
       def resolve_ruby_llm_provider!(model_id)
         return if @ruby_llm_provider && @cached_model_id == model_id
 
         @cached_model_id = model_id
-        @ruby_llm_model, @ruby_llm_provider = ::RubyLLM::Models.resolve(model_id, config: ::RubyLLM.config)
+        @ruby_llm_model, @ruby_llm_provider = ::RubyLLM::Models.resolve(
+          model_id,
+          provider: options.platform&.to_sym,
+          config: ::RubyLLM.config
+        )
       end
 
       # Converts ActiveAgent messages to RubyLLM message format.

data/lib/active_agent/providers/rubyllm_provider.rb

@@ -0,0 +1 @@
+require_relative "ruby_llm_provider"

data/lib/active_agent/telemetry/configuration.rb

@@ -32,8 +32,19 @@ module ActiveAgent
         @local_storage = false
       end
 
+      # Under local storage traces never leave the process, and the
+      # dashboard's Interactions and Evaluations views are built from the
+      # message bodies — so bodies are captured unless the app said
+      # otherwise. An explicit capture_bodies setting wins whichever order
+      # the two options were given in.
       def local_storage=(value)
         @local_storage = value == true
+        @capture_bodies = true if @local_storage && !@capture_bodies_explicit
+      end
+
+      def capture_bodies=(value)
+        @capture_bodies_explicit = true
+        super
       end
 
       def local_storage?

data/lib/active_agent/telemetry/instrumentation.rb

@@ -39,6 +39,7 @@ module ActiveAgent
       module GenerationInstrumentation
         # Wraps process_prompt with telemetry tracing.
         def process_prompt
+          return super if respond_to?(:prompt_options) && prompt_options.is_a?(Hash) && prompt_options[:instrumentation] == false
           return super unless Telemetry.enabled?
 
           # Reuse (or mint) the generation's trace id so the telemetry trace
@@ -73,7 +74,7 @@ module ActiveAgent
               rescue StandardError
                 prompt_options[:instructions].is_a?(String) ? prompt_options[:instructions] : nil
               end
-              if rendered_instructions.present?
+              if rendered_instructions.present? && telemetry_capture_bodies?
                 prompt_span.set_attribute("prompt.input.instructions", telemetry_truncate(Array(rendered_instructions).join("\n\n")))
               end
 
@@ -101,8 +102,14 @@ module ActiveAgent
               # later, in prepare_prompt_parameters. Falling back to it means
               # the message the model actually received is on the trace either
               # way, which is what an evaluation scores.
-              outbound = prompt_options[:messages]
-              outbound = rendered_prompt_messages if outbound.blank?
+              # Bodies only when the configuration asks for them. (The
+              # messages.count above is independent of this switch, but is
+              # itself only present when explicit messages were passed — an
+              # agent rendering its user turn from a template has none at this
+              # point.) Rendering is skipped entirely when bodies are off —
+              # there would be nothing to record.
+              outbound = telemetry_capture_bodies? ? prompt_options[:messages] : nil
+              outbound = rendered_prompt_messages if outbound.blank? && telemetry_capture_bodies?
 
               if outbound.present?
                 serialized = Array(outbound).map { |message|
@@ -174,7 +181,7 @@ module ActiveAgent
 
               # Carry the generation contents so dashboards can show what
               # came back, not just how many tokens it cost.
-              if result.respond_to?(:message) && result.message.respond_to?(:content) && result.message.content.present?
+              if telemetry_capture_bodies? && result.respond_to?(:message) && result.message.respond_to?(:content) && result.message.content.present?
                 llm_span.set_attribute("llm.output.message", telemetry_truncate(result.message.content))
               end
               if result.respond_to?(:finish_reason) && result.finish_reason.present?
@@ -204,6 +211,7 @@ module ActiveAgent
         # don't expose tool calls.
         def tools_function
           base = super
+          return base if respond_to?(:prompt_options) && prompt_options.is_a?(Hash) && prompt_options[:instrumentation] == false
           return base unless Telemetry.enabled?
 
           agent = self
@@ -218,13 +226,16 @@ module ActiveAgent
             # Records which MCP server (if any) serves this tool, so tool
             # traffic can be grouped by service downstream.
             ToolOrigin.annotate(tool_span, tool_name)
+            capture_bodies = agent.send(:telemetry_capture_bodies?)
             arguments = kwargs.presence || (args.length == 1 ? args.first : args.presence)
-            if arguments.present?
+            if arguments.present? && capture_bodies
               tool_span.set_attribute("tool.input.args", agent.send(:telemetry_truncate, JSON.generate(arguments)))
             end
             begin
               result = base.call(tool_name, *args, **kwargs)
-              tool_span.set_attribute("tool.output.result", agent.send(:telemetry_truncate, result))
+              if capture_bodies
+                tool_span.set_attribute("tool.output.result", agent.send(:telemetry_truncate, result))
+              end
               tool_span.set_status(:ok)
               result
             rescue StandardError => e
@@ -238,6 +249,7 @@ module ActiveAgent
 
         # Wraps process_embed with telemetry tracing.
         def process_embed
+          return super if respond_to?(:embed_options) && embed_options.is_a?(Hash) && embed_options[:instrumentation] == false
           return super unless Telemetry.enabled?
 
           Telemetry.trace("#{self.class.name}.embed", span_type: :embedding) do |span|
@@ -269,6 +281,17 @@ module ActiveAgent
         # tool loop) can't bloat the trace payload.
         TELEMETRY_ATTRIBUTE_MAX_CHARS = 4_000
 
+        # Whether message bodies — the rendered system prompt, the outbound
+        # messages, the completion, and tool arguments and results — go on
+        # the spans at all. Off by default (the shared telemetry gem's
+        # contract, and what the docs promise), so an app reporting to a
+        # remote endpoint ships counts, names and tokens but not content
+        # unless it opted in. Configuration turns it on under local_storage,
+        # where bodies never leave the process.
+        def telemetry_capture_bodies?
+          Telemetry.configuration.capture_bodies?
+        end
+
         # The turns this generation will actually send, for an agent that
         # renders its user message from the action's template rather than
         # passing `messages:`. prepare_prompt_parameters is a pure function of

data/lib/active_agent/version.rb

@@ -1,3 +1,3 @@
 module ActiveAgent
-  VERSION = "1.3.1"
+  VERSION = "1.5.0"
 end

data/lib/active_agent.rb

@@ -108,6 +108,7 @@ module ActiveAgent
   autoload :Rescue, "active_agent/concerns/rescue"
   autoload :Tooling, "active_agent/concerns/tooling"
   autoload :View, "active_agent/concerns/view"
+  autoload :Evals
   autoload :Telemetry
 
   class << self

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: activeagent
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.5.0
 platform: ruby
 authors:
 - Justin Bowen
@@ -430,6 +430,21 @@ files:
 - lib/active_agent/delegation/runner.rb
 - lib/active_agent/delegation/schema.rb
 - lib/active_agent/deprecator.rb
+- lib/active_agent/evals.rb
+- lib/active_agent/evals/design_tokens.rb
+- lib/active_agent/evals/diagnosis.rb
+- lib/active_agent/evals/judge.rb
+- lib/active_agent/evals/model_spec.rb
+- lib/active_agent/evals/publisher.rb
+- lib/active_agent/evals/replay.rb
+- lib/active_agent/evals/report.rb
+- lib/active_agent/evals/report_html.rb
+- lib/active_agent/evals/result.rb
+- lib/active_agent/evals/runner.rb
+- lib/active_agent/evals/scenario.rb
+- lib/active_agent/evals/scenario_parser.rb
+- lib/active_agent/evals/scorer.rb
+- lib/active_agent/evals/suite.rb
 - lib/active_agent/generation.rb
 - lib/active_agent/generation_job.rb
 - lib/active_agent/inline_preview_interceptor.rb
@@ -543,6 +558,7 @@ files:
 - lib/active_agent/providers/ruby_llm/request.rb
 - lib/active_agent/providers/ruby_llm/tool_proxy.rb
 - lib/active_agent/providers/ruby_llm_provider.rb
+- lib/active_agent/providers/rubyllm_provider.rb
 - lib/active_agent/railtie.rb
 - lib/active_agent/railtie/schema_generator_extension.rb
 - lib/active_agent/schema_generator.rb
@@ -591,14 +607,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 3.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Rails AI Agents Framework
 test_files: []