activeagent 1.3.0 → 1.4.0

data/lib/active_agent/evals/suite.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # An evaluation suite read from one or more YAML documents, later documents
+    # layered over earlier ones: a scenario with a key already present replaces
+    # it, a new key is appended to its group, and a new group is appended to the
+    # suite. That is how an app keeps a shared suite and lets a deployment add or
+    # reword questions.
+    #
+    #   suite: assistant_dashboard
+    #   description: The V1 question catalog
+    #   groups:
+    #     - key: find_records
+    #       name: Find record(s)
+    #       scenarios:
+    #         - key: find_records_1
+    #           prompt: Which medical search terms are under client control?
+    #           expect:
+    #             tools: [find_records, count_records]
+    #           notes: Uniform locally.
+    #           production_only: false
+    class Suite
+      class NotFound < StandardError; end
+
+      attr_reader :name, :description, :groups
+
+      # Loads the documents at `paths` (missing files are skipped) and raises
+      # NotFound when none exist.
+      def self.load(*paths, name: nil)
+        existing = paths.flatten.map(&:to_s).select { |path| File.exist?(path) }
+        raise NotFound, "no evaluation suite at #{paths.flatten.join(', ')}" if existing.empty?
+
+        documents = existing.map { |path| YAML.safe_load_file(path, aliases: true) || {} }
+        new(documents, name: name || File.basename(existing.first, ".yml"))
+      end
+
+      # @param documents [Array<Hash>] parsed YAML documents, base first
+      def initialize(documents, name: nil)
+        documents = Array(documents).map(&:deep_stringify_keys)
+        @name = documents.filter_map { |doc| doc["suite"] }.last || name
+        @description = documents.filter_map { |doc| doc["description"] }.last
+        @groups = merge_groups(documents)
+      end
+
+      # Scenarios narrowed by group keys, scenario keys, or both; `production_only`
+      # scenarios are dropped unless `include_production_only` is true.
+      def scenarios(groups: nil, keys: nil, include_production_only: true)
+        selected = all_scenarios
+        selected = selected.select { |scenario| Array(groups).map(&:to_s).include?(scenario.group) } if groups.present?
+        selected = selected.select { |scenario| Array(keys).map(&:to_s).include?(scenario.key) } if keys.present?
+        selected = selected.reject(&:production_only?) unless include_production_only
+        selected
+      end
+
+      def all_scenarios
+        @all_scenarios ||= @groups.flat_map do |group|
+          group["scenarios"].each_with_index.map do |entry, index|
+            Scenario.from_hash(entry.merge("position" => index), group: group["key"], group_name: group["name"])
+          end
+        end
+      end
+
+      def group_keys
+        @groups.map { |group| group["key"] }
+      end
+
+      def find(key)
+        all_scenarios.find { |scenario| scenario.key == key.to_s } ||
+          raise(NotFound, "no scenario #{key.inspect} in suite #{name}")
+      end
+
+      private
+
+      def merge_groups(documents)
+        documents.each_with_object([]) do |document, groups|
+          Array(document["groups"]).each do |incoming|
+            existing = groups.find { |group| group["key"] == incoming["key"] }
+
+            if existing
+              existing["name"] = incoming["name"] if incoming["name"].present?
+              existing["description"] = incoming["description"] if incoming["description"].present?
+              merge_scenarios(existing, Array(incoming["scenarios"]))
+            else
+              groups << incoming.merge("scenarios" => Array(incoming["scenarios"]))
+            end
+          end
+        end
+      end
+
+      def merge_scenarios(group, incoming)
+        incoming.each do |scenario|
+          index = group["scenarios"].index { |existing| existing["key"] == scenario["key"] }
+          index ? group["scenarios"][index] = scenario : group["scenarios"] << scenario
+        end
+      end
+    end
+  end
+end

data/lib/active_agent/evals.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "json"
+require "cgi"
+require "yaml"
+require "active_support"
+require "active_support/core_ext/object/blank"
+require "active_support/core_ext/string/filters"
+require "active_support/core_ext/string/access"
+require "active_support/core_ext/string/inflections"
+require "active_support/core_ext/hash/keys"
+require "active_support/core_ext/hash/indifferent_access"
+require "active_support/core_ext/enumerable"
+
+require_relative "evals/scenario"
+require_relative "evals/scenario_parser"
+require_relative "evals/suite"
+require_relative "evals/model_spec"
+require_relative "evals/replay"
+require_relative "evals/scorer"
+require_relative "evals/diagnosis"
+require_relative "evals/judge"
+require_relative "evals/result"
+require_relative "evals/design_tokens"
+require_relative "evals/report_html"
+require_relative "evals/report"
+require_relative "evals/runner"
+
+# Scenario evaluations for agents that answer with tools.
+#
+# `require "active_agent/evals"` loads this module on its own, without the
+# rest of the framework, so an app that calls models some other way (RubyLLM,
+# a plain HTTP client) can still run the same evaluations.
+#
+# The module owns everything about an evaluation except talking to the agent:
+# parsing a pasted list of tasks (ScenarioParser) or a YAML suite (Suite),
+# resolving candidate models (ModelSpec), scoring an answer against rule
+# criteria and the scenario's expectations (Scorer), naming why a scenario
+# fell short and what would fix it (Diagnosis, refined by an optional Judge),
+# and rolling everything up per model (Report). Runner ties them together
+# around one callable you supply: given a scenario and a model, run the
+# agent and return a Replay.
+#
+#   scenarios = ActiveAgent::Evals::ScenarioParser.scenarios(pasted_text)
+#   models    = ActiveAgent::Evals::ModelSpec.parse_all(%w[gpt-5-mini qwen3:8b], default_provider: "openai")
+#
+#   report = ActiveAgent::Evals::Runner.new(
+#     scenarios: scenarios,
+#     models: models,
+#     available_tools: { "find_records" => "Look up records by filters" },
+#     replay: ->(scenario, spec) { my_agent.run(scenario.prompt, model: spec.model, provider: spec.provider) }
+#   ).call
+#
+#   puts report.to_markdown
+module ActiveAgent
+  module Evals
+    # The score at or above which a scenario passes, 0.0..1.0.
+    PASS_THRESHOLD = 0.7
+  end
+end

data/lib/active_agent/providers/ruby_llm/options.rb

@@ -11,6 +11,10 @@ module ActiveAgent
       # provider-specific API key attributes are needed here.
       class Options < Common::BaseModel
         attribute :model, :string
+        # Pins which RubyLLM backend serves the model (RubyLLM's provider:,
+        # e.g. :vertexai, :gemini, :bedrock). A model ID served by several
+        # backends otherwise resolves by RubyLLM's registry preference.
+        attribute :platform, :string
         attribute :temperature, :float
         attribute :max_tokens, :integer
 

data/lib/active_agent/providers/ruby_llm_provider.rb

@@ -12,6 +12,10 @@ module ActiveAgent
     # Provider for RubyLLM's unified API, supporting 15+ LLM providers
     # (OpenAI, Anthropic, Gemini, Bedrock, Azure, Ollama, etc.).
     #
+    # RubyLLM resolves which backend serves a request from the model ID; the
+    # platform option pins it when a model ID is served by more than one
+    # (e.g. Gemini models on the Gemini API vs Vertex AI).
+    #
     # Uses RubyLLM's provider-level API (provider.complete()) rather than
     # the high-level Chat object to avoid conflicts with ActiveAgent's own
     # conversation management and tool execution loop.
@@ -254,13 +258,22 @@ module ActiveAgent
       # Reuses the cached provider if the model hasn't changed (e.g., during
       # multi-turn tool calling loops).
       #
+      # The platform option is forwarded as RubyLLM's provider: so a model ID
+      # served by several backends (e.g. gemini-2.5-flash on the Gemini API
+      # and Vertex AI) can be pinned instead of resolving by RubyLLM's
+      # registry preference.
+      #
       # @param model_id [String] model identifier
       # @return [void]
       def resolve_ruby_llm_provider!(model_id)
         return if @ruby_llm_provider && @cached_model_id == model_id
 
         @cached_model_id = model_id
-        @ruby_llm_model, @ruby_llm_provider = ::RubyLLM::Models.resolve(model_id, config: ::RubyLLM.config)
+        @ruby_llm_model, @ruby_llm_provider = ::RubyLLM::Models.resolve(
+          model_id,
+          provider: options.platform&.to_sym,
+          config: ::RubyLLM.config
+        )
       end
 
       # Converts ActiveAgent messages to RubyLLM message format.

data/lib/active_agent/providers/rubyllm_provider.rb

@@ -0,0 +1 @@
+require_relative "ruby_llm_provider"

data/lib/active_agent/telemetry/configuration.rb

@@ -32,8 +32,19 @@ module ActiveAgent
         @local_storage = false
       end
 
+      # Under local storage traces never leave the process, and the
+      # dashboard's Interactions and Evaluations views are built from the
+      # message bodies — so bodies are captured unless the app said
+      # otherwise. An explicit capture_bodies setting wins whichever order
+      # the two options were given in.
       def local_storage=(value)
         @local_storage = value == true
+        @capture_bodies = true if @local_storage && !@capture_bodies_explicit
+      end
+
+      def capture_bodies=(value)
+        @capture_bodies_explicit = true
+        super
       end
 
       def local_storage?

data/lib/active_agent/telemetry/instrumentation.rb

@@ -73,7 +73,7 @@ module ActiveAgent
               rescue StandardError
                 prompt_options[:instructions].is_a?(String) ? prompt_options[:instructions] : nil
               end
-              if rendered_instructions.present?
+              if rendered_instructions.present? && telemetry_capture_bodies?
                 prompt_span.set_attribute("prompt.input.instructions", telemetry_truncate(Array(rendered_instructions).join("\n\n")))
               end
 
@@ -101,8 +101,14 @@ module ActiveAgent
               # later, in prepare_prompt_parameters. Falling back to it means
               # the message the model actually received is on the trace either
               # way, which is what an evaluation scores.
-              outbound = prompt_options[:messages]
-              outbound = rendered_prompt_messages if outbound.blank?
+              # Bodies only when the configuration asks for them. (The
+              # messages.count above is independent of this switch, but is
+              # itself only present when explicit messages were passed — an
+              # agent rendering its user turn from a template has none at this
+              # point.) Rendering is skipped entirely when bodies are off —
+              # there would be nothing to record.
+              outbound = telemetry_capture_bodies? ? prompt_options[:messages] : nil
+              outbound = rendered_prompt_messages if outbound.blank? && telemetry_capture_bodies?
 
               if outbound.present?
                 serialized = Array(outbound).map { |message|
@@ -174,7 +180,7 @@ module ActiveAgent
 
               # Carry the generation contents so dashboards can show what
               # came back, not just how many tokens it cost.
-              if result.respond_to?(:message) && result.message.respond_to?(:content) && result.message.content.present?
+              if telemetry_capture_bodies? && result.respond_to?(:message) && result.message.respond_to?(:content) && result.message.content.present?
                 llm_span.set_attribute("llm.output.message", telemetry_truncate(result.message.content))
               end
               if result.respond_to?(:finish_reason) && result.finish_reason.present?
@@ -218,13 +224,16 @@ module ActiveAgent
             # Records which MCP server (if any) serves this tool, so tool
             # traffic can be grouped by service downstream.
             ToolOrigin.annotate(tool_span, tool_name)
+            capture_bodies = agent.send(:telemetry_capture_bodies?)
             arguments = kwargs.presence || (args.length == 1 ? args.first : args.presence)
-            if arguments.present?
+            if arguments.present? && capture_bodies
               tool_span.set_attribute("tool.input.args", agent.send(:telemetry_truncate, JSON.generate(arguments)))
             end
             begin
               result = base.call(tool_name, *args, **kwargs)
-              tool_span.set_attribute("tool.output.result", agent.send(:telemetry_truncate, result))
+              if capture_bodies
+                tool_span.set_attribute("tool.output.result", agent.send(:telemetry_truncate, result))
+              end
               tool_span.set_status(:ok)
               result
             rescue StandardError => e
@@ -269,6 +278,17 @@ module ActiveAgent
         # tool loop) can't bloat the trace payload.
         TELEMETRY_ATTRIBUTE_MAX_CHARS = 4_000
 
+        # Whether message bodies — the rendered system prompt, the outbound
+        # messages, the completion, and tool arguments and results — go on
+        # the spans at all. Off by default (the shared telemetry gem's
+        # contract, and what the docs promise), so an app reporting to a
+        # remote endpoint ships counts, names and tokens but not content
+        # unless it opted in. Configuration turns it on under local_storage,
+        # where bodies never leave the process.
+        def telemetry_capture_bodies?
+          Telemetry.configuration.capture_bodies?
+        end
+
         # The turns this generation will actually send, for an agent that
         # renders its user message from the action's template rather than
         # passing `messages:`. prepare_prompt_parameters is a pure function of

data/lib/active_agent/version.rb

@@ -1,3 +1,3 @@
 module ActiveAgent
-  VERSION = "1.3.0"
+  VERSION = "1.4.0"
 end

data/lib/active_agent.rb

@@ -108,6 +108,7 @@ module ActiveAgent
   autoload :Rescue, "active_agent/concerns/rescue"
   autoload :Tooling, "active_agent/concerns/tooling"
   autoload :View, "active_agent/concerns/view"
+  autoload :Evals
   autoload :Telemetry
 
   class << self

metadata

@@ -1,13 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activeagent
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Justin Bowen
+autorequire:
 bindir: bin
 cert_chain: []
-date: 1980-01-02 00:00:00.000000000 Z
+date: 2026-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -430,6 +431,20 @@ files:
 - lib/active_agent/delegation/runner.rb
 - lib/active_agent/delegation/schema.rb
 - lib/active_agent/deprecator.rb
+- lib/active_agent/evals.rb
+- lib/active_agent/evals/design_tokens.rb
+- lib/active_agent/evals/diagnosis.rb
+- lib/active_agent/evals/judge.rb
+- lib/active_agent/evals/model_spec.rb
+- lib/active_agent/evals/replay.rb
+- lib/active_agent/evals/report.rb
+- lib/active_agent/evals/report_html.rb
+- lib/active_agent/evals/result.rb
+- lib/active_agent/evals/runner.rb
+- lib/active_agent/evals/scenario.rb
+- lib/active_agent/evals/scenario_parser.rb
+- lib/active_agent/evals/scorer.rb
+- lib/active_agent/evals/suite.rb
 - lib/active_agent/generation.rb
 - lib/active_agent/generation_job.rb
 - lib/active_agent/inline_preview_interceptor.rb
@@ -543,6 +558,7 @@ files:
 - lib/active_agent/providers/ruby_llm/request.rb
 - lib/active_agent/providers/ruby_llm/tool_proxy.rb
 - lib/active_agent/providers/ruby_llm_provider.rb
+- lib/active_agent/providers/rubyllm_provider.rb
 - lib/active_agent/railtie.rb
 - lib/active_agent/railtie/schema_generator_extension.rb
 - lib/active_agent/schema_generator.rb
@@ -584,6 +600,7 @@ metadata:
   documentation_uri: https://docs.activeagents.ai
   source_code_uri: https://github.com/activeagents/activeagent
   rubygems_mfa_required: 'true'
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -591,14 +608,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 3.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 3.5.22
+signing_key:
 specification_version: 4
 summary: Rails AI Agents Framework
 test_files: []