activeagent 1.4.0 → 1.5.2

data/lib/active_agent/evals/runner.rb

@@ -8,8 +8,10 @@ module ActiveAgent
     # The one thing the runner does not know is how to talk to your agent;
     # `replay` is a callable `(scenario, model_spec) → Replay` (a Hash with the
     # same keys is accepted, and an exception becomes an errored Replay). A
-    # scenario passes when its replay completed, met its expectations, and its
-    # mean score reached `threshold`; anything else carries exactly one fault
+    # scenario passes when its replay completed, met its expectations, and both
+    # its mean score and the mean of its judge grades (task completion, or the
+    # configured llm_judge criteria) reached `threshold`;
+    # anything else carries exactly one fault
     # and a recommendation from Diagnosis, refined by the `judge` for the
     # faults in `refine_faults` (up to `judge_limit` calls per run).
     #
@@ -42,9 +44,17 @@ module ActiveAgent
       # @param instructions [String, nil] the agent's instructions, for the judge
       # @param agent_name [String] how recommendations refer to the agent
       # @param on_result [#call, nil] called with each Result as it lands
+      # @param around_evaluation [#call, nil] called with (scenario, spec) and a
+      #   block that returns the Result. Establishes context for replay, scoring
+      #   and recommendations; must return the block's result. Wrapper errors
+      #   propagate to the caller. Applies to #call, not direct #evaluate calls.
+      # @param require_judge_scores [Boolean] fail an otherwise passing result
+      #   when a requested task/LLM grade is unavailable, rather than falling
+      #   back to rule scores. Does not require a judge for rules-only runs.
       def initialize(scenarios:, models:, replay:, criteria: [], judge: nil, judge_task: true, available_tools: {},
                      instructions: nil, agent_name: "The agent", threshold: PASS_THRESHOLD,
-                     refine_faults: DEFAULT_REFINE_FAULTS, judge_limit: DEFAULT_JUDGE_LIMIT, on_result: nil, metadata: {})
+                     refine_faults: DEFAULT_REFINE_FAULTS, judge_limit: DEFAULT_JUDGE_LIMIT, on_result: nil,
+                     around_evaluation: nil, require_judge_scores: false, metadata: {})
         @scenarios = scenarios
         @models = models
         @replay = replay
@@ -58,6 +68,8 @@ module ActiveAgent
         @refine_faults = refine_faults
         @judge_limit = judge_limit
         @on_result = on_result
+        @around_evaluation = around_evaluation
+        @require_judge_scores = require_judge_scores
         @metadata = metadata
         @judge_calls = 0
         @scorer = Scorer.new(criteria: criteria, judge: judge)
@@ -66,7 +78,7 @@ module ActiveAgent
       def call
         results = @scenarios.flat_map do |scenario|
           @models.map do |spec|
-            evaluate(scenario, spec).tap { |result| @on_result&.call(result) }
+            evaluate_with_context(scenario, spec).tap { |result| @on_result&.call(result) }
           end
         end
 
@@ -85,7 +97,9 @@ module ActiveAgent
         score = Scorer.mean(scores)
 
         diagnosis = Diagnosis.call(scenario: scenario, replay: replay, scores: scores, score: score,
-                                   available_tools: @available_tools.keys, threshold: @threshold, agent_name: @agent_name)
+                                   available_tools: @available_tools.keys, threshold: @threshold, agent_name: @agent_name,
+                                   judge_keys: llm_judge_keys)
+        diagnosis ||= unavailable_judge_diagnosis(scores)
         diagnosis_hash = diagnosis&.to_h
         refine!(diagnosis_hash, scenario, replay, diagnosis) if diagnosis_hash
 
@@ -102,10 +116,51 @@ module ActiveAgent
 
       private
 
+      def evaluate_with_context(scenario, spec)
+        return evaluate(scenario, spec) unless @around_evaluation
+
+        result = @around_evaluation.call(scenario, spec) { evaluate(scenario, spec) }
+        # A wrapper written the natural way — do something, yield, do something
+        # after — returns that last value rather than the Result. Left alone it
+        # reaches on_result and the Report, and fails somewhere far from the
+        # wrapper that caused it. Name the wrapper here instead.
+        unless result.is_a?(Result)
+          raise ArgumentError, "around_evaluation must return the Result its block yields, got #{result.class}"
+        end
+
+        result
+      end
+
       def judge_task?
         @judge && @judge_task && @criteria.none? { |criterion| criterion.to_h.stringify_keys["type"] == "llm_judge" }
       end
 
+      # The keys in `scores` a judge graded, so a low grade is not averaged
+      # away against rule checks.
+      def llm_judge_keys
+        @llm_judge_keys ||= @criteria.filter_map do |criterion|
+          value = criterion.to_h.stringify_keys
+          value["key"] if value["type"] == "llm_judge"
+        end
+      end
+
+      def unavailable_judge_diagnosis(scores)
+        return unless @require_judge_scores
+
+        keys = llm_judge_keys.dup
+        keys << "task_completion" if judge_task?
+        missing = keys.select { |key| scores[key].nil? }
+        return if missing.empty?
+
+        Diagnosis::Result.new(
+          fault: "judge_unavailable",
+          summary: "The evaluation judge did not return a usable score for #{missing.join(', ')}.",
+          recommendation: "Check the judge's credentials, model availability and JSON response, then re-run this evaluation. " \
+                          "The available rule scores do not establish answer quality.",
+          evidence: { "unscored_criteria" => missing }
+        )
+      end
+
       # Whatever the callable raises becomes an errored Replay, so one model
       # rejecting a parameter fails its scenario rather than the whole run.
       # A return value that is neither a Replay nor a Hash is the caller's

data/lib/active_agent/evals/scenario_parser.rb

@@ -15,11 +15,15 @@ module ActiveAgent
     #     options on a line
     #   - a JSON array of strings, or of objects with `prompt` (or `message`),
     #     `group`, `key`, `notes`, `tools`, `contains`, `not_contains`
+    #   - a grouped Suite document in YAML or JSON, retaining its expectations,
+    #     group names, stable keys, notes and production-only flags
     #
     # Every scenario gets a key unique within the paste, derived from its group
     # and position ("blame_3"), unless the line names one. The result is an
     # array of string-keyed hashes; `Scenario.from_hash` builds the structs.
     class ScenarioParser
+      class ParseError < ArgumentError; end
+
       LIST_MARKER = /\A\s*(?:[-*•]|\d+[.)])\s+/
       HEADING = /\A\s*#+\s+(.+?)\s*\z/
       BOLD_HEADING = /\A\s*\*\*(.+?)\*\*:?\s*(?:—.*)?\z/
@@ -28,13 +32,13 @@ module ActiveAgent
       BACKTICK_PROMPT = /\A`([^`]+)`/
       OPTION_KEYS = %w[tools contains not_contains key group notes].freeze
 
-      def self.parse(text)
-        new(text).parse
+      def self.parse(text, include_production_only: true)
+        new(text).parse(include_production_only: include_production_only)
       end
 
       # Parses and builds Scenario structs in one step.
-      def self.scenarios(text)
-        parse(text).map { |attrs| Scenario.from_hash(attrs) }
+      def self.scenarios(text, include_production_only: true)
+        parse(text, include_production_only: include_production_only).map { |attrs| Scenario.from_hash(attrs) }
       end
 
       def initialize(text)
@@ -42,12 +46,19 @@ module ActiveAgent
       end
 
       # @return [Array<Hash>] scenario attributes with string keys
-      def parse
+      def parse(include_production_only: true)
         stripped = @text.strip
         return [] if stripped.empty?
 
-        scenarios = json?(stripped) ? parse_json(stripped) : parse_lines(stripped)
-        assign_keys(scenarios)
+        scenarios = if json?(stripped)
+          parse_json(stripped)
+        elsif stripped.match?(/^(?:suite|groups):(?:\s|$)/)
+          parse_suite_yaml(stripped)
+        else
+          parse_lines(stripped)
+        end
+        assigned = assign_keys(scenarios)
+        include_production_only ? assigned : assigned.reject { |entry| entry["production_only"] }
       end
 
       private
@@ -58,6 +69,8 @@ module ActiveAgent
 
       def parse_json(text)
         parsed = JSON.parse(text)
+        return parse_suite(parsed) if parsed.is_a?(Hash) && parsed.key?("groups")
+
         parsed = parsed["scenarios"] if parsed.is_a?(Hash) && parsed.key?("scenarios")
         parsed = [ parsed ] if parsed.is_a?(Hash)
 
@@ -71,6 +84,39 @@ module ActiveAgent
         parse_lines(text)
       end
 
+      def parse_suite_yaml(text)
+        document = YAML.safe_load(text, aliases: true)
+        raise ParseError, "evaluation suite must contain a groups array" unless document.is_a?(Hash) && document["groups"].is_a?(Array)
+
+        parse_suite(document)
+      rescue Psych::Exception => e
+        raise ParseError, "invalid evaluation suite YAML: #{e.message}"
+      end
+
+      def parse_suite(document)
+        raise ParseError, "evaluation suite must contain a groups array" unless document["groups"].is_a?(Array)
+
+        document["groups"].each do |group|
+          unless group.is_a?(Hash) && (group["scenarios"].nil? || group["scenarios"].is_a?(Array))
+            raise ParseError, "each evaluation group must contain a scenarios array"
+          end
+          Array(group["scenarios"]).each do |entry|
+            unless entry.is_a?(Hash) && entry["prompt"].is_a?(String) && entry["prompt"].present?
+              raise ParseError, "each evaluation scenario must contain a prompt"
+            end
+            expectations = entry["expectations"] || entry["expect"]
+            if expectations && !expectations.is_a?(Hash)
+              raise ParseError, "scenario expectations must be an object"
+            end
+          end
+        end
+
+        Suite.new([ document ]).all_scenarios.map do |item|
+          scenario(prompt: item.prompt, group: item.group, group_name: item.group_name, key: item.key,
+                   notes: item.notes, expectations: item.expectations, production_only: item.production_only?)
+        end
+      end
+
       def scenario_from_hash(entry)
         entry = entry.stringify_keys
         prompt = entry["prompt"] || entry["message"] || entry["input"] || entry["question"]
@@ -84,9 +130,11 @@ module ActiveAgent
         scenario(
           prompt: prompt,
           group: entry["group"],
+          group_name: entry["group_name"],
           key: entry["key"],
           notes: entry["notes"],
-          expectations: expectations
+          expectations: expectations,
+          production_only: entry["production_only"] == true
         )
       end
 
@@ -172,13 +220,15 @@ module ActiveAgent
         text.to_s.gsub(/\*\*|__|`/, "").strip
       end
 
-      def scenario(prompt:, group: nil, key: nil, notes: nil, expectations: {})
+      def scenario(prompt:, group: nil, group_name: nil, key: nil, notes: nil, expectations: {}, production_only: false)
         {
           "prompt" => prompt.to_s.strip,
           "group" => group.presence&.to_s&.strip,
+          "group_name" => group_name.presence&.to_s&.strip,
           "key" => key.presence&.to_s&.strip,
           "notes" => notes.presence,
-          "expectations" => (expectations || {}).reject { |_, value| value.blank? }
+          "expectations" => (expectations || {}).reject { |_, value| value.blank? },
+          "production_only" => production_only
         }
       end
 

data/lib/active_agent/evals.rb

@@ -25,6 +25,7 @@ require_relative "evals/design_tokens"
 require_relative "evals/report_html"
 require_relative "evals/report"
 require_relative "evals/runner"
+require_relative "evals/publisher"
 
 # Scenario evaluations for agents that answer with tools.
 #

data/lib/active_agent/providers/_base_provider.rb

@@ -59,7 +59,8 @@ module ActiveAgent
                     :tools_function,                 # Callback (Tools)
                     :usage_stack,                    # Usage Tracking
                     :stream_usage_index,             # Usage Tracking (Streams)
-                    :max_tool_turns, :tool_turns     # Tool-loop safety
+                    :max_tool_turns, :tool_turns,    # Tool-loop safety
+                    :instrumentation_enabled        # Per-generation privacy
 
       # Upper bound on tool-calling round-trips within one generation. A
       # model that keeps emitting tool calls otherwise recurses until the
@@ -117,6 +118,7 @@ module ActiveAgent
         self.tools_function     = kwargs.delete(:tools_function)
         self.max_tool_turns     = kwargs.delete(:max_tool_turns) || DEFAULT_MAX_TOOL_TURNS
         self.tool_turns         = 0
+        self.instrumentation_enabled = kwargs.delete(:instrumentation) != false
         self.options            = options_klass.new(kwargs.extract!(*options_klass.keys))
         self.context            = kwargs
         self.message_stack      = []
@@ -175,6 +177,8 @@ module ActiveAgent
       # @yield block to instrument
       # @return [Object] block result
       def instrument(name, payload = {}, &block)
+        return block&.call(payload) unless instrumentation_enabled
+
         full_payload = { provider: service_name, provider_module: tag_name, trace_id: }.merge(payload)
         ActiveSupport::Notifications.instrument(name, full_payload, &block)
       end

data/lib/active_agent/providers/mock/messages/base.rb

@@ -21,8 +21,10 @@ module ActiveAgent
               if content_type == :text
                 self.content = value
               else
-                # For image/document, MockProvider doesn't support these, so ignore
-                # (or could raise an error)
+                # No vision here: an image/document stands in as a text
+                # marker, so a media-only turn still has content to validate
+                # and to concatenate with its neighbours when serialized.
+                self.content ||= "[#{content_type}]"
               end
             end
           end

data/lib/active_agent/providers/open_ai/responses/transforms.rb

@@ -295,25 +295,43 @@ module ActiveAgent
               if message.respond_to?(:serialize)
                 message.serialize
               elsif message.is_a?(Hash)
-                # If it has a role, it's a message - convert :text to :content
+                # If it has a role, it's a message. Its :text becomes :content,
+                # and an :image / :document alongside it becomes a content
+                # part — the same `{role:, text:, image:}` shorthand the Chat
+                # API and Anthropic transforms accept, so a caller sending
+                # history plus a multimodal turn gets the same request shape
+                # from every provider.
                 if message.key?(:role)
                   normalized = message.dup
-                  if normalized.key?(:text) && !normalized.key?(:content)
-                    normalized[:content] = normalized.delete(:text)
+                  # The shorthand keys always come off the message: left on,
+                  # they reach the request body as unknown parameters and the
+                  # API rejects the whole call. A blank one contributes no
+                  # part rather than an empty input_image the API would
+                  # refuse (or a nil document, which has no URL to send).
+                  text = normalized.delete(:text)
+                  image = normalized.delete(:image)
+                  document = normalized.delete(:document)
+
+                  unless normalized.key?(:content)
+                    parts = []
+                    parts << { type: "input_text", text: text } if text.present?
+                    parts << { type: "input_image", image_url: image } if image.present?
+                    parts << document_part(document) if document.present?
+
+                    if parts.size == 1 && parts.first[:type] == "input_text"
+                      normalized[:content] = parts.first[:text]
+                    elsif parts.any?
+                      normalized[:content] = parts
+                    end
                   end
                   return normalized
                 end
 
                 # Expand shorthand formats to full structures for content items
-                if message.key?(:image)
+                if message[:image].present?
                   { type: "input_image", image_url: message[:image] }
-                elsif message.key?(:document)
-                  document_value = message[:document]
-                  if document_value.start_with?("data:")
-                    { type: "input_file", filename: "document.pdf", file_data: document_value }
-                  else
-                    { type: "input_file", file_url: document_value }
-                  end
+                elsif message[:document].present?
+                  document_part(message[:document])
                 elsif message.key?(:text) && message.size == 1
                   # Single :text key without :role - treat as user message
                   { role: "user", content: message[:text] }
@@ -336,6 +354,19 @@ module ActiveAgent
               end
             end
 
+            # An input_file part for a document given as a URL or a data URI.
+            #
+            # @param document_value [String] URL or data URI
+            # @return [Hash] input_file content part
+            def document_part(document_value)
+              document_value = document_value.to_s
+              if document_value.start_with?("data:")
+                { type: "input_file", filename: "document.pdf", file_data: document_value }
+              else
+                { type: "input_file", file_url: document_value }
+              end
+            end
+
             # Cleans up serialized request for API submission
             #
             # Removes default values and simplifies input where possible.

data/lib/active_agent/railtie.rb

@@ -114,6 +114,16 @@ module ActiveAgent
     initializer "active_agent.inflections" do
       ActiveSupport::Inflector.inflections do |inflect|
         inflect.acronym "AI"
+
+        # "MCP" alone does not give the plural: an acronym only matches the
+        # whole word, so `mcps` still camelizes to `Mcps`, and a constant
+        # spelled `MCPs` underscores back to `mc_ps` — a name that round-trips
+        # to something no file is called.
+        #
+        # Registering the plural as its own acronym makes both directions
+        # agree: mcps <-> MCPs, alongside mcp_catalog <-> MCPCatalog.
+        inflect.acronym "MCP"
+        inflect.acronym "MCPs"
       end
     end