activeagent 1.3.1 → 1.5.0

data/lib/active_agent/evals/result.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # One scenario replayed under one model: the Replay, its scores, and the
+    # diagnosis when it fell short.
+    #
+    # `status` is "errored" when the run raised, "failed" when a fault was
+    # assigned, and "passed" otherwise. `diagnosis` is the Diagnosis::Result
+    # hash, with a `"judge"` sub-hash when a Judge refined it.
+    Result = Struct.new(:scenario, :spec, :replay, :scores, :score, :status, :diagnosis, keyword_init: true) do
+      def passed?
+        status == "passed"
+      end
+
+      def failed?
+        status == "failed"
+      end
+
+      def errored?
+        status == "errored"
+      end
+
+      def label
+        spec.label
+      end
+
+      def model
+        spec.model
+      end
+
+      def provider
+        spec.provider
+      end
+
+      def fault
+        diagnosis && diagnosis["fault"]
+      end
+
+      def summary
+        diagnosis && diagnosis["summary"]
+      end
+
+      def recommendation
+        diagnosis && diagnosis["recommendation"]
+      end
+
+      def suggested_tool
+        diagnosis&.dig("judge", "suggested_tool")
+      end
+
+      def to_h
+        {
+          "scenario_key" => scenario.key,
+          "group" => scenario.group,
+          "prompt" => scenario.prompt,
+          "label" => label,
+          "provider" => provider,
+          "model" => model,
+          "status" => status,
+          "score" => score,
+          "scores" => scores,
+          "answer" => replay.answer,
+          "tool_calls" => replay.tool_calls,
+          "duration_ms" => replay.duration_ms,
+          "input_tokens" => replay.input_tokens,
+          "output_tokens" => replay.output_tokens,
+          "cost" => replay.cost,
+          "error" => replay.error,
+          "fault" => fault,
+          "recommendation" => recommendation,
+          "diagnosis" => diagnosis,
+          "metadata" => replay.metadata
+        }.compact
+      end
+    end
+  end
+end

data/lib/active_agent/evals/runner.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # Replays every scenario under every candidate model, scores and diagnoses
+    # each answer, and returns a Report.
+    #
+    # The one thing the runner does not know is how to talk to your agent;
+    # `replay` is a callable `(scenario, model_spec) → Replay` (a Hash with the
+    # same keys is accepted, and an exception becomes an errored Replay). A
+    # scenario passes when its replay completed, met its expectations, and both
+    # its mean score and the mean of its judge grades (task completion, or the
+    # configured llm_judge criteria) reached `threshold`;
+    # anything else carries exactly one fault
+    # and a recommendation from Diagnosis, refined by the `judge` for the
+    # faults in `refine_faults` (up to `judge_limit` calls per run).
+    #
+    #   Runner.new(
+    #     scenarios: suite.scenarios(groups: %w[blame]),
+    #     models: ModelSpec.parse_all(%w[gpt-5-mini claude-haiku-4-5], default_provider: "openai"),
+    #     criteria: [{ "key" => "response_present", "type" => "response_present" }],
+    #     available_tools: { "find_records" => "Look up records" },
+    #     instructions: agent.instructions,
+    #     judge: Judge.new(label: "claude-opus-5") { |instructions:, prompt:| ... },
+    #     replay: ->(scenario, spec) { ... },
+    #     on_result: ->(result) { persist(result) }
+    #   ).call
+    class Runner
+      # Faults where a judge can add something the evidence alone cannot: what
+      # tool to add, or how to change the instructions.
+      DEFAULT_REFINE_FAULTS = %w[missing_capability expected_tool_not_called low_quality missing_content].freeze
+      DEFAULT_JUDGE_LIMIT = 25
+
+      attr_reader :scenarios, :models, :criteria, :judge, :threshold
+
+      # @param scenarios [Array<Scenario>]
+      # @param models [Array<ModelSpec>]
+      # @param replay [#call] `(scenario, model_spec) → Replay`
+      # @param criteria [Array<Hash>] Scorer criteria applied to every replay
+      # @param judge [Judge, nil]
+      # @param judge_task [Boolean] with a judge and no llm_judge criterion, also
+      #   score task completion (`Judge#score_task`) as the `task_completion` criterion
+      # @param available_tools [Hash{String=>String}, Array<String>] the agent's tool roster
+      # @param instructions [String, nil] the agent's instructions, for the judge
+      # @param agent_name [String] how recommendations refer to the agent
+      # @param on_result [#call, nil] called with each Result as it lands
+      # @param around_evaluation [#call, nil] called with (scenario, spec) and a
+      #   block that returns the Result. Establishes context for replay, scoring
+      #   and recommendations; must return the block's result. Wrapper errors
+      #   propagate to the caller. Applies to #call, not direct #evaluate calls.
+      # @param require_judge_scores [Boolean] fail an otherwise passing result
+      #   when a requested task/LLM grade is unavailable, rather than falling
+      #   back to rule scores. Does not require a judge for rules-only runs.
+      def initialize(scenarios:, models:, replay:, criteria: [], judge: nil, judge_task: true, available_tools: {},
+                     instructions: nil, agent_name: "The agent", threshold: PASS_THRESHOLD,
+                     refine_faults: DEFAULT_REFINE_FAULTS, judge_limit: DEFAULT_JUDGE_LIMIT, on_result: nil,
+                     around_evaluation: nil, require_judge_scores: false, metadata: {})
+        @scenarios = scenarios
+        @models = models
+        @replay = replay
+        @criteria = criteria
+        @judge = judge
+        @judge_task = judge_task
+        @available_tools = normalize_tools(available_tools)
+        @instructions = instructions
+        @agent_name = agent_name
+        @threshold = threshold
+        @refine_faults = refine_faults
+        @judge_limit = judge_limit
+        @on_result = on_result
+        @around_evaluation = around_evaluation
+        @require_judge_scores = require_judge_scores
+        @metadata = metadata
+        @judge_calls = 0
+        @scorer = Scorer.new(criteria: criteria, judge: judge)
+      end
+
+      def call
+        results = @scenarios.flat_map do |scenario|
+          @models.map do |spec|
+            evaluate_with_context(scenario, spec).tap { |result| @on_result&.call(result) }
+          end
+        end
+
+        Report.new(results: results, models: @models, judge: @judge, instructions: @instructions,
+                   threshold: @threshold, metadata: @metadata)
+      end
+
+      # Scores and diagnoses one replay. Public so a caller that has already run
+      # the agent (a background job per scenario, say) can score without the loop.
+      def evaluate(scenario, spec, replay = nil)
+        replay ||= run_replay(scenario, spec)
+        scores = @scorer.score(scenario, replay)
+        if judge_task? && replay.answer.present?
+          scores["task_completion"] = @judge.score_task(scenario: scenario, answer: replay.answer)
+        end
+        score = Scorer.mean(scores)
+
+        diagnosis = Diagnosis.call(scenario: scenario, replay: replay, scores: scores, score: score,
+                                   available_tools: @available_tools.keys, threshold: @threshold, agent_name: @agent_name,
+                                   judge_keys: llm_judge_keys)
+        diagnosis ||= unavailable_judge_diagnosis(scores)
+        diagnosis_hash = diagnosis&.to_h
+        refine!(diagnosis_hash, scenario, replay, diagnosis) if diagnosis_hash
+
+        Result.new(
+          scenario: scenario,
+          spec: spec,
+          replay: replay,
+          scores: scores,
+          score: score,
+          status: replay.errored? ? "errored" : (diagnosis ? "failed" : "passed"),
+          diagnosis: diagnosis_hash
+        )
+      end
+
+      private
+
+      def evaluate_with_context(scenario, spec)
+        return evaluate(scenario, spec) unless @around_evaluation
+
+        result = @around_evaluation.call(scenario, spec) { evaluate(scenario, spec) }
+        # A wrapper written the natural way — do something, yield, do something
+        # after — returns that last value rather than the Result. Left alone it
+        # reaches on_result and the Report, and fails somewhere far from the
+        # wrapper that caused it. Name the wrapper here instead.
+        unless result.is_a?(Result)
+          raise ArgumentError, "around_evaluation must return the Result its block yields, got #{result.class}"
+        end
+
+        result
+      end
+
+      def judge_task?
+        @judge && @judge_task && @criteria.none? { |criterion| criterion.to_h.stringify_keys["type"] == "llm_judge" }
+      end
+
+      # The keys in `scores` a judge graded, so a low grade is not averaged
+      # away against rule checks.
+      def llm_judge_keys
+        @llm_judge_keys ||= @criteria.filter_map do |criterion|
+          value = criterion.to_h.stringify_keys
+          value["key"] if value["type"] == "llm_judge"
+        end
+      end
+
+      def unavailable_judge_diagnosis(scores)
+        return unless @require_judge_scores
+
+        keys = llm_judge_keys.dup
+        keys << "task_completion" if judge_task?
+        missing = keys.select { |key| scores[key].nil? }
+        return if missing.empty?
+
+        Diagnosis::Result.new(
+          fault: "judge_unavailable",
+          summary: "The evaluation judge did not return a usable score for #{missing.join(', ')}.",
+          recommendation: "Check the judge's credentials, model availability and JSON response, then re-run this evaluation. " \
+                          "The available rule scores do not establish answer quality.",
+          evidence: { "unscored_criteria" => missing }
+        )
+      end
+
+      # Whatever the callable raises becomes an errored Replay, so one model
+      # rejecting a parameter fails its scenario rather than the whole run.
+      # A return value that is neither a Replay nor a Hash is the caller's
+      # bug and raises.
+      def run_replay(scenario, spec)
+        value =
+          begin
+            @replay.call(scenario, spec)
+          rescue StandardError => e
+            return Replay.failed(e)
+          end
+
+        case value
+        when Replay then value
+        when Hash then Replay.new(**value.to_h.symbolize_keys)
+        else raise ArgumentError, "replay must return an ActiveAgent::Evals::Replay or a Hash, got #{value.class}"
+        end
+      end
+
+      def refine!(diagnosis, scenario, replay, result)
+        return unless @judge && @refine_faults.include?(result.fault)
+        return if @judge_calls >= @judge_limit
+
+        @judge_calls += 1
+        refined = @judge.recommend(scenario: scenario, replay: replay, diagnosis: result,
+                                   available_tools: @available_tools, instructions: @instructions)
+        return unless refined
+
+        diagnosis["judge"] = refined
+        diagnosis["recommendation"] = refined["recommendation"].to_s.strip if refined["recommendation"].present?
+      end
+
+      def normalize_tools(tools)
+        case tools
+        when Hash then tools.to_h { |name, description| [ name.to_s, description.to_s ] }
+        else Array(tools).to_h { |tool| tool.respond_to?(:name) ? [ tool.name.to_s, (tool.respond_to?(:description) ? tool.description.to_s : "") ] : [ tool.to_s, "" ] }
+        end
+      end
+    end
+  end
+end

data/lib/active_agent/evals/scenario.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # One task an evaluation replays through the agent: the message a user
+    # would send, the group of related tasks it belongs to, and what a passing
+    # answer is expected to do.
+    #
+    # @!attribute key
+    #   @return [String] stable within a suite ("blame_3"), so results line up across runs
+    # @!attribute group
+    #   @return [String, nil] the group key ("blame")
+    # @!attribute group_name
+    #   @return [String, nil] the group's display name ("Blame / audit")
+    # @!attribute expected_tools
+    #   @return [Array<String>] tool names a passing answer calls (any one of them)
+    # @!attribute expected_patterns
+    #   @return [Array<String>] substrings or patterns the answer must contain
+    # @!attribute forbidden_patterns
+    #   @return [Array<String>] substrings or patterns the answer must avoid
+    # @!attribute production_only
+    #   @return [Boolean] whether a local environment cannot answer with real data
+    Scenario = Struct.new(:key, :group, :group_name, :prompt, :expected_tools, :expected_patterns,
+                          :forbidden_patterns, :notes, :production_only, :position, keyword_init: true) do
+      # Builds a scenario from a hash — ScenarioParser output, a suite entry, or
+      # a persisted record's attributes. Expectations are read from an
+      # `expectations`/`expect` sub-hash (`tools`, `contains`, `not_contains`)
+      # or from those keys at the top level.
+      def self.from_hash(attributes, group: nil, group_name: nil)
+        attrs = attributes.to_h.deep_stringify_keys
+        expect = (attrs["expectations"] || attrs["expect"] || {}).to_h.stringify_keys
+
+        new(
+          key: attrs["key"].to_s,
+          group: attrs["group"].presence || group,
+          group_name: attrs["group_name"].presence || group_name,
+          prompt: attrs.fetch("prompt").to_s.strip,
+          expected_tools: list(expect["tools"] || attrs["tools"]),
+          expected_patterns: list(expect["contains"] || attrs["contains"]),
+          forbidden_patterns: list(expect["not_contains"] || attrs["not_contains"]),
+          notes: attrs["notes"].presence,
+          production_only: attrs["production_only"] == true,
+          position: attrs["position"]
+        )
+      end
+
+      def self.list(value)
+        Array(value).map(&:to_s).reject(&:blank?)
+      end
+
+      def production_only?
+        production_only == true
+      end
+
+      def expectations
+        {
+          "tools" => expected_tools,
+          "contains" => expected_patterns,
+          "not_contains" => forbidden_patterns
+        }.reject { |_, value| value.blank? }
+      end
+
+      def to_h
+        super.compact
+      end
+    end
+  end
+end

data/lib/active_agent/evals/scenario_parser.rb

@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # Turns a pasted list of user messages into scenario attributes. Accepts the
+    # shapes people actually paste:
+    #
+    #   - one message per line, with or without list markers (`-`, `*`, `1.`)
+    #   - `# Heading` or `**Heading**` lines, which start a group; so does a
+    #     short unmarked line ending in a colon (`Find records:`)
+    #   - a message in backticks at the start of the line, followed by notes,
+    #     as in an issue's question catalog:
+    #     `` 3. `Show me all providers with no license on file` — 1,060 locally ``
+    #   - trailing ` | tools: a, b | contains: x | not_contains: y | key: k`
+    #     options on a line
+    #   - a JSON array of strings, or of objects with `prompt` (or `message`),
+    #     `group`, `key`, `notes`, `tools`, `contains`, `not_contains`
+    #   - a grouped Suite document in YAML or JSON, retaining its expectations,
+    #     group names, stable keys, notes and production-only flags
+    #
+    # Every scenario gets a key unique within the paste, derived from its group
+    # and position ("blame_3"), unless the line names one. The result is an
+    # array of string-keyed hashes; `Scenario.from_hash` builds the structs.
+    class ScenarioParser
+      class ParseError < ArgumentError; end
+
+      LIST_MARKER = /\A\s*(?:[-*•]|\d+[.)])\s+/
+      HEADING = /\A\s*#+\s+(.+?)\s*\z/
+      BOLD_HEADING = /\A\s*\*\*(.+?)\*\*:?\s*(?:—.*)?\z/
+      # Only a backticked span that opens the line is the prompt; a message
+      # that merely mentions `some_tool` is kept whole.
+      BACKTICK_PROMPT = /\A`([^`]+)`/
+      OPTION_KEYS = %w[tools contains not_contains key group notes].freeze
+
+      def self.parse(text, include_production_only: true)
+        new(text).parse(include_production_only: include_production_only)
+      end
+
+      # Parses and builds Scenario structs in one step.
+      def self.scenarios(text, include_production_only: true)
+        parse(text, include_production_only: include_production_only).map { |attrs| Scenario.from_hash(attrs) }
+      end
+
+      def initialize(text)
+        @text = text.to_s
+      end
+
+      # @return [Array<Hash>] scenario attributes with string keys
+      def parse(include_production_only: true)
+        stripped = @text.strip
+        return [] if stripped.empty?
+
+        scenarios = if json?(stripped)
+          parse_json(stripped)
+        elsif stripped.match?(/^(?:suite|groups):(?:\s|$)/)
+          parse_suite_yaml(stripped)
+        else
+          parse_lines(stripped)
+        end
+        assigned = assign_keys(scenarios)
+        include_production_only ? assigned : assigned.reject { |entry| entry["production_only"] }
+      end
+
+      private
+
+      def json?(text)
+        text.start_with?("[", "{")
+      end
+
+      def parse_json(text)
+        parsed = JSON.parse(text)
+        return parse_suite(parsed) if parsed.is_a?(Hash) && parsed.key?("groups")
+
+        parsed = parsed["scenarios"] if parsed.is_a?(Hash) && parsed.key?("scenarios")
+        parsed = [ parsed ] if parsed.is_a?(Hash)
+
+        Array(parsed).filter_map do |entry|
+          case entry
+          when String then scenario(prompt: entry)
+          when Hash then scenario_from_hash(entry)
+          end
+        end
+      rescue JSON::ParserError
+        parse_lines(text)
+      end
+
+      def parse_suite_yaml(text)
+        document = YAML.safe_load(text, aliases: true)
+        raise ParseError, "evaluation suite must contain a groups array" unless document.is_a?(Hash) && document["groups"].is_a?(Array)
+
+        parse_suite(document)
+      rescue Psych::Exception => e
+        raise ParseError, "invalid evaluation suite YAML: #{e.message}"
+      end
+
+      def parse_suite(document)
+        raise ParseError, "evaluation suite must contain a groups array" unless document["groups"].is_a?(Array)
+
+        document["groups"].each do |group|
+          unless group.is_a?(Hash) && (group["scenarios"].nil? || group["scenarios"].is_a?(Array))
+            raise ParseError, "each evaluation group must contain a scenarios array"
+          end
+          Array(group["scenarios"]).each do |entry|
+            unless entry.is_a?(Hash) && entry["prompt"].is_a?(String) && entry["prompt"].present?
+              raise ParseError, "each evaluation scenario must contain a prompt"
+            end
+            expectations = entry["expectations"] || entry["expect"]
+            if expectations && !expectations.is_a?(Hash)
+              raise ParseError, "scenario expectations must be an object"
+            end
+          end
+        end
+
+        Suite.new([ document ]).all_scenarios.map do |item|
+          scenario(prompt: item.prompt, group: item.group, group_name: item.group_name, key: item.key,
+                   notes: item.notes, expectations: item.expectations, production_only: item.production_only?)
+        end
+      end
+
+      def scenario_from_hash(entry)
+        entry = entry.stringify_keys
+        prompt = entry["prompt"] || entry["message"] || entry["input"] || entry["question"]
+        return nil if prompt.blank?
+
+        expectations = (entry["expectations"] || entry["expect"] || {}).to_h.stringify_keys
+        %w[tools contains not_contains].each do |field|
+          expectations[field] = Array(entry[field]) if entry.key?(field)
+        end
+
+        scenario(
+          prompt: prompt,
+          group: entry["group"],
+          group_name: entry["group_name"],
+          key: entry["key"],
+          notes: entry["notes"],
+          expectations: expectations,
+          production_only: entry["production_only"] == true
+        )
+      end
+
+      def parse_lines(text)
+        group = nil
+        scenarios = []
+
+        text.each_line do |raw|
+          line = raw.strip
+          next if line.empty?
+
+          if (heading = heading_for(line))
+            group = heading
+            next
+          end
+
+          scenarios << parse_line(line, group)
+        end
+
+        scenarios
+      end
+
+      def heading_for(line)
+        return Regexp.last_match(1).strip if line =~ HEADING
+        return strip_markup(Regexp.last_match(1)) if line =~ BOLD_HEADING
+        return line.chomp(":").strip if colon_heading?(line)
+
+        nil
+      end
+
+      # `Find records:` reads as a heading; a question, or a line carrying
+      # `| options`, does not, however it ends.
+      def colon_heading?(line)
+        line.end_with?(":") && line.length <= 80 && !line.match?(LIST_MARKER) &&
+          !line.include?("?") && !line.include?(" | ")
+      end
+
+      def parse_line(line, group)
+        body = line.sub(LIST_MARKER, "")
+        body, options = split_options(body)
+
+        if (match = body.match(BACKTICK_PROMPT))
+          prompt = match[1].strip
+          notes = body.sub(match[0], "").sub(/\A\s*[—–-]\s*/, "").strip.presence
+        else
+          prompt = strip_markup(body)
+          notes = nil
+        end
+
+        scenario(
+          prompt: prompt,
+          group: options["group"].presence || group,
+          key: options["key"],
+          notes: [ notes, options["notes"] ].compact.join(" ").presence,
+          expectations: options.slice("tools", "contains", "not_contains").transform_values { |value| split_list(value) }
+        )
+      end
+
+      # `prompt | tools: a, b | contains: x` → [prompt, { "tools" => "a, b", ... }]
+      def split_options(body)
+        segments = body.split(/\s+\|\s+/)
+        return [ body, {} ] if segments.size == 1
+
+        options = {}
+        rest = [ segments.shift ]
+        segments.each do |segment|
+          key, value = segment.split(":", 2)
+          if value && OPTION_KEYS.include?(key.strip.downcase)
+            options[key.strip.downcase] = value.strip
+          else
+            rest << segment
+          end
+        end
+
+        [ rest.join(" | "), options ]
+      end
+
+      def split_list(value)
+        value.to_s.split(/\s*[,;]\s*/).map(&:strip).reject(&:blank?)
+      end
+
+      def strip_markup(text)
+        text.to_s.gsub(/\*\*|__|`/, "").strip
+      end
+
+      def scenario(prompt:, group: nil, group_name: nil, key: nil, notes: nil, expectations: {}, production_only: false)
+        {
+          "prompt" => prompt.to_s.strip,
+          "group" => group.presence&.to_s&.strip,
+          "group_name" => group_name.presence&.to_s&.strip,
+          "key" => key.presence&.to_s&.strip,
+          "notes" => notes.presence,
+          "expectations" => (expectations || {}).reject { |_, value| value.blank? },
+          "production_only" => production_only
+        }
+      end
+
+      # A key named on a line is kept; a generated one never collides with a
+      # named key anywhere in the paste; and a named key that repeats an
+      # earlier line's is treated as missing, so no two scenarios share one.
+      def assign_keys(scenarios)
+        named = scenarios.filter_map { |s| s["key"].presence }.to_set
+        taken = Set.new
+        counters = Hash.new(0)
+
+        scenarios.each_with_index do |scenario, index|
+          scenario["position"] = index
+          key = scenario["key"].presence
+          key = nil if key && taken.include?(key)
+
+          unless key
+            base = scenario["group"].to_s.parameterize(separator: "_").first(30).presence || "scenario"
+            loop do
+              counters[base] += 1
+              key = "#{base}_#{counters[base]}"
+              break unless named.include?(key) || taken.include?(key)
+            end
+          end
+
+          taken << key
+          scenario["key"] = key
+        end
+
+        scenarios
+      end
+    end
+  end
+end