activeagent 1.3.0 → 1.4.0

data/lib/active_agent/evals/result.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # One scenario replayed under one model: the Replay, its scores, and the
+    # diagnosis when it fell short.
+    #
+    # `status` is "errored" when the run raised, "failed" when a fault was
+    # assigned, and "passed" otherwise. `diagnosis` is the Diagnosis::Result
+    # hash, with a `"judge"` sub-hash when a Judge refined it.
+    Result = Struct.new(:scenario, :spec, :replay, :scores, :score, :status, :diagnosis, keyword_init: true) do
+      def passed?
+        status == "passed"
+      end
+
+      def failed?
+        status == "failed"
+      end
+
+      def errored?
+        status == "errored"
+      end
+
+      def label
+        spec.label
+      end
+
+      def model
+        spec.model
+      end
+
+      def provider
+        spec.provider
+      end
+
+      def fault
+        diagnosis && diagnosis["fault"]
+      end
+
+      def summary
+        diagnosis && diagnosis["summary"]
+      end
+
+      def recommendation
+        diagnosis && diagnosis["recommendation"]
+      end
+
+      def suggested_tool
+        diagnosis&.dig("judge", "suggested_tool")
+      end
+
+      def to_h
+        {
+          "scenario_key" => scenario.key,
+          "group" => scenario.group,
+          "prompt" => scenario.prompt,
+          "label" => label,
+          "provider" => provider,
+          "model" => model,
+          "status" => status,
+          "score" => score,
+          "scores" => scores,
+          "answer" => replay.answer,
+          "tool_calls" => replay.tool_calls,
+          "duration_ms" => replay.duration_ms,
+          "input_tokens" => replay.input_tokens,
+          "output_tokens" => replay.output_tokens,
+          "cost" => replay.cost,
+          "error" => replay.error,
+          "fault" => fault,
+          "recommendation" => recommendation,
+          "diagnosis" => diagnosis,
+          "metadata" => replay.metadata
+        }.compact
+      end
+    end
+  end
+end

data/lib/active_agent/evals/runner.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # Replays every scenario under every candidate model, scores and diagnoses
+    # each answer, and returns a Report.
+    #
+    # The one thing the runner does not know is how to talk to your agent;
+    # `replay` is a callable `(scenario, model_spec) → Replay` (a Hash with the
+    # same keys is accepted, and an exception becomes an errored Replay). A
+    # scenario passes when its replay completed, met its expectations, and its
+    # mean score reached `threshold`; anything else carries exactly one fault
+    # and a recommendation from Diagnosis, refined by the `judge` for the
+    # faults in `refine_faults` (up to `judge_limit` calls per run).
+    #
+    #   Runner.new(
+    #     scenarios: suite.scenarios(groups: %w[blame]),
+    #     models: ModelSpec.parse_all(%w[gpt-5-mini claude-haiku-4-5], default_provider: "openai"),
+    #     criteria: [{ "key" => "response_present", "type" => "response_present" }],
+    #     available_tools: { "find_records" => "Look up records" },
+    #     instructions: agent.instructions,
+    #     judge: Judge.new(label: "claude-opus-5") { |instructions:, prompt:| ... },
+    #     replay: ->(scenario, spec) { ... },
+    #     on_result: ->(result) { persist(result) }
+    #   ).call
+    class Runner
+      # Faults where a judge can add something the evidence alone cannot: what
+      # tool to add, or how to change the instructions.
+      DEFAULT_REFINE_FAULTS = %w[missing_capability expected_tool_not_called low_quality missing_content].freeze
+      DEFAULT_JUDGE_LIMIT = 25
+
+      attr_reader :scenarios, :models, :criteria, :judge, :threshold
+
+      # @param scenarios [Array<Scenario>]
+      # @param models [Array<ModelSpec>]
+      # @param replay [#call] `(scenario, model_spec) → Replay`
+      # @param criteria [Array<Hash>] Scorer criteria applied to every replay
+      # @param judge [Judge, nil]
+      # @param judge_task [Boolean] with a judge and no llm_judge criterion, also
+      #   score task completion (`Judge#score_task`) as the `task_completion` criterion
+      # @param available_tools [Hash{String=>String}, Array<String>] the agent's tool roster
+      # @param instructions [String, nil] the agent's instructions, for the judge
+      # @param agent_name [String] how recommendations refer to the agent
+      # @param on_result [#call, nil] called with each Result as it lands
+      def initialize(scenarios:, models:, replay:, criteria: [], judge: nil, judge_task: true, available_tools: {},
+                     instructions: nil, agent_name: "The agent", threshold: PASS_THRESHOLD,
+                     refine_faults: DEFAULT_REFINE_FAULTS, judge_limit: DEFAULT_JUDGE_LIMIT, on_result: nil, metadata: {})
+        @scenarios = scenarios
+        @models = models
+        @replay = replay
+        @criteria = criteria
+        @judge = judge
+        @judge_task = judge_task
+        @available_tools = normalize_tools(available_tools)
+        @instructions = instructions
+        @agent_name = agent_name
+        @threshold = threshold
+        @refine_faults = refine_faults
+        @judge_limit = judge_limit
+        @on_result = on_result
+        @metadata = metadata
+        @judge_calls = 0
+        @scorer = Scorer.new(criteria: criteria, judge: judge)
+      end
+
+      def call
+        results = @scenarios.flat_map do |scenario|
+          @models.map do |spec|
+            evaluate(scenario, spec).tap { |result| @on_result&.call(result) }
+          end
+        end
+
+        Report.new(results: results, models: @models, judge: @judge, instructions: @instructions,
+                   threshold: @threshold, metadata: @metadata)
+      end
+
+      # Scores and diagnoses one replay. Public so a caller that has already run
+      # the agent (a background job per scenario, say) can score without the loop.
+      def evaluate(scenario, spec, replay = nil)
+        replay ||= run_replay(scenario, spec)
+        scores = @scorer.score(scenario, replay)
+        if judge_task? && replay.answer.present?
+          scores["task_completion"] = @judge.score_task(scenario: scenario, answer: replay.answer)
+        end
+        score = Scorer.mean(scores)
+
+        diagnosis = Diagnosis.call(scenario: scenario, replay: replay, scores: scores, score: score,
+                                   available_tools: @available_tools.keys, threshold: @threshold, agent_name: @agent_name)
+        diagnosis_hash = diagnosis&.to_h
+        refine!(diagnosis_hash, scenario, replay, diagnosis) if diagnosis_hash
+
+        Result.new(
+          scenario: scenario,
+          spec: spec,
+          replay: replay,
+          scores: scores,
+          score: score,
+          status: replay.errored? ? "errored" : (diagnosis ? "failed" : "passed"),
+          diagnosis: diagnosis_hash
+        )
+      end
+
+      private
+
+      def judge_task?
+        @judge && @judge_task && @criteria.none? { |criterion| criterion.to_h.stringify_keys["type"] == "llm_judge" }
+      end
+
+      # Whatever the callable raises becomes an errored Replay, so one model
+      # rejecting a parameter fails its scenario rather than the whole run.
+      # A return value that is neither a Replay nor a Hash is the caller's
+      # bug and raises.
+      def run_replay(scenario, spec)
+        value =
+          begin
+            @replay.call(scenario, spec)
+          rescue StandardError => e
+            return Replay.failed(e)
+          end
+
+        case value
+        when Replay then value
+        when Hash then Replay.new(**value.to_h.symbolize_keys)
+        else raise ArgumentError, "replay must return an ActiveAgent::Evals::Replay or a Hash, got #{value.class}"
+        end
+      end
+
+      def refine!(diagnosis, scenario, replay, result)
+        return unless @judge && @refine_faults.include?(result.fault)
+        return if @judge_calls >= @judge_limit
+
+        @judge_calls += 1
+        refined = @judge.recommend(scenario: scenario, replay: replay, diagnosis: result,
+                                   available_tools: @available_tools, instructions: @instructions)
+        return unless refined
+
+        diagnosis["judge"] = refined
+        diagnosis["recommendation"] = refined["recommendation"].to_s.strip if refined["recommendation"].present?
+      end
+
+      def normalize_tools(tools)
+        case tools
+        when Hash then tools.to_h { |name, description| [ name.to_s, description.to_s ] }
+        else Array(tools).to_h { |tool| tool.respond_to?(:name) ? [ tool.name.to_s, (tool.respond_to?(:description) ? tool.description.to_s : "") ] : [ tool.to_s, "" ] }
+        end
+      end
+    end
+  end
+end

data/lib/active_agent/evals/scenario.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # One task an evaluation replays through the agent: the message a user
+    # would send, the group of related tasks it belongs to, and what a passing
+    # answer is expected to do.
+    #
+    # @!attribute key
+    #   @return [String] stable within a suite ("blame_3"), so results line up across runs
+    # @!attribute group
+    #   @return [String, nil] the group key ("blame")
+    # @!attribute group_name
+    #   @return [String, nil] the group's display name ("Blame / audit")
+    # @!attribute expected_tools
+    #   @return [Array<String>] tool names a passing answer calls (any one of them)
+    # @!attribute expected_patterns
+    #   @return [Array<String>] substrings or patterns the answer must contain
+    # @!attribute forbidden_patterns
+    #   @return [Array<String>] substrings or patterns the answer must avoid
+    # @!attribute production_only
+    #   @return [Boolean] whether a local environment cannot answer with real data
+    Scenario = Struct.new(:key, :group, :group_name, :prompt, :expected_tools, :expected_patterns,
+                          :forbidden_patterns, :notes, :production_only, :position, keyword_init: true) do
+      # Builds a scenario from a hash — ScenarioParser output, a suite entry, or
+      # a persisted record's attributes. Expectations are read from an
+      # `expectations`/`expect` sub-hash (`tools`, `contains`, `not_contains`)
+      # or from those keys at the top level.
+      def self.from_hash(attributes, group: nil, group_name: nil)
+        attrs = attributes.to_h.deep_stringify_keys
+        expect = (attrs["expectations"] || attrs["expect"] || {}).to_h.stringify_keys
+
+        new(
+          key: attrs["key"].to_s,
+          group: attrs["group"].presence || group,
+          group_name: attrs["group_name"].presence || group_name,
+          prompt: attrs.fetch("prompt").to_s.strip,
+          expected_tools: list(expect["tools"] || attrs["tools"]),
+          expected_patterns: list(expect["contains"] || attrs["contains"]),
+          forbidden_patterns: list(expect["not_contains"] || attrs["not_contains"]),
+          notes: attrs["notes"].presence,
+          production_only: attrs["production_only"] == true,
+          position: attrs["position"]
+        )
+      end
+
+      def self.list(value)
+        Array(value).map(&:to_s).reject(&:blank?)
+      end
+
+      def production_only?
+        production_only == true
+      end
+
+      def expectations
+        {
+          "tools" => expected_tools,
+          "contains" => expected_patterns,
+          "not_contains" => forbidden_patterns
+        }.reject { |_, value| value.blank? }
+      end
+
+      def to_h
+        super.compact
+      end
+    end
+  end
+end

data/lib/active_agent/evals/scenario_parser.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # Turns a pasted list of user messages into scenario attributes. Accepts the
+    # shapes people actually paste:
+    #
+    #   - one message per line, with or without list markers (`-`, `*`, `1.`)
+    #   - `# Heading` or `**Heading**` lines, which start a group; so does a
+    #     short unmarked line ending in a colon (`Find records:`)
+    #   - a message in backticks at the start of the line, followed by notes,
+    #     as in an issue's question catalog:
+    #     `` 3. `Show me all providers with no license on file` — 1,060 locally ``
+    #   - trailing ` | tools: a, b | contains: x | not_contains: y | key: k`
+    #     options on a line
+    #   - a JSON array of strings, or of objects with `prompt` (or `message`),
+    #     `group`, `key`, `notes`, `tools`, `contains`, `not_contains`
+    #
+    # Every scenario gets a key unique within the paste, derived from its group
+    # and position ("blame_3"), unless the line names one. The result is an
+    # array of string-keyed hashes; `Scenario.from_hash` builds the structs.
+    class ScenarioParser
+      LIST_MARKER = /\A\s*(?:[-*•]|\d+[.)])\s+/
+      HEADING = /\A\s*#+\s+(.+?)\s*\z/
+      BOLD_HEADING = /\A\s*\*\*(.+?)\*\*:?\s*(?:—.*)?\z/
+      # Only a backticked span that opens the line is the prompt; a message
+      # that merely mentions `some_tool` is kept whole.
+      BACKTICK_PROMPT = /\A`([^`]+)`/
+      OPTION_KEYS = %w[tools contains not_contains key group notes].freeze
+
+      def self.parse(text)
+        new(text).parse
+      end
+
+      # Parses and builds Scenario structs in one step.
+      def self.scenarios(text)
+        parse(text).map { |attrs| Scenario.from_hash(attrs) }
+      end
+
+      def initialize(text)
+        @text = text.to_s
+      end
+
+      # @return [Array<Hash>] scenario attributes with string keys
+      def parse
+        stripped = @text.strip
+        return [] if stripped.empty?
+
+        scenarios = json?(stripped) ? parse_json(stripped) : parse_lines(stripped)
+        assign_keys(scenarios)
+      end
+
+      private
+
+      def json?(text)
+        text.start_with?("[", "{")
+      end
+
+      def parse_json(text)
+        parsed = JSON.parse(text)
+        parsed = parsed["scenarios"] if parsed.is_a?(Hash) && parsed.key?("scenarios")
+        parsed = [ parsed ] if parsed.is_a?(Hash)
+
+        Array(parsed).filter_map do |entry|
+          case entry
+          when String then scenario(prompt: entry)
+          when Hash then scenario_from_hash(entry)
+          end
+        end
+      rescue JSON::ParserError
+        parse_lines(text)
+      end
+
+      def scenario_from_hash(entry)
+        entry = entry.stringify_keys
+        prompt = entry["prompt"] || entry["message"] || entry["input"] || entry["question"]
+        return nil if prompt.blank?
+
+        expectations = (entry["expectations"] || entry["expect"] || {}).to_h.stringify_keys
+        %w[tools contains not_contains].each do |field|
+          expectations[field] = Array(entry[field]) if entry.key?(field)
+        end
+
+        scenario(
+          prompt: prompt,
+          group: entry["group"],
+          key: entry["key"],
+          notes: entry["notes"],
+          expectations: expectations
+        )
+      end
+
+      def parse_lines(text)
+        group = nil
+        scenarios = []
+
+        text.each_line do |raw|
+          line = raw.strip
+          next if line.empty?
+
+          if (heading = heading_for(line))
+            group = heading
+            next
+          end
+
+          scenarios << parse_line(line, group)
+        end
+
+        scenarios
+      end
+
+      def heading_for(line)
+        return Regexp.last_match(1).strip if line =~ HEADING
+        return strip_markup(Regexp.last_match(1)) if line =~ BOLD_HEADING
+        return line.chomp(":").strip if colon_heading?(line)
+
+        nil
+      end
+
+      # `Find records:` reads as a heading; a question, or a line carrying
+      # `| options`, does not, however it ends.
+      def colon_heading?(line)
+        line.end_with?(":") && line.length <= 80 && !line.match?(LIST_MARKER) &&
+          !line.include?("?") && !line.include?(" | ")
+      end
+
+      def parse_line(line, group)
+        body = line.sub(LIST_MARKER, "")
+        body, options = split_options(body)
+
+        if (match = body.match(BACKTICK_PROMPT))
+          prompt = match[1].strip
+          notes = body.sub(match[0], "").sub(/\A\s*[—–-]\s*/, "").strip.presence
+        else
+          prompt = strip_markup(body)
+          notes = nil
+        end
+
+        scenario(
+          prompt: prompt,
+          group: options["group"].presence || group,
+          key: options["key"],
+          notes: [ notes, options["notes"] ].compact.join(" ").presence,
+          expectations: options.slice("tools", "contains", "not_contains").transform_values { |value| split_list(value) }
+        )
+      end
+
+      # `prompt | tools: a, b | contains: x` → [prompt, { "tools" => "a, b", ... }]
+      def split_options(body)
+        segments = body.split(/\s+\|\s+/)
+        return [ body, {} ] if segments.size == 1
+
+        options = {}
+        rest = [ segments.shift ]
+        segments.each do |segment|
+          key, value = segment.split(":", 2)
+          if value && OPTION_KEYS.include?(key.strip.downcase)
+            options[key.strip.downcase] = value.strip
+          else
+            rest << segment
+          end
+        end
+
+        [ rest.join(" | "), options ]
+      end
+
+      def split_list(value)
+        value.to_s.split(/\s*[,;]\s*/).map(&:strip).reject(&:blank?)
+      end
+
+      def strip_markup(text)
+        text.to_s.gsub(/\*\*|__|`/, "").strip
+      end
+
+      def scenario(prompt:, group: nil, key: nil, notes: nil, expectations: {})
+        {
+          "prompt" => prompt.to_s.strip,
+          "group" => group.presence&.to_s&.strip,
+          "key" => key.presence&.to_s&.strip,
+          "notes" => notes.presence,
+          "expectations" => (expectations || {}).reject { |_, value| value.blank? }
+        }
+      end
+
+      # A key named on a line is kept; a generated one never collides with a
+      # named key anywhere in the paste; and a named key that repeats an
+      # earlier line's is treated as missing, so no two scenarios share one.
+      def assign_keys(scenarios)
+        named = scenarios.filter_map { |s| s["key"].presence }.to_set
+        taken = Set.new
+        counters = Hash.new(0)
+
+        scenarios.each_with_index do |scenario, index|
+          scenario["position"] = index
+          key = scenario["key"].presence
+          key = nil if key && taken.include?(key)
+
+          unless key
+            base = scenario["group"].to_s.parameterize(separator: "_").first(30).presence || "scenario"
+            loop do
+              counters[base] += 1
+              key = "#{base}_#{counters[base]}"
+              break unless named.include?(key) || taken.include?(key)
+            end
+          end
+
+          taken << key
+          scenario["key"] = key
+        end
+
+        scenarios
+      end
+    end
+  end
+end

data/lib/active_agent/evals/scorer.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # Scores one Replay against an evaluation's criteria and the scenario's
+    # expectations. Returns `criterion key => 0.0..1.0`, with nil for a
+    # criterion that could not be scored (an llm_judge criterion with no judge).
+    #
+    # Criteria are `{ "key", "type", "config" }` hashes:
+    #
+    #   response_present  — the answer is non-empty
+    #   min_length        — `config.chars` characters (partial credit below)
+    #   max_latency_ms    — `config.ms` budget (partial credit above)
+    #   token_budget      — `config.output_tokens` budget (partial credit above)
+    #   contains          — `config.pattern` (a substring, or a regex) is present
+    #   not_contains      — `config.pattern` is absent
+    #   llm_judge         — the judge scores the answer against `config.prompt`
+    #
+    # The scenario's own expectations add `expected_tools`, `expected_content`,
+    # `forbidden_content` (when declared) and `tools_succeeded` (when any tool
+    # was called).
+    class Scorer
+      RULE_CRITERION_TYPES = %w[response_present min_length max_latency_ms token_budget contains not_contains].freeze
+      CRITERION_TYPES = (RULE_CRITERION_TYPES + %w[llm_judge]).freeze
+
+      attr_reader :criteria, :judge
+
+      def initialize(criteria: [], judge: nil)
+        @criteria = Array(criteria).map { |criterion| criterion.to_h.deep_stringify_keys }
+        @judge = judge
+      end
+
+      def score(scenario, replay)
+        scores = {}
+        answer = replay.answer.to_s
+
+        @criteria.each do |criterion|
+          scores[criterion["key"]] = answer.present? ? score_criterion(criterion, scenario, replay) : 0.0
+        end
+
+        if scenario.expected_tools.any?
+          scores["expected_tools"] = (scenario.expected_tools & replay.tool_names).any? ? 1.0 : 0.0
+        end
+        if scenario.expected_patterns.any?
+          hits = scenario.expected_patterns.count { |pattern| self.class.matches_pattern?(answer, pattern) }
+          scores["expected_content"] = (hits.to_f / scenario.expected_patterns.size).round(3)
+        end
+        if scenario.forbidden_patterns.any?
+          hit = scenario.forbidden_patterns.any? { |pattern| self.class.matches_pattern?(answer, pattern) }
+          scores["forbidden_content"] = hit ? 0.0 : 1.0
+        end
+        if replay.tool_calls.any?
+          scores["tools_succeeded"] = replay.failed_tool_calls.any? ? 0.0 : 1.0
+        end
+
+        scores
+      end
+
+      # The mean of the scored criteria, or nil when nothing could be scored.
+      def self.mean(scores)
+        scored = scores.values.compact
+        return nil if scored.empty?
+
+        (scored.sum / scored.size).round(3)
+      end
+
+      # How long one pattern may take to match one answer. Patterns are
+      # whatever the scenario's author typed, and a pathological one must not
+      # stall the evaluation.
+      PATTERN_TIMEOUT = 1.0
+
+      # Whether `pattern` occurs in `text`: as a plain substring, case
+      # insensitively, or else as a regex. A pattern that is not a valid
+      # regex, or that takes longer than PATTERN_TIMEOUT, only counts as a
+      # substring.
+      def self.matches_pattern?(text, pattern)
+        pattern = pattern.to_s
+        return false if pattern.blank?
+
+        text = text.to_s
+        return true if text.downcase.include?(pattern.downcase)
+
+        text.match?(Regexp.new(pattern, Regexp::IGNORECASE, timeout: PATTERN_TIMEOUT))
+      rescue RegexpError
+        false
+      end
+
+      private
+
+      def score_criterion(criterion, scenario, replay)
+        config = criterion["config"] || {}
+        answer = replay.answer.to_s
+
+        case criterion["type"]
+        when "response_present"
+          answer.present? ? 1.0 : 0.0
+        when "min_length"
+          min = config.fetch("chars", 40).to_i
+          [ answer.length.to_f / [ min, 1 ].max, 1.0 ].min
+        when "max_latency_ms"
+          budget = config.fetch("ms", 5_000).to_f
+          duration = replay.duration_ms.to_f
+          duration.zero? || duration <= budget ? 1.0 : [ budget / duration, 1.0 ].min
+        when "token_budget"
+          budget = config.fetch("output_tokens", 1_000).to_f
+          tokens = replay.output_tokens.to_f
+          tokens <= budget ? 1.0 : [ budget / tokens, 1.0 ].min
+        when "contains"
+          self.class.matches_pattern?(answer, config["pattern"]) ? 1.0 : 0.0
+        when "not_contains"
+          self.class.matches_pattern?(answer, config["pattern"]) ? 0.0 : 1.0
+        when "llm_judge"
+          @judge&.score_criterion(criterion: criterion, prompt: scenario.prompt, answer: answer)
+        end
+      end
+    end
+  end
+end