brute 5.0.4 → 5.1.0

data/lib/brute/eval/case.rb

@@ -0,0 +1,249 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "brute"
+
+module Brute
+  module Eval
+    # The budget a case allows itself. Lenient on purpose: an agent that took
+    # one search too many is worth knowing about, but it is not the same
+    # fault as an agent that answered wrongly.
+    Budget = Data.define(:iterations, :tool_calls, :tokens, :seconds) do
+      def initialize(iterations: 10, tool_calls: 8, tokens: 100_000, seconds: 180)
+        super
+      end
+    end
+
+    # One evaluation case: what the agent is told, the world it is told it
+    # in, and what must be true of the turn afterwards.
+    #
+    #   Brute::Eval::Case.new(
+    #     "searches for what it cannot know",
+    #     said:     "what did the Bank of England do yesterday?",
+    #     stubs:    { "search" => RATE_DECISION },
+    #     calls:    { "search" => { "query" => /bank|rate/i } },
+    #     mentions: %w[4.25],
+    #     budget:   Brute::Eval::Budget.new(tool_calls: 2),
+    #   )
+    #
+    # The expectations are deliberately about what the turn DID, not about
+    # prose: a call that was made, a call that was not, the order two calls
+    # came in, a word the answer has to contain, a budget it has to stay
+    # inside. What none of those can say goes in the block, which is handed
+    # the transcript.
+    #
+    # `said` reaches the agent through the world -- an inbox on disk, a queue,
+    # whatever that deployment's world does with it -- unless `via: :start`
+    # hands it straight to the turn. `files` and `conversation` are the
+    # world's to lay out and mean nothing to a world that keeps no state.
+    class Case
+      # A system prompt that tells an agent to say plainly when it found
+      # nothing is graded on this. It is a crude reading -- a judge would do
+      # it properly -- and it is English, so a deployment whose agents answer
+      # in another language passes its own `absence:`.
+      ABSENCE = /\b(no|not|none|nothing|cannot|can't|couldn't|didn't|don't|doesn't|isn't|aren't|unable|unfortunately|missing|without)\b/i
+
+      Verdict = Data.define(:failures) do
+        def passed? = failures.empty?
+      end
+
+      attr_reader :name, :said, :via, :files, :conversation, :stubs, :budget, :runs
+
+      def initialize(
+        name,
+        said: nil,
+        via: :world,
+        files: {},
+        conversation: [],
+        stubs: {},
+        calls: {},
+        never: [],
+        order: [],
+        mentions: [],
+        absent: false,
+        absence: ABSENCE,
+        silent: false,
+        budget: Budget.new,
+        runs: 1,
+        &check
+      )
+        @name = name
+        @said = said
+        @via = via
+        @files = files
+        @conversation = conversation
+        @stubs = stubs
+        @calls = calls
+        @never = never
+        @order = order
+        @mentions = mentions
+        @absent = absent
+        @absence = absence
+        @silent = silent
+        @budget = budget
+        @runs = runs
+        @check = check
+      end
+
+      def verdict(transcript)
+        Verdict.new(
+          [
+            answered(transcript),
+            called(transcript),
+            ordered(transcript),
+            said_it(transcript),
+            afforded(transcript),
+            checked(transcript),
+          ].flatten.compact.uniq
+        )
+      end
+
+      private
+
+        def answered(transcript)
+          [].tap do |failures|
+            if transcript.error
+              failures << "raised #{transcript.error.class}: #{transcript.error.message}"
+            end
+
+            transcript.failures.each { |failure| failures << "the model call failed -- #{failure}" }
+
+            if transcript.reply.empty? && !@silent
+              failures << "said nothing"
+            end
+
+            if !transcript.reply.empty? && @silent
+              failures << "answered when it had nothing to answer"
+            end
+          end
+        end
+
+        def called(transcript)
+          [].tap do |failures|
+            @calls.each do |name, arguments|
+              unless transcript.called?(name, arguments || {})
+                failures << "never called #{name}#{about(arguments)}"
+              end
+            end
+
+            @never.each do |name|
+              if transcript.called?(name)
+                failures << "called #{name}"
+              end
+            end
+          end
+        end
+
+        def ordered(transcript)
+          @order.each_cons(2).filter_map do |first, second|
+            unless transcript.before?(first, second)
+              if transcript.called?(first)
+                "called #{second} before #{first}"
+              else
+                "never called #{first}"
+              end
+            end
+          end
+        end
+
+        def said_it(transcript)
+          @mentions.filter_map { |word|
+            unless transcript.reply.downcase.include?(word.downcase)
+              "never said #{word.inspect}"
+            end
+          }.tap do |failures|
+            if @absent && !@absence.match?(transcript.reply)
+              failures << "did not say it had nothing"
+            end
+          end
+        end
+
+        def afforded(transcript)
+          [
+            over("iterations", transcript.iterations, @budget.iterations),
+            over("tool calls", transcript.calls.length, @budget.tool_calls),
+            over("tokens", transcript.tokens, @budget.tokens),
+            over("seconds", transcript.seconds.round, @budget.seconds),
+          ]
+        end
+
+        def checked(transcript)
+          if @check && !@check.call(transcript)
+            "failed the case's own check"
+          end
+        end
+
+        def over(what, spent, allowed)
+          if spent > allowed
+            "spent #{spent} #{what}, budget #{allowed}"
+          end
+        end
+
+        def about(arguments)
+          if arguments.nil? || arguments.empty?
+            ""
+          else
+            " with #{arguments.inspect}"
+          end
+        end
+    end
+  end
+end
+
+__END__
+
+describe "brute/eval/case" do
+  it "grades a turn on what it did, and says what was wrong when it did not" do
+    turn = Struct.new(:names, :reply, :iterations, :tokens, :seconds, :error, :calls) do
+      def failures = []
+      def called?(name, arguments = {}) = names.include?(name.to_s) && arguments.empty?
+      def before?(first, second) = names.index(first).to_i < (names.index(second) || 99)
+    end
+
+    searched = turn.new(%w[search], "It held at 4.25%.", 2, 900, 3, nil, [1])
+
+    good = Brute::Eval::Case.new(
+      "searches for what it cannot know",
+      said: "what did the bank do?",
+      calls: { "search" => {} },
+      mentions: %w[4.25],
+      never: %w[create_event]
+    )
+    good.verdict(searched).passed?.should.be.true
+    good.via.should == :world
+    good.runs.should == 1
+
+    fussy = Brute::Eval::Case.new(
+      "does not search for what it knows",
+      said: "how many minutes in an hour?",
+      via: :start,
+      never: %w[search],
+      mentions: %w[sixty],
+      order: %w[read search],
+      budget: Brute::Eval::Budget.new(iterations: 1)
+    ) { |graded| graded.tokens < 100 }
+
+    fussy.via.should == :start
+    fussy.verdict(searched).failures.should == [
+      "called search",
+      "never called read",
+      'never said "sixty"',
+      "spent 2 iterations, budget 1",
+      "failed the case's own check",
+    ]
+
+    quiet = turn.new([], "", 1, 10, 1, nil, [])
+    Brute::Eval::Case.new("answers", said: "hi").verdict(quiet).failures.should == ["said nothing"]
+    Brute::Eval::Case.new("holds its tongue", silent: true).verdict(quiet).passed?.should.be.true
+
+    broken = turn.new([], "", 1, 10, 1, ArgumentError.new("no such agent file"), [])
+    Brute::Eval::Case.new("loads", said: "hi").verdict(broken).failures.first.should ==
+      "raised ArgumentError: no such agent file"
+
+    nothing_found = turn.new([], "The search turned up nothing about that.", 1, 10, 1, nil, [])
+    Brute::Eval::Case.new("admits it", said: "when does it ship?", absent: true)
+      .verdict(nothing_found).passed?.should.be.true
+    Brute::Eval::Case.new("admits it in French", said: "?", absent: true, absence: /rien/i)
+      .verdict(nothing_found).failures.should == ["did not say it had nothing"]
+  end
+end

data/lib/brute/eval/suite.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "brute"
+require "brute/eval/transcript"
+require "brute/eval/world"
+
+module Brute
+  module Eval
+    # Runs the cases against one agent and reports what happened.
+    #
+    #   Brute::Eval::Suite.new(agent: "agent.ru", cases: CASES).run
+    #   Brute::Eval::Suite.new(agent: -> { build_agent }, world: Room.new, cases: CASES).run
+    #
+    # The agent is built fresh for every attempt -- from its .ru file, or
+    # from a block that answers a new pipeline -- so nothing carries from one
+    # case to the next but the world, which is laid out again first. A case
+    # with `runs:` above one is run that many times and passes only if every
+    # run did: the model is not deterministic, and a case that passes two
+    # times in three is a case that fails.
+    #
+    # #run answers a process exit status, so an eval script ends `exit(...)`.
+    class Suite
+      Result = Data.define(:kase, :run, :transcript, :verdict)
+
+      def initialize(agent:, cases:, world: World.new, out: $stdout)
+        @agent = agent
+        @cases = cases
+        @world = world
+        @out = out
+      end
+
+      def run
+        results = @cases.flat_map { |kase| attempts(kase) }
+        summarise(results)
+
+        if results.all? { |result| result.verdict.passed? }
+          0
+        else
+          1
+        end
+      end
+
+      private
+
+        def attempts(kase)
+          (1..kase.runs).map do |run|
+            attempt(kase, run).tap { |result| report(result) }
+          end
+        end
+
+        def attempt(kase, run)
+          input = @world.prepare(kase)
+          transcript = Transcript.new
+          started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+          begin
+            agent = build
+            transcript.subscribe(agent)
+            @world.stub(agent, kase.stubs)
+            agent.start(input)
+          rescue StandardError => e
+            transcript.error = e
+          end
+
+          transcript.seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+          transcript.published = @world.published.dup
+
+          Result.new(
+            kase: kase,
+            run: run,
+            transcript: transcript,
+            verdict: kase.verdict(transcript)
+          )
+        end
+
+        def build
+          if @agent.respond_to?(:call)
+            @agent.call
+          else
+            Brute.load_agent(@agent)
+          end
+        end
+
+        def report(result)
+          transcript = result.transcript
+          budget = result.kase.budget
+
+          @out.puts
+          @out.puts "[#{outcome(result)}] #{result.kase.name}#{run_of(result)}"
+          @out.puts "  tools: #{transcript.counts}  errors: #{transcript.errors}"
+          @out.puts(
+            "  spent: #{transcript.iterations}/#{budget.iterations} iterations, " \
+            "#{transcript.calls.length}/#{budget.tool_calls} calls, " \
+            "#{transcript.tokens}/#{budget.tokens} tokens, " \
+            "#{transcript.seconds.round(1)}s"
+          )
+          result.verdict.failures.each { |failure| @out.puts "  - #{failure}" }
+          @out.puts "  said:"
+          transcript.reply.each_line { |line| @out.puts "    #{line.chomp}" }
+        end
+
+        def summarise(results)
+          cases = results.group_by { |result| result.kase }
+          passed = cases.count { |_kase, attempts| attempts.all? { |result| result.verdict.passed? } }
+
+          @out.puts
+          @out.puts "=== #{passed}/#{cases.length} cases passed ==="
+          @out.puts "tokens: #{results.sum { |result| result.transcript.tokens }}"
+          @out.puts "time: #{results.sum { |result| result.transcript.seconds }.round(1)}s"
+        end
+
+        def outcome(result)
+          if result.verdict.passed?
+            "PASS"
+          else
+            "FAIL"
+          end
+        end
+
+        def run_of(result)
+          if result.kase.runs == 1
+            ""
+          else
+            " (run #{result.run}/#{result.kase.runs})"
+          end
+        end
+    end
+  end
+end
+
+__END__
+
+require "stringio"
+require "tmpdir"
+
+describe "brute/eval/suite" do
+  it "runs every case against a freshly built agent and reports what happened" do
+    out = StringIO.new
+
+    suite = Brute::Eval::Suite.new(
+      agent: -> { Brute.agent.run(->(env) { env[:messages].assistant("it held at 4.25%") }) },
+      cases: [
+        Brute::Eval::Case.new("answers", said: "what did the bank do?", mentions: %w[4.25]),
+        Brute::Eval::Case.new("searches", said: "what did the bank do?", calls: { "search" => {} }),
+      ],
+      out: out
+    )
+
+    suite.run.should == 1
+    out.string.should.include "[PASS] answers"
+    out.string.should.include "[FAIL] searches"
+    out.string.should.include "- never called search"
+    out.string.should.include "=== 1/2 cases passed ==="
+
+    Dir.mktmpdir do |dir|
+      path = File.join(dir, "agent.ru")
+      File.write(path, 'run ->(env) { env[:messages].assistant("from the ru file") }')
+
+      loaded = Brute::Eval::Suite.new(
+        agent: path,
+        cases: [Brute::Eval::Case.new("loads a ru file", said: "hi", mentions: ["ru file"])],
+        out: out
+      )
+      loaded.run.should == 0
+
+      missing = Brute::Eval::Suite.new(
+        agent: File.join(dir, "nowhere.ru"),
+        cases: [Brute::Eval::Case.new("answers", said: "hi")],
+        out: out
+      )
+      missing.run.should == 1
+      out.string.should.include "raised ArgumentError"
+    end
+  end
+end

data/lib/brute/eval/transcript.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "brute"
+require "brute/env"
+
+module Brute
+  module Eval
+    # What one turn did.
+    #
+    # Every observation comes off the agent's own hooks, so a transcript is
+    # what the run itself reported: the tool calls in the order they were
+    # made, with the result each came back with, what the model finally said,
+    # what the provider charged for it, and how the call failed when it did.
+    # Nothing in the agent knows it is being watched.
+    #
+    #   transcript = Brute::Eval::Transcript.new
+    #   transcript.subscribe(agent)
+    #   agent.start("what changed?")
+    #
+    #   transcript.called?("search", "query" => /fed/i)
+    #   transcript.before?("read", "write")
+    class Transcript
+      attr_reader :calls, :usage, :failures
+      attr_accessor :seconds, :published, :error
+
+      def initialize
+        @calls = []
+        @usage = Hash.new(0)
+        @failures = []
+        @published = []
+        @env = {}
+        @seconds = 0.0
+      end
+
+      # The call env the tool pipeline hands its subscribers is one mutable
+      # hash per call, so the entry kept here at :before_tool carries the
+      # result the tool answered with by the time anyone reads it.
+      def subscribe(agent)
+        agent
+          .on(:before_tool) { |_env, call| @calls << call }
+          .on(:after_llm) { |env| account(env[:metadata][:last_llm_usage]) }
+          .on(:faraday_error) { |_env, error| @failures << describe(error) }
+          .on(:open_router_server_error) { |_env, error| @failures << describe(error) }
+          .on(:standard_error) { |_env, error| @failures << describe(error) }
+          .on(:turn_end) { |env| @env = env }
+      end
+
+      def names = @calls.map { |call| call[:name] }
+
+      def counts = names.tally
+
+      def iterations = @env[:current_iteration] || 0
+
+      def tokens = @usage[:total]
+
+      def errors = @calls.count { |call| call[:result].to_s.start_with?("Error") }
+
+      def reply
+        if @env[:messages].nil?
+          ""
+        else
+          @env.extend(Brute::Env).reply&.content.to_s
+        end
+      end
+
+      # A call the turn made, matched on name and on whatever arguments the
+      # case cares about -- `===`, so a case says `"query" => /fed/i` as
+      # readily as `"count" => 3`.
+      def called?(name, arguments = {})
+        @calls.any? { |call|
+          call[:name] == name.to_s &&
+            arguments.all? { |key, wanted| wanted === call[:arguments][key.to_s] }
+        }
+      end
+
+      def before?(first, second)
+        at = names.index(first.to_s)
+        then_at = names.index(second.to_s)
+
+        if at.nil?
+          false
+        else
+          then_at.nil? || at < then_at
+        end
+      end
+
+      private
+
+        def describe(error) = "#{error.class}: #{error.message}"
+
+        # Providers report what they report: a total that was never sent is
+        # not derived here, it is added up from the parts that were.
+        def account(usage)
+          if usage
+            @usage[:input] += usage.input.to_i
+            @usage[:output] += usage.output.to_i
+            @usage[:total] += usage.total || usage.input.to_i + usage.output.to_i
+          end
+        end
+    end
+  end
+end
+
+__END__
+
+describe "brute/eval/transcript" do
+  it "records what the turn called, what it answered, and what it cost" do
+    search = Brute::Turn::ToolPipeline.new(name: "search", description: "search the web") do
+      run ->(env) { env[:result] = "the bank held rates at 4.25%" }
+    end
+
+    replies = [
+      Brute::Message.new(
+        role: :assistant,
+        content: "",
+        tool_calls: [{ id: "1", name: "search", arguments: { "query" => "bank rate" } }]
+      ),
+      Brute::Message.new(role: :assistant, content: "It held at 4.25%."),
+    ]
+
+    agent = Brute.agent
+      .use(Brute::Middleware::Loop::ToolResult)
+      .use(Brute::Middleware::DefaultToolPipeline, tools: [search])
+      .run(->(env) { env[:messages] << replies.shift })
+
+    transcript = Brute::Eval::Transcript.new
+    transcript.subscribe(agent)
+    agent.start("what did the bank do?")
+
+    transcript.names.should == ["search"]
+    transcript.counts.should == { "search" => 1 }
+    transcript.called?("search", "query" => /bank/i).should.be.true
+    transcript.called?("search", "query" => /ecb/i).should.be.false
+    transcript.called?("fetch").should.be.false
+    transcript.before?("search", "fetch").should.be.true
+    transcript.before?("fetch", "search").should.be.false
+    transcript.reply.should == "It held at 4.25%."
+    transcript.iterations.should == 2
+    transcript.errors.should == 0
+    transcript.failures.should.be.empty
+
+    failed = Brute::Eval::Transcript.new
+    failed.reply.should == ""
+    failed.tokens.should == 0
+  end
+end

data/lib/brute/eval/world.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "brute"
+
+module Brute
+  module Eval
+    # The world a case wakes up in.
+    #
+    # This one keeps nothing: what was said is handed straight to the turn,
+    # and the tools answer from the case's stubs. It is what a plain agent
+    # needs, and it is the contract a deployment's own world implements --
+    # a world is anything that answers:
+    #
+    #   #prepare(case)      lay the world out for this case, and answer what
+    #                       the turn should be started with (nil when the
+    #                       world delivered what was said some other way, an
+    #                       inbox on disk say)
+    #   #stub(agent, stubs) install the case's canned tool results
+    #   #published          whatever the turn sent outward, for the record
+    #
+    # Subclass to give a case somewhere to wake up:
+    #
+    #   class Room < Brute::Eval::World
+    #     def prepare(kase)
+    #       super.tap { |input| inbox.append(kase.said) if input.nil? }
+    #     end
+    #   end
+    class World
+      attr_reader :published
+
+      def initialize
+        @published = []
+      end
+
+      def prepare(kase)
+        @published.clear
+        kase.said
+      end
+
+      # :before_tool is handed a mutable call env, and a :result set on it is
+      # answered without the tool ever running -- so a stub replaces the web,
+      # the calendar or the shell without the agent being built differently.
+      # A stub that answers to #call is handed the arguments.
+      def stub(agent, stubs)
+        agent.on(:before_tool) do |_env, call|
+          canned = stubs[call[:name]]
+
+          unless canned.nil?
+            if canned.respond_to?(:call)
+              call[:result] = canned.call(call[:arguments])
+            else
+              call[:result] = canned
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+__END__
+
+describe "brute/eval/world" do
+  it "hands what was said to the turn, and answers the tools from the case's stubs" do
+    world = Brute::Eval::World.new
+    kase = Brute::Eval::Case.new("asks", said: "what does it say?", stubs: { "search" => "canned" })
+
+    world.prepare(kase).should == "what does it say?"
+    world.published.should.be.empty
+
+    search = Brute::Turn::ToolPipeline.new(name: "search", description: "search") do
+      run ->(env) { env[:result] = "the live web" }
+    end
+
+    agent = Brute.agent
+      .use(Brute::Middleware::DefaultToolPipeline, tools: [search])
+      .run(
+        ->(env) {
+          env[:messages] << Brute::Message.new(
+            role: :assistant,
+            content: "",
+            tool_calls: [{ id: "1", name: "search", arguments: {} }]
+          )
+        }
+      )
+
+    world.stub(agent, kase.stubs)
+    agent.start("go")[:messages].last.content.should == "canned"
+
+    unstubbed = Brute.agent
+      .use(Brute::Middleware::DefaultToolPipeline, tools: [search])
+      .run(
+        ->(env) {
+          env[:messages] << Brute::Message.new(
+            role: :assistant,
+            content: "",
+            tool_calls: [{ id: "1", name: "search", arguments: {} }]
+          )
+        }
+      )
+
+    world.stub(unstubbed, {})
+    unstubbed.start("go")[:messages].last.content.should == "the live web"
+  end
+end