brute 5.0.5 → 6.0.0

data/lib/brute/eval.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "brute"
+
+require_relative "eval/case"
+require_relative "eval/suite"
+require_relative "eval/transcript"
+require_relative "eval/world"
+
+module Brute
+  # Evaluating an agent, rather than testing a middleware.
+  #
+  # A spec asks whether one layer does what it says. An eval asks whether the
+  # whole assembled agent -- its system prompt, its tools, the model behind
+  # it -- behaves. That answer is not a boolean about code: it is a real turn,
+  # against a real model, graded on what the turn DID.
+  #
+  #   CASES = [
+  #     Brute::Eval::Case.new(
+  #       "searches for what it cannot know",
+  #       said:     "what did the Bank of England do yesterday?",
+  #       stubs:    { "search" => RATE_DECISION },
+  #       calls:    { "search" => { "query" => /bank|rate/i } },
+  #       mentions: %w[4.25],
+  #     ),
+  #     Brute::Eval::Case.new(
+  #       "does not search for what it already knows",
+  #       said:   "how many minutes are there in an hour?",
+  #       never:  %w[search],
+  #       budget: Brute::Eval::Budget.new(iterations: 2, tool_calls: 0),
+  #     ),
+  #   ]
+  #
+  #   exit(Brute::Eval::Suite.new(agent: "agent.ru", cases: CASES).run)
+  #
+  # Everything the harness sees comes off the agent's own hooks -- the same
+  # registry any other subscriber uses -- so the agent under evaluation is the
+  # agent that ships: no eval-only middleware, no branch in agent.ru. The
+  # model and the tool schemas are real; the tools themselves answer from the
+  # case's stubs, installed on :tool_start, which answers a call without
+  # executing it.
+  #
+  # Where a case wakes up is the World's business, and a deployment that
+  # delivers what was said through an inbox, a queue or a room subclasses it.
+  module Eval
+  end
+end

data/lib/brute/events/handler.rb

@@ -18,7 +18,9 @@ module Brute
       # thing, then call super (or don't, to swallow the event).
       def <<(event)
         tap do
-          @inner << event if @inner
+          if @inner
+            @inner << event
+          end
         end
       end
     end

data/lib/brute/events/prefixed_terminal_output.rb

@@ -29,7 +29,9 @@ module Brute
         data = event.to_h[:data]
 
         method = "on_#{type}"
-        send(method, data) if respond_to?(method, true)
+        if respond_to?(method, true)
+          send(method, data)
+        end
 
         super
       end

data/lib/brute/events/terminal_output_handler.rb

@@ -15,7 +15,7 @@ module Brute
         method = "on_#{type}"
 
         if respond_to?(method, true)
-          send(method, data) 
+          send(method, data)
         end
 
         super

data/lib/brute/hooks.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 require "bundler/setup"
+require "delegate"
+require "securerandom"
+
 require "brute"
 
 module Brute
@@ -9,76 +12,69 @@ module Brute
   #   Brute.agent
   #     .use(Brute::Middleware::MaxIterations)
   #     .run(->(env) { env[:messages].assistant("done") })
-  #     .on(:before_llm) { |env| ... }
-  #     .on(:approve_tool) { |_env, call| call[:name] != "exec" }
-  #
-  # Every subscriber is called with the turn env first, followed by whatever
-  # extras that event carries. A block that only wants the env can take one
-  # argument and ignore the rest.
-  #
-  # Emission points and payloads:
+  #     .on(:llm_start) { |env| ... }
+  #     .on(:tool_approve) { |_env, call| call[:name] != "exec" }
   #
-  #   :turn_start, :turn_end  → the turn env (AgentPipeline#start; turn_end
-  #                             fires from an ensure, so it also fires on
-  #                             error)
-  #   :turn_duration          → env, started, finished: the turn's work is
-  #                             this event's block
-  #   :middleware_added       → an empty env, then the middleware and every
-  #                             argument `use` was given (fires at build time,
-  #                             so only subscribers registered before the `use`
-  #                             see it)
-  #   :enter                  → env, the middleware instance, before the
-  #                             layer does anything
-  #   :duration               → env, started, finished, the middleware
-  #                             instance: the layer's work is this event's
-  #                             block, so it reports how long that took
-  #   :exit                   → env, the middleware instance, marking the
-  #                             layer done (from an ensure, so it fires on
-  #                             error too)
-  #   :before_llm, :after_llm → the turn env, around every LLM call
-  #   :llm_duration           → env, started, finished: the provider call is
-  #                             this event's block
-  #   :llm_failure            → the turn env, when the LLM call raises; the
-  #                             completion middleware then emits one of
-  #                             :faraday_error, :open_router_server_error or
-  #                             :standard_error with the exception as an extra
-  #   :before_tool            → env, call env {name:, arguments:, result:,
-  #                             denied:, events:, metadata:, turn_env:} —
-  #                             mutate :arguments to rewrite the call, or set
-  #                             :result to answer it without executing
-  #   :approve_tool           → env, call env — set :denied to true to deny
-  #                             the call, or to a String to deny it with that
-  #                             message
-  #   :tool_duration          → env, started, finished, call env: the tool's
-  #                             own execution is this event's block, so a
-  #                             skipped or denied call never fires it
-  #   :after_tool             → env, call env — mutate :result
-  #
-  # Subscribers run inline (tool events may fire from parallel threads).
-  # Exceptions propagate to the caller — layers that want fail-open semantics
-  # rescue in their own subscriber.
-  # Include this in anything that emits or subscribes and the event names are
-  # first class there: ENTER_EVENT rather than Brute::Hooks::ENTER_EVENT.
-  # The registry itself is Hooks::Registry, and Brute::Hooks.new builds one.
   module Hooks
     TURN_START_EVENT = :turn_start
     TURN_DURATION_EVENT = :turn_duration
     TURN_END_EVENT = :turn_end
+    TURN_FAILURE_EVENT = :turn_failure
     MIDDLEWARE_ADDED_EVENT = :middleware_added
-    ENTER_EVENT = :enter
-    DURATION_EVENT = :duration
-    EXIT_EVENT = :exit
-    BEFORE_LLM_EVENT = :before_llm
+    MIDDLEWARE_START_EVENT = :middleware_start
+    MIDDLEWARE_DURATION_EVENT = :middleware_duration
+    MIDDLEWARE_END_EVENT = :middleware_end
+    MIDDLEWARE_FAILURE_EVENT = :middleware_failure
+    LLM_START_EVENT = :llm_start
     LLM_DURATION_EVENT = :llm_duration
-    AFTER_LLM_EVENT = :after_llm
+    LLM_END_EVENT = :llm_end
     LLM_FAILURE_EVENT = :llm_failure
     FARADAY_ERROR_EVENT = :faraday_error
     OPEN_ROUTER_SERVER_ERROR_EVENT = :open_router_server_error
     STANDARD_ERROR_EVENT = :standard_error
-    BEFORE_TOOL_EVENT = :before_tool
-    APPROVE_TOOL_EVENT = :approve_tool
+    COMPACT_START_EVENT = :compact_start
+    COMPACT_DURATION_EVENT = :compact_duration
+    COMPACT_END_EVENT = :compact_end
+    COMPACT_FAILURE_EVENT = :compact_failure
+    CONTENT_EVENT = :content
+    REASONING_EVENT = :reasoning
+    TOOL_CALLS_EVENT = :tool_calls
+    TOOL_START_EVENT = :tool_start
+    TOOL_APPROVE_EVENT = :tool_approve
     TOOL_DURATION_EVENT = :tool_duration
-    AFTER_TOOL_EVENT = :after_tool
+    TOOL_END_EVENT = :tool_end
+    TOOL_FAILURE_EVENT = :tool_failure
+
+    # The env, wrapped, for as long as one call lasts. Everything emitted
+    # through it carries that call's id, and a call opened inside another
+    # delegates to it -- so the wrapper is the parent and unwrapping ends the
+    # call. It is still the env: SimpleDelegator forwards the rest.
+    class Trace < SimpleDelegator
+      attr_reader :id, :hooks
+
+      def initialize(env, hooks: nil, id: SecureRandom.uuid)
+        super(env)
+        if hooks
+          @hooks = hooks
+        elsif env.is_a?(Trace)
+          @hooks = env.hooks
+        end
+        @id = id
+      end
+
+      def current_trace = self
+
+      def emit(event, *extras, &work) = @hooks.emit(
+        event,
+        self,
+        *extras,
+        @id,
+        &work
+      )
+
+      def emit_trace(&block) = self.class.new(self).tap(&block).then { |trace| trace.__getobj__ }
+    end
+
 
     # The pub/sub registry a pipeline owns; `use` and `run` bind an emit to it.
     class Registry
@@ -91,40 +87,33 @@ module Brute
         self
       end
 
-      # Fire an event. An emitter announces; it answers nothing, and what a
-      # subscriber's block happens to evaluate to is not a signal. A layer
-      # that wants to take part in a turn does it by mutating what it was
-      # handed, never by returning something.
-      #
-      # Given a block, the event is timed instead: the block is the work, and
-      # subscribers fire once it is done, called as
-      # `|env, started, finished, *extras|` rather than `|env, *extras|`.
-      # Both stamps are monotonic, so a clock adjustment mid-turn cannot
-      # produce a negative duration.
-      #
       # The block is the work and nothing more: `emit` answers nothing in
       # either form, so a caller that needs the work's value takes it inside
       # the block.
       #
       #   result = nil
-      #   emit(DURATION_EVENT, env, self) { result = @app.call(env) }
-      #   # => .on(DURATION_EVENT) { |env, started, finished, layer| ... }
+      #   emit(MIDDLEWARE_DURATION_EVENT, env, self) { result = @app.call(env) }
+      #   # => .on(MIDDLEWARE_DURATION_EVENT) { |env, started, finished, layer| ... }
       #
       # Subscribers fire from an ensure, so work that raises is still timed
       # and still reported before the exception carries on up.
       def emit(event, env, *extras, &block)
-        unless block
+        if block
+          started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+          begin
+            block.call
+          ensure
+            finished = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            @subscribers[event.to_sym].each { |subscriber| subscriber.call(
+              env,
+              started,
+              finished,
+              *extras,
+            ) }
+          end
+        else
           @subscribers[event.to_sym].each { |subscriber| subscriber.call(env, *extras) }
-          return nil
-        end
-
-        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-
-        begin
-          block.call
-        ensure
-          finished = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-          @subscribers[event.to_sym].each { |subscriber| subscriber.call(env, started, finished, *extras) }
         end
 
         nil
@@ -132,8 +121,6 @@ module Brute
 
       def any?(event) = @subscribers[event.to_sym].any?
     end
-
-    def self.new(...) = Registry.new(...)
   end
 end
 
@@ -141,38 +128,38 @@ __END__
 
 describe "brute/hooks" do
   it "emits to subscribers in registration order" do
-    hooks = Brute::Hooks.new
+    hooks = Brute::Hooks::Registry.new
     seen = []
-    hooks.on(:before_llm) { |env| seen << "a#{env}" }
-    hooks.on(:before_llm) { |env| seen << "b#{env}" }
-    hooks.emit(:before_llm, 1)
+    hooks.on(:llm_start) { |env| seen << "a#{env}" }
+    hooks.on(:llm_start) { |env| seen << "b#{env}" }
+    hooks.emit(:llm_start, 1)
     seen.should == ["a1", "b1"]
   end
 
   it "answers nothing: a subscriber takes part by mutating, not by returning" do
-    hooks = Brute::Hooks.new
-    hooks.on(:approve_tool) { |_env, call| call[:denied] = true }
+    hooks = Brute::Hooks::Registry.new
+    hooks.on(:tool_approve) { |_env, call| call[:denied] = true }
 
     call = {}
-    hooks.emit(:approve_tool, {}, call).should.be.nil
+    hooks.emit(:tool_approve, {}, call).should.be.nil
     call[:denied].should.be.true
   end
 
   it "hands every subscriber the env first and the extras after" do
-    hooks = Brute::Hooks.new
+    hooks = Brute::Hooks::Registry.new
     seen = []
-    hooks.on(:after_tool) { |env, call| seen << [env, call] }
-    hooks.emit(:after_tool, :turn, :call)
+    hooks.on(:tool_end) { |env, call| seen << [env, call] }
+    hooks.emit(:tool_end, :turn, :call)
     seen.should == [[:turn, :call]]
   end
 
   it "times a block, hands subscribers start and finish, and reports even when the work raises" do
     seen = []
-    hooks = Brute::Hooks.new
-    hooks.on(:exit) { |env, started, finished, subject| seen << [env, started, finished, subject] }
+    hooks = Brute::Hooks::Registry.new
+    hooks.on(:middleware_end) { |env, started, finished, subject| seen << [env, started, finished, subject] }
 
     ran = nil
-    hooks.emit(:exit, :env, :layer) { ran = :work_result }.should.be.nil
+    hooks.emit(:middleware_end, :env, :layer) { ran = :work_result }.should.be.nil
     ran.should == :work_result
     seen.size.should == 1
     env, started, finished, subject = seen.first
@@ -181,19 +168,19 @@ describe "brute/hooks" do
     (finished - started).should.be >= 0
 
     # Work that raises is still timed and still reported.
-    should.raise(RuntimeError) { hooks.emit(:exit, :env, :layer) { raise "boom" } }
+    should.raise(RuntimeError) { hooks.emit(:middleware_end, :env, :layer) { raise "boom" } }
     seen.size.should == 2
 
     # Without a block it stays a point event: no timing argument.
     args = []
-    hooks2 = Brute::Hooks.new
-    hooks2.on(:exit) { |*received| args << received }
-    hooks2.emit(:exit, :env, :layer)
+    hooks2 = Brute::Hooks::Registry.new
+    hooks2.on(:middleware_end) { |*received| args << received }
+    hooks2.emit(:middleware_end, :env, :layer)
     args.should == [[:env, :layer]]
   end
 
   it "answers any? and stays chainable" do
-    hooks = Brute::Hooks.new
+    hooks = Brute::Hooks::Registry.new
     hooks.any?(:turn_start).should.be.false
     hooks.on(:turn_start) { nil }.should.equal?(hooks)
     hooks.any?(:turn_start).should.be.true

data/lib/brute/message_transport/anthropic.rb

@@ -55,7 +55,9 @@ module Brute
         when :assistant
           if message.tool_call?
             blocks = []
-            blocks << { type: "text", text: message.content } unless message.content.to_s.empty?
+            unless message.content.to_s.empty?
+              blocks << { type: "text", text: message.content }
+            end
             blocks += message.tool_calls.map { |tc| { type: "tool_use", id: tc.id, name: tc.name, input: tc.arguments } }
             { role: "assistant", content: blocks }
           else
@@ -79,14 +81,22 @@ module Brute
 
           text = blocks.select { |b| b.type == :text }.map(&:text).join
           tool_calls = blocks.select { |b| b.type == :tool_use }.map do |b|
-            arguments = b.input.respond_to?(:to_h) ? b.input.to_h : b.input
+            if b.input.respond_to?(:to_h)
+              arguments = b.input.to_h
+            else
+              arguments = b.input
+            end
             Brute::ToolCall.new(id: b.id, name: b.name, arguments: arguments)
           end
 
+          if tool_calls.empty?
+            tool_calls = nil
+          end
+
           Brute::Message.new(
             role:       :assistant,
             content:    text,
-            tool_calls: (tool_calls unless tool_calls.empty?),
+            tool_calls: tool_calls,
           )
         end
     end

data/lib/brute/message_transport/llm.rb

@@ -39,8 +39,11 @@ module Brute
             wire = message.tool_calls.map do |tc|
               { id: tc.id, type: "function", function: { name: tc.name, arguments: JSON.generate(tc.arguments) } }
             end
-            ::LLM::Message.new(:assistant, message.content.to_s,
-                               { tool_calls: wire, original_tool_calls: wire })
+            ::LLM::Message.new(
+              :assistant,
+              message.content.to_s,
+              { tool_calls: wire, original_tool_calls: wire },
+            )
           else
             ::LLM::Message.new(:assistant, message.content)
           end
@@ -53,8 +56,9 @@ module Brute
 
         # LLM::Message (from an LLM::Response) -> Brute::Message.
         def wrap(message)
-          tool_calls = if message.tool_call?
-            message.tool_calls.map do |tc|
+          tool_calls = nil
+          if message.tool_call?
+            tool_calls = message.tool_calls.map do |tc|
               Brute::ToolCall.new(id: tc.id, name: tc["name"], arguments: tc.arguments.to_h)
             end
           end

data/lib/brute/message_transport/open_router.rb

@@ -26,9 +26,11 @@ module Brute
       # An OpenRouter::Response's messages (one per choice; in practice
       # OpenRouter returns exactly one).
       def messages
-        return @result.choices.map { |choice| choice["message"] || choice[:message] } if @result.respond_to?(:choices)
-
-        super
+        if @result.respond_to?(:choices)
+          @result.choices.map { |choice| choice["message"] || choice[:message] }
+        else
+          super
+        end
       end
 
       private
@@ -36,14 +38,25 @@ module Brute
         def wrap(message)
           # Coerce string keys to symbol keys if necessary
           hash = message.to_h.transform_keys(&:to_sym)
-          hash[:role] = hash[:role].to_sym if hash.key?(:role)
-          hash[:tool_calls] = hash[:tool_calls].map { |tc| wrap_tool_call(tc) } if hash[:tool_calls]
+          if hash.key?(:role)
+            hash[:role] = hash[:role].to_sym
+          end
+          if hash[:tool_calls]
+            hash[:tool_calls] = hash[:tool_calls].map { |tc| wrap_tool_call(tc) }
+          end
 
           case hash
           in { role: (:system | :user | :assistant | :tool) }
             # Slice away provider extras (refusal, reasoning, model, ...)
             # that Brute::Message doesn't know.
-            Brute::Message.new(**hash.slice(:role, :content, :tool_calls, :tool_call_id))
+            Brute::Message.new(
+              **hash.slice(
+                :role,
+                :content,
+                :tool_calls,
+                :tool_call_id,
+              ),
+            )
           else
             raise "Unrecognised message format #{message.inspect}"
           end
@@ -53,16 +66,17 @@ module Brute
         # -> the flat { id:, name:, arguments: Hash } Brute::Message understands.
         def wrap_tool_call(tool_call)
           tc = tool_call.to_h.transform_keys(&:to_sym)
-          return tc unless tc[:function] # already flat { id:, name:, arguments: }
-
-          function = tc[:function].to_h.transform_keys(&:to_sym)
-          arguments = function[:arguments].to_s
-
-          {
-            id: tc[:id],
-            name: function[:name],
-            arguments: JSON.parse(arguments.empty? ? "{}" : arguments),
-          }
+          if tc[:function]
+            function = tc[:function].to_h.transform_keys(&:to_sym)
+            arguments = function[:arguments].to_s
+            {
+              id:        tc[:id],
+              name:      function[:name],
+              arguments: JSON.parse(arguments.empty? ? "{}" : arguments),
+            }
+          else
+            tc
+          end
         end
     end
   end

data/lib/brute/message_transport/openai.rb

@@ -28,9 +28,14 @@ module Brute
           { role: "tool", tool_call_id: message.tool_call_id, content: message.content.to_s }
         when :assistant
           if message.tool_call?
+            content = message.content
+            if content.to_s.empty?
+              content = nil
+            end
+
             {
               role:       "assistant",
-              content:    (message.content unless message.content.to_s.empty?),
+              content:    content,
               tool_calls: message.tool_calls.map { |tc|
                 { id: tc.id, type: "function", function: { name: tc.name, arguments: JSON.generate(tc.arguments) } }
               },
@@ -45,9 +50,11 @@ module Brute
 
       # A chat completion response's messages (one per choice).
       def messages
-        return @result.choices.map(&:message) if @result.respond_to?(:choices)
-
-        super
+        if @result.respond_to?(:choices)
+          @result.choices.map(&:message)
+        else
+          super
+        end
       end
 
       private
@@ -56,10 +63,10 @@ module Brute
         # arguments arrive as a JSON string; parse them into a Hash.
         def wrap(message)
           tool_calls = message.tool_calls&.map do |tc|
-            arguments = begin
-              JSON.parse(tc.function.arguments.to_s)
+            begin
+              arguments = JSON.parse(tc.function.arguments.to_s)
             rescue JSON::ParserError
-              {}
+              arguments = {}
             end
             Brute::ToolCall.new(id: tc.id, name: tc.function.name, arguments: arguments)
           end

data/lib/brute/message_transport/ruby_llm.rb

@@ -24,32 +24,36 @@ module Brute
           role:         message.role,
           content:      message.content,
           tool_calls:   tool_calls,
-          tool_call_id: message.tool_call_id
+          tool_call_id: message.tool_call_id,
         )
       end
 
       private
 
       # RubyLLM::Message -> Brute::Message.
-      def wrap(message)
-        raw_calls = message.tool_calls
-        calls_list = raw_calls.respond_to?(:values) ? raw_calls.values : raw_calls
+        def wrap(message)
+          raw_calls = message.tool_calls
+          if raw_calls.respond_to?(:values)
+            calls_list = raw_calls.values
+        else
+          calls_list = raw_calls
+        end
+
+          tool_calls = calls_list&.map do |tc|
+            Brute::ToolCall.new(
+              id:        tc.id,
+              name:      tc.name,
+              arguments: tc.arguments,
+            )
+          end
 
-        tool_calls = calls_list&.map do |tc|
-          Brute::ToolCall.new(
-            id:        tc.id,
-            name:      tc.name,
-            arguments: tc.arguments
+          Brute::Message.new(
+            role:         message.role,
+            content:      message.content&.to_s, # Preserves nil safely
+            tool_calls:   tool_calls,
+            tool_call_id: message.tool_call_id,
           )
         end
-
-        Brute::Message.new(
-          role:         message.role,
-          content:      message.content&.to_s, # Preserves nil safely
-          tool_calls:   tool_calls,
-          tool_call_id: message.tool_call_id
-        )
-      end
     end
   end
 end